From b89e0445f9b93cce331fee633a15cd9ea9dd7675 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Thu, 13 Aug 2026 14:11:11 +0800 Subject: [PATCH 01/45] branch-4.1: add external metadata cache memory governance --- fe/fe-benchmark/pom.xml | 42 + .../run-external-meta-cache-size-benchmark.sh | 56 + .../doris/benchmark/BenchmarkHarness.java | 101 ++ .../HivePartitionValuesSizeBenchmark.java | 295 ++++ .../iceberg/IcebergCacheSizeBenchmark.java | 411 ++++++ .../MetaCacheSoftValueBenchmark.java | 175 +++ .../paimon/PaimonCacheSizeBenchmark.java | 272 ++++ .../java/org/apache/doris/common/Config.java | 5 + .../org/apache/doris/common/CacheFactory.java | 37 +- .../doris/datasource/ExternalCatalog.java | 30 +- .../datasource/ExternalMetaCacheMgr.java | 159 +- .../doris/DorisExternalMetaCache.java | 7 +- .../datasource/hive/HMSExternalCatalog.java | 6 - .../hive/HiveCacheSizeEstimator.java | 54 + .../hive/HiveExternalMetaCache.java | 287 +++- .../hudi/HudiExternalMetaCache.java | 7 +- .../iceberg/IcebergCacheSizeEstimator.java | 254 ++++ .../iceberg/IcebergExternalMetaCache.java | 221 ++- .../iceberg/IcebergExternalTable.java | 4 + .../iceberg/IcebergMetadataOps.java | 36 +- .../datasource/iceberg/IcebergPartition.java | 34 + .../iceberg/IcebergPartitionInfo.java | 43 +- .../iceberg/IcebergSnapshotCacheValue.java | 526 ++++++- .../iceberg/IcebergSnapshotEntryKey.java | 115 ++ .../iceberg/IcebergTableCacheValue.java | 85 +- .../iceberg/IcebergTransaction.java | 2 +- .../datasource/iceberg/IcebergUtils.java | 34 +- .../IcebergCherrypickSnapshotAction.java | 2 +- .../action/IcebergExpireSnapshotsAction.java | 2 +- .../action/IcebergFastForwardAction.java | 2 +- .../action/IcebergPublishChangesAction.java | 2 +- .../action/IcebergRewriteManifestsAction.java | 2 +- .../IcebergRollbackToSnapshotAction.java | 2 +- .../IcebergRollbackToTimestampAction.java | 2 +- .../IcebergSetCurrentSnapshotAction.java | 2 +- .../iceberg/cache/ManifestCacheValue.java | 174 ++- .../iceberg/source/IcebergScanNode.java | 5 +- .../MaxComputeExternalMetaCache.java | 8 +- .../metacache/AbstractExternalMetaCache.java | 111 +- .../doris/datasource/metacache/CacheSpec.java | 223 ++- .../metacache/CatalogEntryGroup.java | 8 + .../metacache/ExternalMetaCache.java | 4 + .../ExternalMetaCacheBudgetManager.java | 414 ++++++ .../datasource/metacache/MetaCacheEntry.java | 1060 ++++++++++++- .../metacache/MetaCacheEntryDef.java | 23 +- .../metacache/MetaCacheEntryStats.java | 72 +- .../metacache/MetaCacheSizeEstimate.java | 64 + .../metacache/MetaCacheSizeEstimator.java | 46 + .../metacache/MetaCacheWeightUtils.java | 70 + .../paimon/PaimonCacheSizeEstimator.java | 193 +++ .../paimon/PaimonExternalMetaCache.java | 44 +- .../paimon/PaimonPartitionInfo.java | 52 + .../paimon/PaimonSnapshotCacheValue.java | 23 + .../paimon/PaimonSnapshotEntryKey.java | 80 + .../paimon/PaimonTableCacheValue.java | 26 +- .../doris/datasource/paimon/PaimonUtil.java | 9 +- .../ExternalMetaCacheRouteResolverTest.java | 124 ++ .../hive/HiveMetaStoreCacheTest.java | 119 ++ .../iceberg/IcebergDDLAndDMLPlanTest.java | 2 + .../iceberg/IcebergExternalMetaCacheTest.java | 894 ++++++++++- .../IcebergExternalTableBranchAndTagTest.java | 4 +- .../IcebergMetadataOpsValidationTest.java | 74 +- .../iceberg/IcebergPartitionInfoTest.java | 23 + .../iceberg/IcebergTransactionTest.java | 42 +- .../AbstractExternalMetaCacheTest.java | 192 +++ .../datasource/metacache/CacheSpecTest.java | 56 + .../ExternalMetaCacheBudgetManagerTest.java | 191 +++ .../metacache/MetaCacheEntryTest.java | 1310 ++++++++++++++++- .../paimon/PaimonExternalMetaCacheTest.java | 327 +++- .../datasource/paimon/PaimonUtilTest.java | 36 + fe/pom.xml | 6 + 71 files changed, 9035 insertions(+), 388 deletions(-) create mode 100644 fe/fe-benchmark/pom.xml create mode 100755 fe/fe-benchmark/run-external-meta-cache-size-benchmark.sh create mode 100644 fe/fe-benchmark/src/main/java/org/apache/doris/benchmark/BenchmarkHarness.java create mode 100644 fe/fe-benchmark/src/main/java/org/apache/doris/datasource/hive/HivePartitionValuesSizeBenchmark.java create mode 100644 fe/fe-benchmark/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeBenchmark.java create mode 100644 fe/fe-benchmark/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSoftValueBenchmark.java create mode 100644 fe/fe-benchmark/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeBenchmark.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotEntryKey.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimate.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotEntryKey.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java diff --git a/fe/fe-benchmark/pom.xml b/fe/fe-benchmark/pom.xml new file mode 100644 index 00000000000000..2de619284a9f69 --- /dev/null +++ b/fe/fe-benchmark/pom.xml @@ -0,0 +1,42 @@ + + + + 4.0.0 + + + org.apache.doris + fe + ${revision} + ../pom.xml + + + fe-benchmark + Doris FE Benchmarks + + + + ${project.groupId} + fe-core + ${project.version} + + + diff --git a/fe/fe-benchmark/run-external-meta-cache-size-benchmark.sh b/fe/fe-benchmark/run-external-meta-cache-size-benchmark.sh new file mode 100755 index 00000000000000..12e28e0dd9a7a4 --- /dev/null +++ b/fe/fe-benchmark/run-external-meta-cache-size-benchmark.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash + +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +set -euo pipefail + +BENCHMARK_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd) +FE_DIR=$(cd -- "${BENCHMARK_DIR}/.." && pwd) +CLASSPATH_FILE=$(mktemp) +trap 'rm -f "${CLASSPATH_FILE}"' EXIT + +( + cd "${FE_DIR}" + mvn -Pbenchmark -pl fe-benchmark -am compile -DskipTests -Dskip.clean=true + mvn -Pbenchmark -pl fe-benchmark -am dependency:build-classpath \ + -Dskip.clean=true \ + -DincludeScope=test \ + -Dmdep.outputFile="${CLASSPATH_FILE}" +) + +REACTOR_CLASSES=$(find "${FE_DIR}" -type d -path '*/target/classes' -printf '%p:') +DEPENDENCY_CLASSES=$(tr -d '\n' < "${CLASSPATH_FILE}") +BENCHMARK_FILTER=${BENCHMARK_FILTER:-'HivePartitionValuesSizeBenchmark|IcebergCacheSizeBenchmark|PaimonCacheSizeBenchmark|MetaCacheSoftValueBenchmark'} + +BENCHMARK_CLASSES=( + org.apache.doris.datasource.hive.HivePartitionValuesSizeBenchmark + org.apache.doris.datasource.iceberg.IcebergCacheSizeBenchmark + org.apache.doris.datasource.paimon.PaimonCacheSizeBenchmark + org.apache.doris.datasource.metacache.MetaCacheSoftValueBenchmark +) + +for BENCHMARK_CLASS in "${BENCHMARK_CLASSES[@]}"; do + if [[ "${BENCHMARK_CLASS##*.}" =~ ${BENCHMARK_FILTER} ]]; then + java \ + -Xms1g \ + -Xmx4g \ + -classpath "${REACTOR_CLASSES}${DEPENDENCY_CLASSES}" \ + "${BENCHMARK_CLASS}" \ + "$@" + fi +done diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/benchmark/BenchmarkHarness.java b/fe/fe-benchmark/src/main/java/org/apache/doris/benchmark/BenchmarkHarness.java new file mode 100644 index 00000000000000..c1e9234c70fa67 --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/benchmark/BenchmarkHarness.java @@ -0,0 +1,101 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.benchmark; + +import java.util.Locale; +import java.util.concurrent.TimeUnit; + +/** Small dependency-free harness for opt-in FE microbenchmarks. */ +public final class BenchmarkHarness { + private static final long WARMUP_MILLIS = Long.getLong("benchmark.warmup.millis", 500L); + private static final long MEASUREMENT_MILLIS = Long.getLong("benchmark.measurement.millis", 500L); + private static final int MEASUREMENT_ITERATIONS = Integer.getInteger("benchmark.iterations", 3); + private static final boolean PRINT_RESULT = Boolean.getBoolean("benchmark.print.result"); + private static volatile Object sink; + + private BenchmarkHarness() { + } + + @FunctionalInterface + public interface Operation { + Object run() throws Exception; + } + + public static void measure(String name, TimeUnit outputUnit, Operation operation) throws Exception { + runWindow(operation, WARMUP_MILLIS); + double totalNanosPerOperation = 0.0D; + long totalOperations = 0L; + for (int iteration = 0; iteration < MEASUREMENT_ITERATIONS; iteration++) { + Window result = runWindow(operation, MEASUREMENT_MILLIS); + totalNanosPerOperation += result.nanosPerOperation; + totalOperations += result.operations; + } + double averageNanos = totalNanosPerOperation / MEASUREMENT_ITERATIONS; + String result = PRINT_RESULT ? ", result=" + sink : ""; + System.out.printf(Locale.ROOT, "%-72s %12.3f %s/op (%d ops%s)%n", + name, convertFromNanos(averageNanos, outputUnit), unitName(outputUnit), totalOperations, result); + } + + private static Window runWindow(Operation operation, long minimumMillis) throws Exception { + long start = System.nanoTime(); + long deadline = start + TimeUnit.MILLISECONDS.toNanos(minimumMillis); + long operations = 0L; + do { + sink = operation.run(); + operations++; + } while (System.nanoTime() < deadline); + long elapsed = System.nanoTime() - start; + return new Window(operations, (double) elapsed / operations); + } + + private static double convertFromNanos(double nanos, TimeUnit outputUnit) { + if (outputUnit == TimeUnit.NANOSECONDS) { + return nanos; + } else if (outputUnit == TimeUnit.MICROSECONDS) { + return nanos / 1_000.0D; + } else if (outputUnit == TimeUnit.MILLISECONDS) { + return nanos / 1_000_000.0D; + } else if (outputUnit == TimeUnit.SECONDS) { + return nanos / 1_000_000_000.0D; + } + throw new IllegalArgumentException("unsupported benchmark time unit: " + outputUnit); + } + + private static String unitName(TimeUnit outputUnit) { + if (outputUnit == TimeUnit.NANOSECONDS) { + return "ns"; + } else if (outputUnit == TimeUnit.MICROSECONDS) { + return "us"; + } else if (outputUnit == TimeUnit.MILLISECONDS) { + return "ms"; + } else if (outputUnit == TimeUnit.SECONDS) { + return "s"; + } + return outputUnit.name().toLowerCase(Locale.ROOT); + } + + private static final class Window { + private final long operations; + private final double nanosPerOperation; + + private Window(long operations, double nanosPerOperation) { + this.operations = operations; + this.nanosPerOperation = nanosPerOperation; + } + } +} diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/hive/HivePartitionValuesSizeBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/hive/HivePartitionValuesSizeBenchmark.java new file mode 100644 index 00000000000000..7e3acd9e2f97ac --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/hive/HivePartitionValuesSizeBenchmark.java @@ -0,0 +1,295 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.hive; + +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.benchmark.BenchmarkHarness; +import org.apache.doris.catalog.ListPartitionItem; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.Type; +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.hive.HiveExternalMetaCache.HivePartitionValues; +import org.apache.doris.datasource.hive.HiveExternalMetaCache.PartitionValueCacheKey; +import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.EntryBudget; +import org.apache.doris.datasource.metacache.MetaCacheEntry; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; + +import com.google.common.collect.HashBiMap; +import com.google.common.collect.Maps; +import com.google.common.util.concurrent.MoreExecutors; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.OptionalLong; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; + +/** Measures count publication, weighted preparation, and prepared-value admission separately. */ +public class HivePartitionValuesSizeBenchmark { + private static final int TAIL_PAYLOAD_BYTES = 1024 * 1024; + private static final long MAX_WEIGHT_BYTES = 4L * 1024L * 1024L * 1024L; + + public int countPublicationBaseline(UnsealedState state) { + state.partitionValues.rebuildSortedPartitionRangesForPublication(); + return state.partitionValues.getSortedPartitionRanges() + .map(ranges -> ranges.sortedPartitions.size() + ranges.defaultPartitions.size()) + .orElse(0); + } + + public int sealPublicationWithoutEstimate(UnsealedState state) { + state.partitionValues.sealForPublication(); + return state.partitionValues.getIdToPartitionItem().size(); + } + + public long weightedPublication(UnsealedState state) { + state.partitionValues.rebuildSortedPartitionRangesForPublication(); + state.partitionValues.prepareForCachePublication(state.key); + return requireComplete(state.partitionValues); + } + + public long eventCopySealAndEstimate(PreparedState state) { + HivePartitionValues copy = state.partitionValues.mutableCopy(); + copy.rebuildSortedPartitionRangesForPublication(); + copy.prepareForCachePublication(state.key); + return requireComplete(copy); + } + + public long preparedSizeProvider(PreparedState state) { + return state.partitionValues.getSizeEstimate().getBytes(); + } + + public long estimateFormula(PreparedState state) { + state.partitionValues.prepareSizeEstimate(state.key); + return requireComplete(state.partitionValues); + } + + public void replacementAdmission(PreparedState state) { + MetaCacheEntry.ReplaceResult result = state.cacheEntry.tryReplace( + state.key, state.currentPartitionValues, state.nextPartitionValues); + if (result != MetaCacheEntry.ReplaceResult.REPLACED) { + throw new IllegalStateException("replacement failed: " + result); + } + state.currentPartitionValues = state.nextPartitionValues; + state.nextPartitionValues = state.nextPartitionValues == state.partitionValues + ? state.replacementPartitionValues : state.partitionValues; + } + + public HivePartitionValues countStrongCacheHit(PreparedState state) { + return state.countCacheEntry.getIfPresent(state.key); + } + + public HivePartitionValues weightedSoftCacheHit(PreparedState state) { + return state.cacheEntry.getIfPresent(state.key); + } + + public static void main(String[] args) throws Exception { + HivePartitionValuesSizeBenchmark benchmark = new HivePartitionValuesSizeBenchmark(); + for (int partitionCount : new int[] {1000, 10000, 100000}) { + for (String distribution : new String[] {"uniform", "tail_skew"}) { + String suffix = "[partitions=" + partitionCount + ",distribution=" + distribution + "]"; + BenchmarkHarness.measure("hive.countPublicationBaseline" + suffix, TimeUnit.MILLISECONDS, () -> { + UnsealedState state = unsealedState(partitionCount, distribution); + return benchmark.countPublicationBaseline(state); + }); + BenchmarkHarness.measure("hive.sealPublicationWithoutEstimate" + suffix, + TimeUnit.MILLISECONDS, () -> { + UnsealedState state = unsealedState(partitionCount, distribution); + return benchmark.sealPublicationWithoutEstimate(state); + }); + BenchmarkHarness.measure("hive.weightedPublication" + suffix, TimeUnit.MILLISECONDS, () -> { + UnsealedState state = unsealedState(partitionCount, distribution); + return benchmark.weightedPublication(state); + }); + + PreparedState prepared = new PreparedState(); + prepared.partitionCount = partitionCount; + prepared.distribution = distribution; + prepared.setup(); + try { + BenchmarkHarness.measure("hive.eventCopySealAndEstimate" + suffix, + TimeUnit.MILLISECONDS, () -> benchmark.eventCopySealAndEstimate(prepared)); + BenchmarkHarness.measure("hive.preparedSizeProvider" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.preparedSizeProvider(prepared)); + BenchmarkHarness.measure("hive.estimateFormula" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.estimateFormula(prepared)); + BenchmarkHarness.measure("hive.countStrongCacheHit" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.countStrongCacheHit(prepared)); + BenchmarkHarness.measure("hive.weightedSoftCacheHit" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.weightedSoftCacheHit(prepared)); + BenchmarkHarness.measure("hive.replacementAdmission" + suffix, + TimeUnit.NANOSECONDS, () -> { + benchmark.replacementAdmission(prepared); + return null; + }); + } finally { + prepared.tearDown(); + } + } + } + } + + private static UnsealedState unsealedState(int partitionCount, String distribution) throws Exception { + UnsealedState state = new UnsealedState(); + state.partitionCount = partitionCount; + state.distribution = distribution; + state.setupInvocation(); + return state; + } + + /** Fresh graph per invocation so all publication work stays inside the measured method. */ + public static class UnsealedState { + public int partitionCount; + + public String distribution; + + private PartitionValueCacheKey key; + private HivePartitionValues partitionValues; + + public void setupInvocation() throws Exception { + List types = benchmarkTypes(); + key = benchmarkKey(types); + partitionValues = createPartitionValues(partitionCount, distribution, types); + } + } + + public static class PreparedState { + public int partitionCount; + + public String distribution; + + private PartitionValueCacheKey key; + private HivePartitionValues partitionValues; + private HivePartitionValues replacementPartitionValues; + private HivePartitionValues currentPartitionValues; + private HivePartitionValues nextPartitionValues; + private MetaCacheEntry cacheEntry; + private MetaCacheEntry countCacheEntry; + private ExecutorService cacheExecutor; + + public void setup() throws Exception { + List types = benchmarkTypes(); + key = benchmarkKey(types); + partitionValues = createPartitionValues(partitionCount, distribution, types); + partitionValues.prepareForCachePublication(key); + requireComplete(partitionValues); + + // A distinct value root makes Caffeine perform a real replacement while sharing + // immutable payload objects to keep the fixture's resident heap bounded. + replacementPartitionValues = new HivePartitionValues( + partitionValues.getIdToPartitionItem(), + partitionValues.getPartitionNameToIdMap(), + partitionValues.getPartitionValuesMap()); + replacementPartitionValues.prepareForCachePublication(key); + requireComplete(replacementPartitionValues); + + cacheExecutor = MoreExecutors.newDirectExecutorService(); + ExternalMetaCacheBudgetManager budgetManager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(MAX_WEIGHT_BYTES)); + EntryBudget entryBudget = budgetManager.createEntryBudget( + 1L, "hive", "partition_values_benchmark", OptionalLong.empty(), OptionalLong.empty()); + cacheEntry = new MetaCacheEntry<>( + "partition_values_benchmark", + ignored -> partitionValues, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 1L, MAX_WEIGHT_BYTES), + cacheExecutor, + false, + false, + (entryKey, value) -> value.prepareForCachePublication(entryKey), + entryBudget); + cacheEntry.put(key, partitionValues); + countCacheEntry = new MetaCacheEntry<>( + "partition_values_count_benchmark", + ignored -> partitionValues, + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 1L), + cacheExecutor, + false, + false); + countCacheEntry.put(key, partitionValues); + currentPartitionValues = partitionValues; + nextPartitionValues = replacementPartitionValues; + } + + public void tearDown() { + if (cacheEntry != null) { + cacheEntry.close(); + } + if (countCacheEntry != null) { + countCacheEntry.close(); + } + if (cacheExecutor != null) { + cacheExecutor.shutdownNow(); + } + } + } + + private static List benchmarkTypes() { + return Collections.singletonList(Type.STRING); + } + + private static PartitionValueCacheKey benchmarkKey(List types) { + return new PartitionValueCacheKey( + NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"), types); + } + + private static long requireComplete(HivePartitionValues value) { + MetaCacheSizeEstimate estimate = value.getSizeEstimate(); + if (!estimate.isComplete()) { + throw new IllegalStateException("benchmark graph is not fully measurable: " + + estimate.getIncompleteReason()); + } + return estimate.getBytes(); + } + + private static HivePartitionValues createPartitionValues( + int count, String distribution, List types) throws Exception { + HashBiMap nameToId = HashBiMap.create(count); + Map idToItem = Maps.newHashMapWithExpectedSize(count); + Map> idToValues = Maps.newHashMapWithExpectedSize(count); + String tailPayload = "tail_skew".equals(distribution) ? repeat('x', TAIL_PAYLOAD_BYTES) : null; + long partitionNameCharacterCount = 0L; + + for (int i = 0; i < count; i++) { + long id = i; + String value = i == count - 1 && tailPayload != null ? tailPayload : "value-" + i; + String name = "p=" + value; + partitionNameCharacterCount += name.length(); + List rawValues = Collections.singletonList(new PartitionValue(value)); + PartitionKey partitionKey = PartitionKey.createListPartitionKeyWithTypes(rawValues, types, true); + List keys = new ArrayList<>(1); + keys.add(partitionKey); + + nameToId.put(name, id); + idToItem.put(id, new ListPartitionItem(keys)); + idToValues.put(id, new ArrayList<>(Collections.singletonList(value))); + } + return new HivePartitionValues( + idToItem, nameToId, idToValues, partitionNameCharacterCount, types.size()); + } + + private static String repeat(char value, int count) { + char[] chars = new char[count]; + Arrays.fill(chars, value); + return new String(chars); + } +} diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeBenchmark.java new file mode 100644 index 00000000000000..2cca17a879b625 --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeBenchmark.java @@ -0,0 +1,411 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.benchmark.BenchmarkHarness; +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import com.google.common.collect.ImmutableList; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.ManifestContent; +import org.apache.iceberg.Metrics; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotParser; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.StaticTableOperations; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; +import org.apache.iceberg.inmemory.InMemoryFileIO; +import org.apache.iceberg.types.Types; + +import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Optional; +import java.util.concurrent.TimeUnit; + +/** Measures Iceberg table, long-history table, snapshot and manifest publication. */ +public class IcebergCacheSizeBenchmark { + public int tableValueConstruction(TablePublicationState state) { + return new IcebergTableCacheValue(state.table).getIcebergTable().schema().schemaId(); + } + + public long tablePublication(TablePublicationState state) { + IcebergTableCacheValue value = new IcebergTableCacheValue(state.table); + value.prepareForCachePublication(state.mapping); + return requireComplete(value.getSizeEstimate()); + } + + public long tablePayloadCounter(TablePublicationState state) { + return IcebergCacheSizeEstimator.retainedTablePayloadBytes(state.table); + } + + public long longHistoryTablePublication(LongHistoryTablePublicationState state) { + IcebergTableCacheValue value = new IcebergTableCacheValue(state.table); + value.prepareForCachePublication(state.mapping); + return requireComplete(value.getSizeEstimate()); + } + + public long snapshotPublication(SnapshotPublicationState state) { + IcebergSnapshotCacheValue value = new IcebergSnapshotCacheValue( + state.partitionInfo, new IcebergSnapshot(7L, 0L), Optional.empty(), state.table); + value.prepareForCachePublication(state.snapshotKey); + return requireComplete(value.getSizeEstimate()); + } + + public int snapshotValueConstruction(SnapshotPublicationState state) { + return new IcebergSnapshotCacheValue( + state.partitionInfo, new IcebergSnapshot(7L, 0L), Optional.empty(), state.table) + .getPartitionInfo().getNameToIcebergPartition().size(); + } + + public long preparedSnapshotCacheHit(SnapshotPublicationState state) { + return state.preparedSnapshotValue.getIcebergTable().get() + .currentSnapshot().snapshotId(); + } + + public long manifestPublication(ManifestState state) { + return requireComplete(IcebergCacheSizeEstimator.estimateManifestEntry(state.key, state.value)); + } + + public int manifestValueConstruction(ManifestState state) { + return ManifestCacheValue.forDataFiles(state.files).getDataFiles().size(); + } + + public int denseManifestReaderBaseline(DenseManifestState state) { + List collected = new ArrayList<>(); + for (DataFile file : state.files) { + collected.add(file.copy()); + } + return ImmutableList.copyOf(collected).size(); + } + + public int denseManifestValueConstruction(DenseManifestState state) { + ManifestCacheValue.Builder builder = ManifestCacheValue.dataFilesBuilder(); + for (DataFile file : state.files) { + builder.addDataFile(file.copy()); + } + return builder.build().getDataFiles().size(); + } + + public long preparedWeightProvider(PreparedState state) { + return state.preparedTableValue.getSizeEstimate().getBytes(); + } + + public long tableFormula(PreparedState state) { + return requireComplete(IcebergCacheSizeEstimator.estimateTableEntry( + state.mapping, state.preparedTableValue)); + } + + public int preparedTableCacheHit(PreparedState state) { + return state.preparedTableValue.getIcebergTable().schema().schemaId(); + } + + public static void main(String[] args) throws Exception { + IcebergCacheSizeBenchmark benchmark = new IcebergCacheSizeBenchmark(); + for (int fieldCount : new int[] {10, 100}) { + String suffix = "[fields=" + fieldCount + "]"; + TablePublicationState tableState = new TablePublicationState(); + tableState.fieldCount = fieldCount; + tableState.setup(); + BenchmarkHarness.measure("iceberg.tableValueConstruction" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.tableValueConstruction(tableState)); + BenchmarkHarness.measure("iceberg.tablePayloadCounter" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePayloadCounter(tableState)); + BenchmarkHarness.measure("iceberg.tablePublication" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePublication(tableState)); + + PreparedState prepared = new PreparedState(); + prepared.fieldCount = fieldCount; + prepared.setup(); + BenchmarkHarness.measure("iceberg.preparedWeightProvider" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.preparedWeightProvider(prepared)); + BenchmarkHarness.measure("iceberg.tableFormula" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.tableFormula(prepared)); + BenchmarkHarness.measure("iceberg.preparedTableCacheHit" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.preparedTableCacheHit(prepared)); + } + for (int snapshotCount : new int[] {1000, 10000}) { + String suffix = "[snapshots=" + snapshotCount + "]"; + LongHistoryTablePublicationState state = new LongHistoryTablePublicationState(); + state.snapshotCount = snapshotCount; + state.setup(); + BenchmarkHarness.measure("iceberg.longHistoryTablePublication" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.longHistoryTablePublication(state)); + } + for (int fieldCount : new int[] {10, 100}) { + for (int partitionCount : new int[] {1000, 10000}) { + String suffix = "[fields=" + fieldCount + ",partitions=" + partitionCount + "]"; + SnapshotPublicationState state = new SnapshotPublicationState(); + state.fieldCount = fieldCount; + state.partitionCount = partitionCount; + state.setup(); + BenchmarkHarness.measure("iceberg.snapshotValueConstruction" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.snapshotValueConstruction(state)); + BenchmarkHarness.measure("iceberg.snapshotPublication" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.snapshotPublication(state)); + BenchmarkHarness.measure("iceberg.preparedSnapshotCacheHit" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.preparedSnapshotCacheHit(state)); + } + } + for (int fileCount : new int[] {100, 10000}) { + ManifestState state = new ManifestState(); + state.fileCount = fileCount; + state.setup(); + BenchmarkHarness.measure("iceberg.manifestPublication[files=" + fileCount + "]", + TimeUnit.MICROSECONDS, () -> benchmark.manifestPublication(state)); + BenchmarkHarness.measure("iceberg.manifestValueConstruction[files=" + fileCount + "]", + TimeUnit.MICROSECONDS, () -> benchmark.manifestValueConstruction(state)); + } + for (int metricColumns : new int[] {100, 1000}) { + for (int fileCount : new int[] {100, 10000}) { + DenseManifestState state = new DenseManifestState(); + state.metricColumns = metricColumns; + state.fileCount = fileCount; + state.setup(); + String suffix = "[files=" + fileCount + ",metricColumns=" + metricColumns + "]"; + BenchmarkHarness.measure("iceberg.denseManifestReaderBaseline" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.denseManifestReaderBaseline(state)); + BenchmarkHarness.measure("iceberg.denseManifestValueConstruction" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.denseManifestValueConstruction(state)); + } + } + } + + public static class TablePublicationState { + public int fieldCount; + + private NameMapping mapping; + private Table table; + + public void setup() { + mapping = NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"); + table = newTable(fieldCount); + } + } + + public static class SnapshotPublicationState { + public int fieldCount; + + public int partitionCount; + + private Table table; + private IcebergSnapshotEntryKey snapshotKey; + private IcebergPartitionInfo partitionInfo; + private IcebergSnapshotCacheValue preparedSnapshotValue; + + public void setup() { + NameMapping mapping = NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"); + table = newTable(fieldCount); + partitionInfo = newPartitionInfo(partitionCount); + snapshotKey = IcebergSnapshotEntryKey.tryCreate(mapping, table) + .orElseThrow(() -> new IllegalStateException("benchmark table has no generation key")); + preparedSnapshotValue = new IcebergSnapshotCacheValue( + partitionInfo, new IcebergSnapshot(7L, 0L), Optional.empty(), table); + preparedSnapshotValue.prepareForCachePublication(snapshotKey); + requireComplete(preparedSnapshotValue.getSizeEstimate()); + } + } + + public static class LongHistoryTablePublicationState { + public int snapshotCount; + + private NameMapping mapping; + private Table table; + + public void setup() { + mapping = NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"); + table = newLongHistoryTable(snapshotCount); + } + } + + public static class PreparedState { + public int fieldCount; + + private IcebergTableCacheValue preparedTableValue; + private NameMapping mapping; + + public void setup() { + mapping = NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"); + Table table = newTable(fieldCount); + preparedTableValue = new IcebergTableCacheValue(table); + preparedTableValue.prepareForCachePublication(mapping); + requireComplete(preparedTableValue.getSizeEstimate()); + } + } + + public static class ManifestState { + public int fileCount; + + private IcebergManifestEntryKey key; + private ManifestCacheValue value; + private List files; + + public void setup() { + key = new IcebergManifestEntryKey("/benchmark/manifest.avro", ManifestContent.DATA); + files = new ArrayList<>(fileCount); + for (int index = 0; index < fileCount; index++) { + files.add(DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/benchmark/data/file-" + index + ".parquet") + .withFileSizeInBytes(1024L + index) + .withRecordCount(10L + index) + .build()); + } + value = ManifestCacheValue.forDataFiles(files); + } + } + + public static class DenseManifestState { + public int fileCount; + + public int metricColumns; + + private List files; + + public void setup() { + int poolSize = Math.min(fileCount, 256); + List filePool = new ArrayList<>(poolSize); + for (int fileIndex = 0; fileIndex < poolSize; fileIndex++) { + HashMap lowerBounds = new HashMap<>(metricColumns); + HashMap upperBounds = new HashMap<>(metricColumns); + for (int columnIndex = 0; columnIndex < metricColumns; columnIndex++) { + lowerBounds.put(columnIndex, ByteBuffer.allocate(16)); + upperBounds.put(columnIndex, ByteBuffer.allocate(32)); + } + Metrics metrics = new Metrics( + 10L, + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + Collections.emptyMap(), + lowerBounds, + upperBounds); + filePool.add(DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/benchmark/data/dense-" + fileIndex + ".parquet") + .withFileSizeInBytes(1024L) + .withMetrics(metrics) + .build()); + } + files = new ArrayList<>(fileCount); + for (int index = 0; index < fileCount; index++) { + files.add(filePool.get(index % poolSize)); + } + } + } + + private static Table newTable(int fieldCount) { + List fields = new ArrayList<>(fieldCount); + for (int index = 0; index < fieldCount; index++) { + fields.add(Types.NestedField.optional(index + 1, "field_" + index, Types.StringType.get())); + } + Schema schema = new Schema(fields); + TableMetadata metadata = TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), "file:/benchmark/table", Collections.emptyMap()); + InMemoryFileIO fileIO = new InMemoryFileIO(); + StringBuilder snapshotJson = new StringBuilder("{\"snapshot-id\":7,\"timestamp-ms\":1,") + .append("\"summary\":{\"operation\":\"append\"},\"manifests\":["); + for (int index = 0; index < 10; index++) { + if (index > 0) { + snapshotJson.append(','); + } + String manifestPath = "/benchmark/manifest-" + index + ".avro"; + snapshotJson.append('"').append(manifestPath).append('"'); + fileIO.addFile(manifestPath, new byte[0]); + } + Snapshot snapshot = SnapshotParser.fromJson(snapshotJson.append("],\"schema-id\":0}").toString()); + metadata = TableMetadata.buildFrom(metadata).setBranchSnapshot(snapshot, SnapshotRef.MAIN_BRANCH) + .discardChanges() + .withMetadataLocation("file:/benchmark/table/metadata/v1.json").build(); + return new BaseTable(new StaticTableOperations(metadata, fileIO), "benchmark.table"); + } + + private static Table newLongHistoryTable(int snapshotCount) { + long currentSnapshotId = 1000L + snapshotCount - 1L; + StringBuilder json = new StringBuilder() + .append("{\"format-version\":2,\"table-uuid\":\"benchmark-table\",") + .append("\"location\":\"file:/benchmark/table\",\"last-sequence-number\":") + .append(snapshotCount).append(",\"last-updated-ms\":").append(snapshotCount) + .append(",\"last-column-id\":1,\"current-schema-id\":0,") + .append("\"schemas\":[{\"type\":\"struct\",\"schema-id\":0,\"fields\":[") + .append("{\"id\":1,\"name\":\"field\",\"required\":false,\"type\":\"string\"}]}],") + .append("\"default-spec-id\":0,\"partition-specs\":[{\"spec-id\":0,\"fields\":[]}],") + .append("\"last-partition-id\":999,\"default-sort-order-id\":0,") + .append("\"sort-orders\":[{\"order-id\":0,\"fields\":[]}],\"properties\":{},") + .append("\"current-snapshot-id\":").append(currentSnapshotId) + .append(",\"refs\":{\"main\":{\"snapshot-id\":").append(currentSnapshotId) + .append(",\"type\":\"branch\"}},\"snapshots\":["); + for (int index = 0; index < snapshotCount; index++) { + if (index > 0) { + json.append(','); + } + json.append("{\"sequence-number\":").append(index + 1L) + .append(",\"snapshot-id\":").append(1000L + index) + .append(",\"timestamp-ms\":").append(index + 1L) + .append(",\"summary\":{\"operation\":\"append\"},") + .append("\"manifest-list\":\"/benchmark/history/list-").append(index) + .append(".avro\",\"schema-id\":0}"); + } + json.append("],\"statistics\":[],\"partition-statistics\":[],\"snapshot-log\":["); + for (int index = 0; index < snapshotCount; index++) { + if (index > 0) { + json.append(','); + } + json.append("{\"timestamp-ms\":").append(index + 1L) + .append(",\"snapshot-id\":").append(1000L + index).append('}'); + } + json.append("],\"metadata-log\":[]}"); + TableMetadata metadata = TableMetadataParser.fromJson( + "file:/benchmark/table/metadata/v1.json", json.toString()); + return new BaseTable( + new StaticTableOperations(metadata, new InMemoryFileIO()), "benchmark.table"); + } + + private static IcebergPartitionInfo newPartitionInfo(int partitionCount) { + HashMap partitions = new HashMap<>(partitionCount); + long retainedPayloadBytes = 0L; + for (int index = 0; index < partitionCount; index++) { + String name = "partition_key=value_" + index; + IcebergPartition partition = new IcebergPartition(name, 0, 10L + index, 1024L + index, + 1L, 1_700_000_000_000L + index, 7L, + Collections.singletonList("value_" + index), Collections.singletonList("identity")); + partitions.put(name, partition); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, partition.getRetainedPayloadBytes()); + } + return new IcebergPartitionInfo( + Collections.emptyMap(), partitions, Collections.emptyMap(), retainedPayloadBytes); + } + + private static long requireComplete(MetaCacheSizeEstimate estimate) { + if (!estimate.isComplete()) { + throw new IllegalStateException("incomplete benchmark estimate: " + estimate.getIncompleteReason()); + } + return estimate.getBytes(); + } + +} diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSoftValueBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSoftValueBenchmark.java new file mode 100644 index 00000000000000..b488f428e8b9a9 --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSoftValueBenchmark.java @@ -0,0 +1,175 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.EntryBudget; + +import com.github.benmanes.caffeine.cache.LoadingCache; +import com.google.common.util.concurrent.MoreExecutors; + +import java.lang.ref.Reference; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.OptionalLong; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.LockSupport; + +/** Measures reservation cleanup after Caffeine reports soft values as COLLECTED. */ +public final class MetaCacheSoftValueBenchmark { + private static final int SAMPLE_COUNT = 3; + private static final int VALUE_COUNT = 10_000; + private static final long MAX_WEIGHT_BYTES = 16L * 1024L * 1024L; + + private MetaCacheSoftValueBenchmark() { + } + + public static void main(String[] args) throws Exception { + long totalNanos = 0L; + for (int sample = 0; sample < SAMPLE_COUNT; sample++) { + CollectedState state = CollectedState.create(VALUE_COUNT); + try { + state.enqueueAll(); + long start = System.nanoTime(); + state.cleanUp(); + totalNanos += System.nanoTime() - start; + } finally { + state.close(); + } + } + double averageNanos = (double) totalNanos / SAMPLE_COUNT; + System.out.printf(Locale.ROOT, + "%-72s %12.3f us/batch (%.3f ns/value, %d samples)%n", + "metacache.collectedCleanup[values=" + VALUE_COUNT + "]", + averageNanos / TimeUnit.MICROSECONDS.toNanos(1L), + averageNanos / VALUE_COUNT, + SAMPLE_COUNT); + } + + private static final class CollectedState implements AutoCloseable { + private final ExecutorService executor; + private final ExternalMetaCacheBudgetManager budgetManager; + private final MetaCacheEntry entry; + private final LoadingCache loadingCache; + private final List> references; + + private CollectedState(ExecutorService executor, + ExternalMetaCacheBudgetManager budgetManager, + MetaCacheEntry entry, + LoadingCache loadingCache, + List> references) { + this.executor = executor; + this.budgetManager = budgetManager; + this.entry = entry; + this.loadingCache = loadingCache; + this.references = references; + } + + private static CollectedState create(int valueCount) throws Exception { + ExecutorService executor = MoreExecutors.newDirectExecutorService(); + ExternalMetaCacheBudgetManager budgetManager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(MAX_WEIGHT_BYTES)); + EntryBudget budget = budgetManager.createEntryBudget( + 1L, "benchmark", "soft_cleanup", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "soft_cleanup", + key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, valueCount, MAX_WEIGHT_BYTES), + executor, + false, + false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), + budget); + for (int index = 0; index < valueCount; index++) { + entry.put("key-" + index, new byte[1]); + } + + LoadingCache loadingCache = (LoadingCache) readField(entry, "loadingData"); + Object boundedLocalCache = readField(loadingCache, "cache"); + Map nodes = (Map) readField(boundedLocalCache, "data"); + List> references = new ArrayList<>(nodes.size()); + for (Object node : nodes.values()) { + Method valueReferenceMethod = findMethod(node.getClass(), "getValueReference"); + references.add((Reference) valueReferenceMethod.invoke(node)); + } + if (references.size() != valueCount) { + entry.close(); + executor.shutdownNow(); + throw new IllegalStateException( + "benchmark admission retained " + references.size() + " of " + valueCount + " values"); + } + return new CollectedState(executor, budgetManager, entry, loadingCache, references); + } + + private void enqueueAll() { + for (Reference reference : references) { + reference.clear(); + reference.enqueue(); + } + } + + private void cleanUp() { + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(30L); + while (budgetManager.getGlobalUsedWeight() != 0L + && System.nanoTime() < deadline) { + loadingCache.cleanUp(); + LockSupport.parkNanos(TimeUnit.MICROSECONDS.toNanos(100L)); + } + if (budgetManager.getGlobalUsedWeight() != 0L) { + throw new IllegalStateException( + "COLLECTED cleanup retained " + budgetManager.getGlobalUsedWeight() + " bytes"); + } + } + + @Override + public void close() { + entry.close(); + executor.shutdownNow(); + } + } + + private static Object readField(Object target, String name) throws Exception { + for (Class type = target.getClass(); type != null; type = type.getSuperclass()) { + try { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } catch (NoSuchFieldException ignored) { + // Continue through Caffeine's generated cache hierarchy. + } + } + throw new NoSuchFieldException(name); + } + + private static Method findMethod(Class type, String name) throws Exception { + for (Class current = type; current != null; current = current.getSuperclass()) { + try { + Method method = current.getDeclaredMethod(name); + method.setAccessible(true); + return method; + } catch (NoSuchMethodException ignored) { + // Continue through Caffeine's generated node hierarchy. + } + } + throw new NoSuchMethodException(name); + } +} diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeBenchmark.java new file mode 100644 index 00000000000000..f73eca412f3c44 --- /dev/null +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeBenchmark.java @@ -0,0 +1,272 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.benchmark.BenchmarkHarness; +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import org.apache.paimon.fs.Path; +import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.partition.Partition; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.AppendOnlyFileStoreTable; +import org.apache.paimon.table.CatalogEnvironment; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VarCharType; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.concurrent.TimeUnit; + +/** Measures Paimon 1.4.2 nested-schema/non-empty snapshot publication and prepared weight lookup. */ +public class PaimonCacheSizeBenchmark { + public long snapshotPublication(PublicationState state) { + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue(state.partitionInfo, state.snapshot); + value.prepareForCachePublication(state.key); + return requireComplete(value.getSizeEstimate()); + } + + public long tablePayloadCounter(PublicationState state) { + return PaimonCacheSizeEstimator.retainedTablePayloadBytes(state.snapshot.getTable()); + } + + public long preparedWeightProvider(PreparedState state) { + return state.value.getSizeEstimate().getBytes(); + } + + public long snapshotFormula(PreparedState state) { + return requireComplete(PaimonCacheSizeEstimator.estimateSnapshotEntry(state.key, state.value)); + } + + public long partitionMapBaseline(PartitionPayloadState state) { + return buildPartitionInfo(state, false); + } + + public long partitionMapWithRetainedCounter(PartitionPayloadState state) { + return buildPartitionInfo(state, true); + } + + public static void main(String[] args) throws Exception { + PaimonCacheSizeBenchmark benchmark = new PaimonCacheSizeBenchmark(); + for (int fieldCount : new int[] {10, 100}) { + for (int partitionCount : new int[] {1000, 10000}) { + String suffix = "[fields=" + fieldCount + ",partitions=" + partitionCount + "]"; + PublicationState publication = new PublicationState(); + publication.fieldCount = fieldCount; + publication.partitionCount = partitionCount; + publication.setup(); + BenchmarkHarness.measure("paimon.tablePayloadCounter" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePayloadCounter(publication)); + BenchmarkHarness.measure("paimon.snapshotPublication" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.snapshotPublication(publication)); + + PreparedState prepared = new PreparedState(); + prepared.fieldCount = fieldCount; + prepared.partitionCount = partitionCount; + prepared.setup(); + BenchmarkHarness.measure("paimon.preparedWeightProvider" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.preparedWeightProvider(prepared)); + BenchmarkHarness.measure("paimon.snapshotFormula" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.snapshotFormula(prepared)); + } + } + for (int partitionCount : new int[] {1000, 10000}) { + for (boolean tailSkew : new boolean[] {false, true}) { + PartitionPayloadState state = new PartitionPayloadState(); + state.partitionCount = partitionCount; + state.tailSkew = tailSkew; + state.setup(); + String suffix = "[partitions=" + partitionCount + + ",distribution=" + (tailSkew ? "tail-skew" : "uniform") + "]"; + BenchmarkHarness.measure("paimon.partitionMapBaseline" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.partitionMapBaseline(state)); + BenchmarkHarness.measure("paimon.partitionMapWithRetainedCounter" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.partitionMapWithRetainedCounter(state)); + } + } + } + + public static class PublicationState { + public int fieldCount; + + public int partitionCount; + + private PaimonSnapshotEntryKey key; + private PaimonPartitionInfo partitionInfo; + private PaimonSnapshot snapshot; + + public void setup() throws Exception { + Fixture fixture = newFixture(fieldCount, partitionCount); + key = fixture.key; + partitionInfo = fixture.value.getPartitionInfo(); + snapshot = fixture.value.getSnapshot(); + } + } + + public static class PreparedState { + public int fieldCount; + + public int partitionCount; + + private PaimonSnapshotEntryKey key; + private PaimonSnapshotCacheValue value; + + public void setup() throws Exception { + Fixture fixture = newFixture(fieldCount, partitionCount); + key = fixture.key; + value = fixture.value; + value.prepareForCachePublication(fixture.key); + requireComplete(value.getSizeEstimate()); + } + } + + public static class PartitionPayloadState { + public int partitionCount; + + public boolean tailSkew; + + private List partitions; + + public void setup() { + partitions = new ArrayList<>(partitionCount); + String longTail = String.join("", Collections.nCopies(64 * 1024, "x")); + for (int index = 0; index < partitionCount; index++) { + String value = tailSkew && index % 997 == 0 ? longTail : String.valueOf(index); + LinkedHashMap typedSpec = new LinkedHashMap<>(); + for (int field = 0; field < 4; field++) { + typedSpec.put("partition_key_" + field, value + '_' + field); + } + String displayName = "partition_key_0=" + value; + partitions.add(new PartitionPayload( + displayName, new ArrayList<>(typedSpec.values()), index)); + } + } + } + + private static Fixture newFixture(int fieldCount, int partitionCount) throws Exception { + List fields = new ArrayList<>(fieldCount + 2); + fields.add(new DataField(0, "partition_key", new IntType())); + for (int index = 0; index < fieldCount; index++) { + fields.add(new DataField(index + 1, "field_" + index, new VarCharType())); + } + List nestedFields = new ArrayList<>(); + for (int index = 0; index < 8; index++) { + nestedFields.add(DataTypes.FIELD(fieldCount + index + 1, + "nested_field_" + index, DataTypes.STRING())); + } + fields.add(new DataField(fieldCount + 9, "nested_payload", new RowType(nestedFields))); + TableSchema schema = new TableSchema( + 0L, fields, fieldCount + 9, Collections.singletonList("partition_key"), + Collections.emptyList(), Collections.emptyMap(), null); + FileStoreTable table = new AppendOnlyFileStoreTable( + LocalFileIO.create(), new Path("file:/tmp/doris-paimon-cache-size-benchmark"), + schema, CatalogEnvironment.empty()); + NameMapping mapping = NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey(mapping, 7L, schema.id(), 1L); + HashMap partitions = new HashMap<>(partitionCount); + long retainedPartitionPayloadBytes = 0L; + for (int index = 0; index < partitionCount; index++) { + String name = "partition_key=" + index; + String value = String.valueOf(index); + partitions.put(name, new Partition(Collections.singletonMap("partition_key", value), + 10L + index, 1024L + index, 1L, 1_700_000_000_000L + index, 1, true, + 1_700_000_000_000L, "benchmark", 1_700_000_000_000L + index, "benchmark", + Collections.singletonMap("source", "benchmark"))); + retainedPartitionPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPartitionPayloadBytes, MetaCacheWeightUtils.estimatedStringBytes(name)); + retainedPartitionPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPartitionPayloadBytes, MetaCacheWeightUtils.estimatedStringBytes("partition_key")); + retainedPartitionPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPartitionPayloadBytes, MetaCacheWeightUtils.estimatedStringBytes(value)); + } + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + new PaimonPartitionInfo(Collections.emptyMap(), partitions, retainedPartitionPayloadBytes), + new PaimonSnapshot(7L, schema.id(), table)); + return new Fixture(key, value); + } + + private static long requireComplete(MetaCacheSizeEstimate estimate) { + if (!estimate.isComplete()) { + throw new IllegalStateException("incomplete benchmark estimate: " + estimate.getIncompleteReason()); + } + return estimate.getBytes(); + } + + private long buildPartitionInfo(PartitionPayloadState state, boolean countPayload) { + HashMap partitions = new HashMap<>(state.partitionCount); + long retainedPayloadBytes = 0L; + for (PartitionPayload payload : state.partitions) { + LinkedHashMap typedSpec = new LinkedHashMap<>(); + for (int field = 0; field < payload.values.size(); field++) { + String fieldName = "partition_key_" + field; + String fieldValue = payload.values.get(field); + typedSpec.put(fieldName, fieldValue); + if (countPayload) { + retainedPayloadBytes = PaimonPartitionInfo.addRetainedStringPayload( + retainedPayloadBytes, fieldName); + retainedPayloadBytes = PaimonPartitionInfo.addRetainedStringPayload( + retainedPayloadBytes, fieldValue); + } + } + if (countPayload) { + retainedPayloadBytes = PaimonPartitionInfo.addRetainedStringPayload( + retainedPayloadBytes, payload.displayName); + } + int index = payload.index; + partitions.put(payload.displayName, new Partition( + typedSpec, 10L + index, 1024L + index, 1L, + 1_700_000_000_000L + index, 1, false)); + } + PaimonPartitionInfo partitionInfo = new PaimonPartitionInfo( + Collections.emptyMap(), partitions, retainedPayloadBytes); + return MetaCacheWeightUtils.saturatedAdd( + partitionInfo.getNameToPartition().size(), partitionInfo.getRetainedPayloadBytes()); + } + + private static class PartitionPayload { + private final String displayName; + private final List values; + private final int index; + + private PartitionPayload( + String displayName, List values, int index) { + this.displayName = displayName; + this.values = values; + this.index = index; + } + } + + private static class Fixture { + private final PaimonSnapshotEntryKey key; + private final PaimonSnapshotCacheValue value; + + private Fixture(PaimonSnapshotEntryKey key, PaimonSnapshotCacheValue value) { + this.key = key; + this.value = value; + } + } +} diff --git a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java index 4e535950aae0e3..c3e95e2d4ff24a 100644 --- a/fe/fe-common/src/main/java/org/apache/doris/common/Config.java +++ b/fe/fe-common/src/main/java/org/apache/doris/common/Config.java @@ -2462,6 +2462,11 @@ public class Config extends ConfigBase { }) public static long external_cache_refresh_time_minutes = 10; // 10 mins + @ConfField(mutable = false, masterOnly = false, + description = {"FE-wide maximum weight for managed external metadata caches. Supports byte units " + + "or a percentage of the JVM max heap; 0 disables the global quota."}) + public static String external_meta_cache_max_weight = "0"; + // Enable manual miss load for external meta cache to avoid blocking replayer on slow loaders. @ConfField(mutable = true, masterOnly = false, description = {"Whether external meta cache uses manual miss load instead of Caffeine sync load."}) diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java b/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java index 674bf0aa39cd5b..7ad55174e303e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/CacheFactory.java @@ -24,6 +24,7 @@ import com.github.benmanes.caffeine.cache.LoadingCache; import com.github.benmanes.caffeine.cache.RemovalListener; import com.github.benmanes.caffeine.cache.Ticker; +import com.github.benmanes.caffeine.cache.Weigher; import org.jetbrains.annotations.NotNull; import java.time.Duration; @@ -49,7 +50,10 @@ public class CacheFactory { private OptionalLong expireAfterAccessSec; private OptionalLong refreshAfterWriteSec; private long maxSize; + private OptionalLong maxWeight; + private Weigher weigher; private boolean enableStats; + private boolean softValues; // Ticker is used to provide a time source for the cache. // Only used for test, to provide a fake time source. // If not provided, the system time is used. @@ -61,11 +65,34 @@ public CacheFactory( long maxSize, boolean enableStats, Ticker ticker) { + this(expireAfterAccessSec, refreshAfterWriteSec, maxSize, OptionalLong.empty(), null, enableStats, ticker); + } + + @SuppressWarnings("unchecked") + public CacheFactory( + OptionalLong expireAfterAccessSec, + OptionalLong refreshAfterWriteSec, + long maxSize, + OptionalLong maxWeight, + Weigher weigher, + boolean enableStats, + Ticker ticker) { this.expireAfterAccessSec = expireAfterAccessSec; this.refreshAfterWriteSec = refreshAfterWriteSec; this.maxSize = maxSize; + this.maxWeight = maxWeight; + this.weigher = (Weigher) weigher; this.enableStats = enableStats; this.ticker = ticker; + if (maxWeight.isPresent() && this.weigher == null) { + throw new IllegalArgumentException("maximumWeight requires a weigher"); + } + } + + /** Configure values as soft references so unused cache entries may be reclaimed under heap pressure. */ + public CacheFactory withSoftValues() { + softValues = true; + return this; } // Build a loading cache, without executor, it will use fork-join pool for refresh @@ -116,7 +143,11 @@ public AsyncLoadingCache buildAsyncCache(AsyncCacheLoader cac @NotNull private Caffeine buildWithParams() { Caffeine builder = Caffeine.newBuilder(); - builder.maximumSize(maxSize); + if (maxWeight.isPresent()) { + builder.maximumWeight(maxWeight.getAsLong()).weigher(weigher); + } else { + builder.maximumSize(maxSize); + } if (expireAfterAccessSec.isPresent()) { builder.expireAfterAccess(Duration.ofSeconds(expireAfterAccessSec.getAsLong())); @@ -129,6 +160,10 @@ private Caffeine buildWithParams() { builder.recordStats(); } + if (softValues) { + builder.softValues(); + } + if (ticker != null) { builder.ticker(ticker); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index 481c1fcb3aca5f..f32c77a7015818 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -49,6 +49,7 @@ import org.apache.doris.datasource.lakesoul.LakeSoulExternalDatabase; import org.apache.doris.datasource.lance.LanceExternalDatabase; import org.apache.doris.datasource.maxcompute.MaxComputeExternalDatabase; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCache; import org.apache.doris.datasource.operations.ExternalMetadataOps; import org.apache.doris.datasource.paimon.PaimonExternalDatabase; @@ -446,6 +447,19 @@ protected void checkProperties(CatalogProperty property) throws DdlException { } } + try { + Env currentEnv = Env.getCurrentEnv(); + ExternalMetaCacheMgr extMetaCacheMgr = currentEnv == null ? null : currentEnv.getExtMetaCacheMgr(); + if (extMetaCacheMgr == null) { + // This fallback is only for isolated construction tests before Env is initialized. + ExternalMetaCacheBudgetManager.fromConfig().validateCatalogMaxWeight(properties); + } else { + extMetaCacheMgr.validateCatalogCacheProperties(this, properties); + } + } catch (IllegalArgumentException e) { + throw new DdlException(e.getMessage()); + } + // check schema.cache.ttl-second parameter String schemaCacheTtlSecond = property.getOrDefault(SCHEMA_CACHE_TTL_SECOND, null); if (java.util.Objects.nonNull(schemaCacheTtlSecond) && NumberUtils.toInt(schemaCacheTtlSecond, CACHE_NO_TTL) @@ -1367,9 +1381,21 @@ public int hashCode() { public void notifyPropertiesUpdated(Map updatedProps) { CatalogIf.super.notifyPropertiesUpdated(updatedProps); String schemaCacheTtl = updatedProps.getOrDefault(SCHEMA_CACHE_TTL_SECOND, null); - if (java.util.Objects.nonNull(schemaCacheTtl)) { - ExternalMetaCacheMgr extMetaCacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); + ExternalMetaCacheMgr extMetaCacheMgr = Env.getCurrentEnv().getExtMetaCacheMgr(); + if (java.util.Objects.nonNull(schemaCacheTtl) + || updatedProps.containsKey(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY)) { extMetaCacheMgr.removeCatalog(id); + return; + } + for (String key : updatedProps.keySet()) { + if (key == null || !key.startsWith("meta.cache.")) { + continue; + } + String remainder = key.substring("meta.cache.".length()); + int separator = remainder.indexOf('.'); + if (separator > 0) { + extMetaCacheMgr.removeCatalogByEngine(id, remainder.substring(0, separator)); + } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index 007e850e54e24e..a80d0c8b71485f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -27,6 +27,7 @@ import org.apache.doris.datasource.maxcompute.MaxComputeExternalMetaCache; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.ExternalMetaCache; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.ExternalMetaCacheRegistry; import org.apache.doris.datasource.metacache.ExternalMetaCacheRouteResolver; import org.apache.doris.datasource.metacache.LegacyMetaCacheFactory; @@ -38,6 +39,7 @@ import com.github.benmanes.caffeine.cache.stats.CacheStats; import com.google.common.collect.Maps; +import com.google.common.util.concurrent.Striped; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -47,8 +49,11 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import java.util.Set; import java.util.concurrent.ExecutorService; +import java.util.concurrent.locks.Lock; import java.util.function.Consumer; +import java.util.stream.Collectors; import javax.annotation.Nullable; /** @@ -95,6 +100,10 @@ public class ExternalMetaCacheMgr { private final ExternalMetaCacheRegistry cacheRegistry; private final ExternalMetaCacheRouteResolver routeResolver; private final LegacyMetaCacheFactory legacyMetaCacheFactory; + private final ExternalMetaCacheBudgetManager budgetManager; + // Catalog property publication and cache-group replacement share this striped lifecycle fence. + // Initialized lookups retain the lock-free fast path above it. + private final Striped catalogLifecycleLocks = Striped.lock(64); // all catalogs could share the same fsCache. private FileSystemCache fsCache; @@ -102,6 +111,7 @@ public class ExternalMetaCacheMgr { private ExternalRowCountCache rowCountCache; public ExternalMetaCacheMgr(boolean isCheckpointCatalog) { + budgetManager = ExternalMetaCacheBudgetManager.fromConfig(); rowCountRefreshExecutor = newThreadPool(isCheckpointCatalog, Config.max_external_cache_loader_thread_pool_size, Config.max_external_cache_loader_thread_pool_size * 1000, @@ -191,28 +201,114 @@ public DorisExternalMetaCache doris(long catalogId) { } public void prepareCatalog(long catalogId) { - Map catalogProperties = findCatalogProperties(catalogId); - if (catalogProperties == null) { - logMissingCatalogSkip(catalogId, "prepareCatalog"); - return; + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + Map catalogProperties = findCatalogProperties(catalogId); + if (catalogProperties == null) { + logMissingCatalogSkip(catalogId, "prepareCatalog"); + return; + } + validateCatalogCachePropertiesForRuntime(catalogProperties); + routeCatalogEngines(catalogId, cache -> cache.initCatalog(catalogId, catalogProperties)); + } finally { + lifecycleLock.unlock(); } - routeCatalogEngines(catalogId, cache -> cache.initCatalog(catalogId, catalogProperties)); } public void prepareCatalogByEngine(long catalogId, String engine) { - Map catalogProperties = findCatalogProperties(catalogId); - if (catalogProperties == null) { - logMissingCatalogSkip(catalogId, "prepareCatalogByEngine"); + ExternalMetaCache targetCache = this.engine(engine); + if (targetCache.isCatalogInitialized(catalogId)) { return; } - prepareCatalogByEngine(catalogId, engine, catalogProperties); + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + if (targetCache.isCatalogInitialized(catalogId)) { + return; + } + Map catalogProperties = findCatalogProperties(catalogId); + if (catalogProperties == null) { + logMissingCatalogSkip(catalogId, "prepareCatalogByEngine"); + return; + } + prepareCatalogByEngineLocked(catalogId, targetCache, catalogProperties); + } finally { + lifecycleLock.unlock(); + } } public void prepareCatalogByEngine(long catalogId, String engine, Map catalogProperties) { + ExternalMetaCache targetCache = this.engine(engine); + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + prepareCatalogByEngineLocked(catalogId, targetCache, catalogProperties); + } finally { + lifecycleLock.unlock(); + } + } + + private void prepareCatalogByEngineLocked( + long catalogId, ExternalMetaCache targetCache, Map catalogProperties) { Map safeCatalogProperties = catalogProperties == null ? Maps.newHashMap() : Maps.newHashMap(catalogProperties); - routeSpecifiedEngine(engine, cache -> cache.initCatalog(catalogId, safeCatalogProperties)); + validateCatalogCachePropertiesForRuntime(safeCatalogProperties); + targetCache.initCatalog(catalogId, safeCatalogProperties); + } + + public void validateCatalogCacheProperties(Map catalogProperties) { + budgetManager.validateCatalogMaxWeight(catalogProperties); + validateCatalogCachePropertyNamespaces(catalogProperties); + cacheRegistry.allCaches().forEach(cache -> cache.validateCatalogProperties(catalogProperties)); + } + + private void validateCatalogCachePropertiesForRuntime(Map catalogProperties) { + budgetManager.parseCatalogMaxWeight(catalogProperties); + validateCatalogCachePropertyNamespaces(catalogProperties); + } + + private void validateCatalogCachePropertyNamespaces(Map catalogProperties) { + String globalKey = ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY; + String prefix = "meta.cache."; + for (String key : catalogProperties.keySet()) { + if (key == null || globalKey.equals(key) || !key.startsWith(prefix)) { + continue; + } + String remainder = key.substring(prefix.length()); + int separator = remainder.indexOf('.'); + if (separator <= 0) { + throw new IllegalArgumentException("Unknown external meta cache property: " + key); + } + String configuredEngine = remainder.substring(0, separator); + ExternalMetaCache resolved = cacheRegistry.resolve(configuredEngine); + if (!resolved.engine().equals(configuredEngine)) { + throw new IllegalArgumentException("External meta cache properties must use canonical engine '" + + resolved.engine() + "' instead of alias '" + configuredEngine + "': " + key); + } + } + } + + /** Strict DDL validation also rejects a valid engine namespace not routed by the catalog type. */ + public void validateCatalogCacheProperties(CatalogIf catalog, Map catalogProperties) { + validateCatalogCacheProperties(catalogProperties); + Set routedEngines = routeResolver.resolveCatalogCaches(catalog.getId(), catalog).stream() + .map(ExternalMetaCache::engine) + .collect(Collectors.toSet()); + for (String key : catalogProperties.keySet()) { + if (key == null || ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY.equals(key) + || !key.startsWith("meta.cache.")) { + continue; + } + String remainder = key.substring("meta.cache.".length()); + int separator = remainder.indexOf('.'); + String configuredEngine = separator < 0 ? remainder : remainder.substring(0, separator); + if (!routedEngines.contains(configuredEngine)) { + throw new IllegalArgumentException("External meta cache engine '" + configuredEngine + + "' is not supported by catalog type " + catalog.getClass().getSimpleName() + ": " + key); + } + } } public void invalidateCatalog(long catalogId) { @@ -228,15 +324,27 @@ public void invalidateCatalogByEngine(long catalogId, String engine) { } public void removeCatalog(long catalogId) { - routeCatalogEngines(catalogId, cache -> safeInvalidate( - cache, catalogId, "removeCatalog", - () -> cache.invalidateCatalog(catalogId))); + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "removeCatalog", + () -> cache.invalidateCatalog(catalogId))); + } finally { + lifecycleLock.unlock(); + } } public void removeCatalogByEngine(long catalogId, String engine) { - routeSpecifiedEngine(engine, cache -> safeInvalidate( - cache, catalogId, "removeCatalogByEngine", - () -> cache.invalidateCatalog(catalogId))); + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + routeSpecifiedEngine(engine, cache -> safeInvalidate( + cache, catalogId, "removeCatalogByEngine", + () -> cache.invalidateCatalog(catalogId))); + } finally { + lifecycleLock.unlock(); + } } public void invalidateDb(long catalogId, String dbName) { @@ -302,13 +410,13 @@ private void initEngineCaches() { } private void registerBuiltinEngineCaches() { - cacheRegistry.register(new DefaultExternalMetaCache(ENGINE_DEFAULT, commonRefreshExecutor)); - cacheRegistry.register(new HiveExternalMetaCache(commonRefreshExecutor, fileListingExecutor)); - cacheRegistry.register(new HudiExternalMetaCache(commonRefreshExecutor)); - cacheRegistry.register(new IcebergExternalMetaCache(commonRefreshExecutor)); - cacheRegistry.register(new PaimonExternalMetaCache(commonRefreshExecutor)); - cacheRegistry.register(new MaxComputeExternalMetaCache(commonRefreshExecutor)); - cacheRegistry.register(new DorisExternalMetaCache(commonRefreshExecutor)); + cacheRegistry.register(new DefaultExternalMetaCache(ENGINE_DEFAULT, commonRefreshExecutor, budgetManager)); + cacheRegistry.register(new HiveExternalMetaCache(commonRefreshExecutor, fileListingExecutor, budgetManager)); + cacheRegistry.register(new HudiExternalMetaCache(commonRefreshExecutor, budgetManager)); + cacheRegistry.register(new IcebergExternalMetaCache(commonRefreshExecutor, budgetManager)); + cacheRegistry.register(new PaimonExternalMetaCache(commonRefreshExecutor, budgetManager)); + cacheRegistry.register(new MaxComputeExternalMetaCache(commonRefreshExecutor, budgetManager)); + cacheRegistry.register(new DorisExternalMetaCache(commonRefreshExecutor, budgetManager)); } private void routeCatalogEngines(long catalogId, Consumer action) { @@ -428,8 +536,9 @@ void replaceEngineCachesForTest(List caches) { * loading/invalidation. No engine-specific metadata (partitions/files/snapshots) is cached. */ private static class DefaultExternalMetaCache extends AbstractExternalMetaCache { - DefaultExternalMetaCache(String engine, ExecutorService refreshExecutor) { - super(engine, refreshExecutor); + DefaultExternalMetaCache(String engine, ExecutorService refreshExecutor, + ExternalMetaCacheBudgetManager budgetManager) { + super(engine, refreshExecutor, budgetManager); registerEntry(MetaCacheEntryDef.of( ENTRY_SCHEMA, SchemaCacheKey.class, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/DorisExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/DorisExternalMetaCache.java index d14ba5645bf269..e7487c065b5ce3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/DorisExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/doris/DorisExternalMetaCache.java @@ -26,6 +26,7 @@ import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; @@ -72,7 +73,11 @@ public class DorisExternalMetaCache extends AbstractExternalMetaCache { private final EntryHandle schemaEntry; public DorisExternalMetaCache(ExecutorService refreshExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public DorisExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); backendsEntry = registerEntry(MetaCacheEntryDef.contextualOnly( ENTRY_BACKENDS, String.class, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java index e2d73fd7a16edf..883a38780149d7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java @@ -28,10 +28,8 @@ import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.InitCatalogLog; import org.apache.doris.datasource.SessionContext; -import org.apache.doris.datasource.hudi.HudiExternalMetaCache; import org.apache.doris.datasource.iceberg.IcebergMetadataOps; import org.apache.doris.datasource.iceberg.IcebergUtils; -import org.apache.doris.datasource.metacache.CacheSpec; import org.apache.doris.datasource.operations.ExternalMetadataOperations; import org.apache.doris.datasource.property.metastore.AbstractHiveProperties; import org.apache.doris.fs.FileSystemProvider; @@ -218,10 +216,6 @@ public void notifyPropertiesUpdated(Map updatedProps) { if (Objects.nonNull(fileMetaCacheTtl) || Objects.nonNull(partitionCacheTtl)) { Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), HiveExternalMetaCache.ENGINE); } - if (updatedProps.keySet().stream() - .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, HudiExternalMetaCache.ENGINE))) { - Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), HudiExternalMetaCache.ENGINE); - } } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java new file mode 100644 index 00000000000000..c26a021890e880 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java @@ -0,0 +1,54 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.hive; + +import org.apache.doris.datasource.hive.HiveExternalMetaCache.HivePartitionValues; +import org.apache.doris.datasource.hive.HiveExternalMetaCache.PartitionValueCacheKey; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +/** Constant-time retained-weight formula for Hive partition-value cache entries. */ +final class HiveCacheSizeEstimator { + // Calibrated against complete 4.1 object graphs. The per-character reserve covers the + // partition name plus derived value/literal strings and therefore remains skew-sensitive. + private static final long ENTRY_BASE_BYTES = 2L * 1024L; + private static final long PARTITION_BASE_BYTES = 4L * 1024L; + private static final long PARTITION_COLUMN_BYTES = 384L; + private static final long PARTITION_NAME_CHARACTER_BYTES = 8L; + + private HiveCacheSizeEstimator() { + } + + static MetaCacheSizeEstimate estimatePartitionValuesEntry( + PartitionValueCacheKey key, HivePartitionValues value) { + long partitionCount = value.getIdToPartitionItem() == null + ? 0L : value.getIdToPartitionItem().size(); + long perPartitionBytes = MetaCacheWeightUtils.saturatedAdd( + PARTITION_BASE_BYTES, + MetaCacheWeightUtils.saturatedMultiply( + value.getPartitionColumnCount(), PARTITION_COLUMN_BYTES)); + long bytes = MetaCacheWeightUtils.saturatedAdd( + ENTRY_BASE_BYTES, MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply(partitionCount, perPartitionBytes)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply( + value.getPartitionNameCharacterCount(), PARTITION_NAME_CHARACTER_BYTES)); + return MetaCacheSizeEstimate.complete(bytes); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java index 73986138c51cb0..527497237b1bf2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java @@ -42,8 +42,12 @@ import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.doris.fs.DirectoryLister; import org.apache.doris.fs.FileSystemCache; import org.apache.doris.fs.FileSystemDirectoryLister; @@ -59,10 +63,12 @@ import com.google.common.base.Strings; import com.google.common.collect.BiMap; import com.google.common.collect.HashBiMap; +import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Streams; import lombok.Data; +import lombok.Getter; import org.apache.hadoop.fs.BlockLocation; import org.apache.hadoop.fs.Path; import org.apache.hadoop.hive.metastore.api.Partition; @@ -108,6 +114,7 @@ */ public class HiveExternalMetaCache extends AbstractExternalMetaCache { private static final Logger LOG = LogManager.getLogger(HiveExternalMetaCache.class); + private static final int PARTITION_EVENT_REPLACE_MAX_RETRIES = 8; public static final String ENGINE = "hive"; public static final String ENTRY_SCHEMA = "schema"; @@ -127,7 +134,12 @@ public class HiveExternalMetaCache extends AbstractExternalMetaCache { private final PartitionCacheCoordinator partitionCacheCoordinator = new PartitionCacheCoordinator(); public HiveExternalMetaCache(ExecutorService refreshExecutor, ExecutorService fileListingExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, fileListingExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public HiveExternalMetaCache(ExecutorService refreshExecutor, ExecutorService fileListingExecutor, + ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); this.fileListingExecutor = fileListingExecutor; schemaEntry = registerEntry(MetaCacheEntryDef.of( @@ -144,7 +156,8 @@ public HiveExternalMetaCache(ExecutorService refreshExecutor, ExecutorService fi CacheSpec.of( true, Config.external_cache_expire_time_seconds_after_access, - Config.max_hive_partition_table_cache_num))); + Config.max_hive_partition_table_cache_num)) + .withSizeEstimator((key, value) -> value.prepareForCachePublication(key))); partitionEntry = registerEntry(MetaCacheEntryDef.of( ENTRY_PARTITION, PartitionCacheKey.class, @@ -292,9 +305,12 @@ private HivePartitionValues loadPartitionValues(PartitionValueCacheKey key) { Map idToPartitionItem = Maps.newHashMapWithExpectedSize(partitionNames.size()); BiMap partitionNameToIdMap = HashBiMap.create(partitionNames.size()); + long partitionNameCharacterCount = 0L; String localDbName = nameMapping.getLocalDbName(); String localTblName = nameMapping.getLocalTblName(); for (String partitionName : partitionNames) { + partitionNameCharacterCount = MetaCacheWeightUtils.saturatedAdd( + partitionNameCharacterCount, partitionName.length()); long partitionId = Util.genIdByName(catalog.getName(), localDbName, localTblName, partitionName); ListPartitionItem listPartitionItem = toListPartitionItem(partitionName, key.types, catalog.getName()); idToPartitionItem.put(partitionId, listPartitionItem); @@ -302,7 +318,15 @@ private HivePartitionValues loadPartitionValues(PartitionValueCacheKey key) { } Map> partitionValuesMap = ListPartitionPrunerV2.getPartitionValuesMap(idToPartitionItem); - return new HivePartitionValues(idToPartitionItem, partitionNameToIdMap, partitionValuesMap); + HivePartitionValues partitionValues = + new HivePartitionValues(idToPartitionItem, partitionNameToIdMap, partitionValuesMap, + partitionNameCharacterCount, key.types == null ? 0 : key.types.size()); + preparePartitionValuesForPublication(partitionValues); + return partitionValues; + } + + private void preparePartitionValuesForPublication(HivePartitionValues partitionValues) { + partitionValues.rebuildSortedPartitionRangesForPublication(); } private ListPartitionItem toListPartitionItem(String partitionName, List types, String catalogName) { @@ -635,7 +659,7 @@ private void invalidatePartitionCache(NameMapping nameMapping, String partitionN List values = HiveUtil.toPartitionValues(partitionName); PartitionCacheKey partKey = new PartitionCacheKey(nameMapping, values); - HivePartition partition = partitionEntry.getIfPresent(partKey); + HivePartition partition = partitionEntry.peekIfPresent(partKey); if (partition == null) { // Partition metadata cache miss: the exact FileCacheKey cannot be rebuilt here because it // needs the partition path and input format carried by HivePartition. Invalidate this @@ -715,41 +739,62 @@ private void addPartitionsCache(NameMapping nameMapping, } PartitionValueCacheKey key = new PartitionValueCacheKey(nameMapping, partitionColumnTypes); - HivePartitionValues partitionValues = partitionValuesEntry.getIfPresent(key); - if (partitionValues == null) { - return; - } - - HivePartitionValues copy = partitionValues.copy(); - Map idToPartitionItemBefore = copy.getIdToPartitionItem(); - Map partitionNameToIdMapBefore = copy.getPartitionNameToIdMap(); - Map idToPartitionItem = new HashMap<>(); - HMSExternalCatalog catalog = hmsCatalog(catalogId); String localDbName = nameMapping.getLocalDbName(); String localTblName = nameMapping.getLocalTblName(); - for (String partitionName : partitionNames) { - if (partitionNameToIdMapBefore.containsKey(partitionName)) { - LOG.info("addPartitionsCache partitionName:[{}] has exist in table:[{}]", - partitionName, localTblName); - continue; + for (int attempt = 0; attempt < PARTITION_EVENT_REPLACE_MAX_RETRIES; attempt++) { + HivePartitionValues current = partitionValuesEntry.peekIfPresent(key); + if (current == null) { + // Fence a concurrent miss load that may have read HMS before this event. + partitionValuesEntry.invalidateKey(key); + return; } - long partitionId = Util.genIdByName(catalog.getName(), localDbName, localTblName, partitionName); - ListPartitionItem listPartitionItem = toListPartitionItem(partitionName, key.types, catalog.getName()); - idToPartitionItemBefore.put(partitionId, listPartitionItem); - idToPartitionItem.put(partitionId, listPartitionItem); - partitionNameToIdMapBefore.put(partitionName, partitionId); - } - Map> partitionValuesMapBefore = copy.getPartitionValuesMap(); - Map> partitionValuesMap = ListPartitionPrunerV2.getPartitionValuesMap(idToPartitionItem); - partitionValuesMapBefore.putAll(partitionValuesMap); - copy.rebuildSortedPartitionRanges(); - - HivePartitionValues partitionValuesCur = partitionValuesEntry.getIfPresent(key); - if (partitionValuesCur == partitionValues) { - partitionValuesEntry.put(key, copy); + HivePartitionValues copy = current.mutableCopy(); + Map allItems = copy.getIdToPartitionItem(); + Map allNames = copy.getPartitionNameToIdMap(); + Map addedItems = new HashMap<>(); + for (String partitionName : partitionNames) { + if (allNames.containsKey(partitionName)) { + LOG.info("addPartitionsCache partitionName:[{}] has exist in table:[{}]", + partitionName, localTblName); + continue; + } + long partitionId = Util.genIdByName( + catalog.getName(), localDbName, localTblName, partitionName); + ListPartitionItem item = toListPartitionItem(partitionName, key.types, catalog.getName()); + allItems.put(partitionId, item); + addedItems.put(partitionId, item); + allNames.put(partitionName, partitionId); + copy.addPartitionNameCharacters(partitionName.length()); + } + if (addedItems.isEmpty()) { + // Even a replay/no-op event must fence a refresh that started before the event. + // Otherwise that refresh could replace this already-correct graph with stale HMS data. + if (partitionValuesEntry.fenceInFlightLoadIfSame(key, current)) { + return; + } + continue; + } + copy.getPartitionValuesMap().putAll( + ListPartitionPrunerV2.getPartitionValuesMap(addedItems)); + preparePartitionValuesForPublication(copy); + + MetaCacheEntry.ReplaceResult result = partitionValuesEntry.tryReplace(key, current, copy); + if (result == MetaCacheEntry.ReplaceResult.REPLACED + || result == MetaCacheEntry.ReplaceResult.DISABLED) { + return; + } + if (result == MetaCacheEntry.ReplaceResult.REJECTED + && partitionValuesEntry.invalidateKeyIfSame(key, current)) { + LOG.warn("Invalidated stale partition-values cache after add event was rejected: {}", key); + return; + } } + // Repeated conflicts mean we cannot prove the cached graph contains this event. Force + // the next reader to rebuild it from HMS rather than retaining a possibly stale value. + partitionValuesEntry.invalidateKey(key); + LOG.warn("Invalidated partition-values cache after repeated add-event conflicts: {}", key); } private void dropPartitionsCache(ExternalTable dorisTable, @@ -765,41 +810,63 @@ private void dropPartitionsCache(ExternalTable dorisTable, } PartitionValueCacheKey key = new PartitionValueCacheKey(nameMapping, null); - HivePartitionValues partitionValues = partitionValuesEntry.getIfPresent(key); - if (partitionValues == null) { - return; + if (invalidPartitionCache) { + for (String partitionName : partitionNames) { + invalidatePartitionCache(nameMapping, partitionName); + } } - HivePartitionValues copy = partitionValues.copy(); - Map partitionNameToIdMapBefore = copy.getPartitionNameToIdMap(); - Map idToPartitionItemBefore = copy.getIdToPartitionItem(); - Map> partitionValuesMap = copy.getPartitionValuesMap(); - - for (String partitionName : partitionNames) { - if (!partitionNameToIdMapBefore.containsKey(partitionName)) { - LOG.info("dropPartitionsCache partitionName:[{}] not exist in table:[{}]", - partitionName, nameMapping.getFullLocalName()); + for (int attempt = 0; attempt < PARTITION_EVENT_REPLACE_MAX_RETRIES; attempt++) { + HivePartitionValues current = partitionValuesEntry.peekIfPresent(key); + if (current == null) { + // Fence a concurrent miss load that may have read HMS before this event. + partitionValuesEntry.invalidateKey(key); + return; + } + HivePartitionValues copy = current.mutableCopy(); + Map allNames = copy.getPartitionNameToIdMap(); + Map allItems = copy.getIdToPartitionItem(); + Map> allValues = copy.getPartitionValuesMap(); + boolean changed = false; + for (String partitionName : partitionNames) { + Long partitionId = allNames.remove(partitionName); + if (partitionId == null) { + LOG.info("dropPartitionsCache partitionName:[{}] not exist in table:[{}]", + partitionName, nameMapping.getFullLocalName()); + continue; + } + allItems.remove(partitionId); + allValues.remove(partitionId); + copy.removePartitionNameCharacters(partitionName.length()); + changed = true; + } + if (!changed) { + // See the add-event no-op path: event ordering still has to win over an older refresh. + if (partitionValuesEntry.fenceInFlightLoadIfSame(key, current)) { + return; + } continue; } - Long partitionId = partitionNameToIdMapBefore.remove(partitionName); - idToPartitionItemBefore.remove(partitionId); - partitionValuesMap.remove(partitionId); - - if (invalidPartitionCache) { - invalidatePartitionCache(nameMapping, partitionName); + preparePartitionValuesForPublication(copy); + MetaCacheEntry.ReplaceResult result = partitionValuesEntry.tryReplace(key, current, copy); + if (result == MetaCacheEntry.ReplaceResult.REPLACED + || result == MetaCacheEntry.ReplaceResult.DISABLED) { + return; + } + if (result == MetaCacheEntry.ReplaceResult.REJECTED + && partitionValuesEntry.invalidateKeyIfSame(key, current)) { + LOG.warn("Invalidated stale partition-values cache after drop event was rejected: {}", key); + return; } } - - copy.rebuildSortedPartitionRanges(); - HivePartitionValues partitionValuesCur = partitionValuesEntry.getIfPresent(key); - if (partitionValuesCur == partitionValues) { - partitionValuesEntry.put(key, copy); - } + partitionValuesEntry.invalidateKey(key); + LOG.warn("Invalidated partition-values cache after repeated drop-event conflicts: {}", key); } } @VisibleForTesting public void putPartitionValuesCacheForTest(PartitionValueCacheKey key, HivePartitionValues values) { + preparePartitionValuesForPublication(values); partitionValuesEntry.get(key.getNameMapping().getCtlId()).put(key, values); } @@ -842,15 +909,15 @@ public List getFilesByTransaction(List partitions /** * The key of hive partition values cache. */ - @Data + @Getter public static class PartitionValueCacheKey { - private NameMapping nameMapping; + private final NameMapping nameMapping; // Not part of cache identity. - private List types; + private final List types; public PartitionValueCacheKey(NameMapping nameMapping, List types) { this.nameMapping = nameMapping; - this.types = types; + this.types = types == null ? null : ImmutableList.copyOf(types); } @Override @@ -1037,7 +1104,7 @@ public static class HiveFileStatus { AcidInfo acidInfo; } - @Data + @Getter public static class HivePartitionValues { private BiMap partitionNameToIdMap; private Map idToPartitionItem; @@ -1045,6 +1112,12 @@ public static class HivePartitionValues { // Sorted partition ranges for binary search filtering. private SortedPartitionRanges sortedPartitionRanges; + // Prepared once after construction/update; the cache weigher only reads this value. + private transient volatile MetaCacheSizeEstimate sizeEstimate; + // Maintained while the metadata is already being loaded or updated. Admission only reads it. + private long partitionNameCharacterCount; + private int partitionColumnCount; + private transient boolean sortedPartitionRangesPrepared; public HivePartitionValues() { } @@ -1052,22 +1125,98 @@ public HivePartitionValues() { public HivePartitionValues(Map idToPartitionItem, BiMap partitionNameToIdMap, Map> partitionValuesMap) { + this(idToPartitionItem, partitionNameToIdMap, partitionValuesMap, + countPartitionNameCharacters(partitionNameToIdMap), + inferPartitionColumnCount(partitionValuesMap)); + } + + HivePartitionValues(Map idToPartitionItem, + BiMap partitionNameToIdMap, + Map> partitionValuesMap, + long partitionNameCharacterCount, + int partitionColumnCount) { this.idToPartitionItem = idToPartitionItem; this.partitionNameToIdMap = partitionNameToIdMap; this.partitionValuesMap = partitionValuesMap; - this.sortedPartitionRanges = buildSortedPartitionRanges(); + this.partitionNameCharacterCount = partitionNameCharacterCount; + this.partitionColumnCount = partitionColumnCount; } - public HivePartitionValues copy() { + HivePartitionValues mutableCopy() { HivePartitionValues copy = new HivePartitionValues(); - copy.setPartitionNameToIdMap(partitionNameToIdMap == null ? null : HashBiMap.create(partitionNameToIdMap)); - copy.setIdToPartitionItem(idToPartitionItem == null ? null : Maps.newHashMap(idToPartitionItem)); - copy.setPartitionValuesMap(partitionValuesMap == null ? null : Maps.newHashMap(partitionValuesMap)); + copy.partitionNameToIdMap = partitionNameToIdMap == null ? null : HashBiMap.create(partitionNameToIdMap); + copy.idToPartitionItem = idToPartitionItem == null ? null : Maps.newHashMap(idToPartitionItem); + copy.partitionValuesMap = partitionValuesMap == null ? null : Maps.newHashMap(partitionValuesMap); + copy.partitionNameCharacterCount = partitionNameCharacterCount; + copy.partitionColumnCount = partitionColumnCount; return copy; } - public void rebuildSortedPartitionRanges() { - this.sortedPartitionRanges = buildSortedPartitionRanges(); + /** Compatibility hook for tests and benchmarks; publication uses copy-on-write updates. */ + void sealForPublication() { + if (!sortedPartitionRangesPrepared) { + rebuildSortedPartitionRangesForPublication(); + } + } + + void rebuildSortedPartitionRangesForPublication() { + sortedPartitionRanges = buildSortedPartitionRanges(); + sortedPartitionRangesPrepared = true; + } + + MetaCacheSizeEstimate prepareForCachePublication(PartitionValueCacheKey key) { + if (sizeEstimate == null) { + sizeEstimate = MetaCacheSizeEstimator.estimateSafely( + "hive_partition_values_preparation_failed", () -> { + prepareSizeEstimate(key); + return getSizeEstimate(); + }); + } + return sizeEstimate; + } + + public MetaCacheSizeEstimate getSizeEstimate() { + MetaCacheSizeEstimate result = sizeEstimate; + return result == null ? MetaCacheSizeEstimate.incomplete("estimate_not_prepared") : result; + } + + void prepareSizeEstimate(PartitionValueCacheKey key) { + sizeEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, this); + } + + long getPartitionNameCharacterCount() { + return partitionNameCharacterCount; + } + + int getPartitionColumnCount() { + return partitionColumnCount; + } + + private void addPartitionNameCharacters(int characters) { + partitionNameCharacterCount = MetaCacheWeightUtils.saturatedAdd( + partitionNameCharacterCount, characters); + } + + private void removePartitionNameCharacters(int characters) { + partitionNameCharacterCount = Math.max(0L, partitionNameCharacterCount - characters); + } + + private static long countPartitionNameCharacters(BiMap names) { + long characters = 0L; + if (names != null) { + for (String name : names.keySet()) { + characters = MetaCacheWeightUtils.saturatedAdd(characters, name.length()); + } + } + return characters; + } + + private static int inferPartitionColumnCount(Map> values) { + if (values == null || values.isEmpty()) { + return 0; + } + List first = values.values().iterator().next(); + return first == null ? 0 : first.size(); } public java.util.Optional> getSortedPartitionRanges() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java index 74d2aa99900340..a83777b1e53e68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java @@ -28,6 +28,7 @@ import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HiveMetaStoreClientHelper; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; @@ -83,7 +84,11 @@ public class HudiExternalMetaCache extends AbstractExternalMetaCache { private final EntryHandle schemaEntry; public HudiExternalMetaCache(ExecutorService refreshExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public HudiExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); partitionEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_PARTITION, HudiPartitionCacheKey.class, TablePartitionValues.class, this::loadPartitionValuesCacheValue, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(HudiPartitionCacheKey::getNameMapping))); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java new file mode 100644 index 00000000000000..e60f78eef90182 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java @@ -0,0 +1,254 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.types.Type; +import org.apache.iceberg.types.Types; + +import java.nio.ByteBuffer; +import java.util.Map; + +/** Constant-time retained-weight formulas for Iceberg cache entries. */ +final class IcebergCacheSizeEstimator { + private static final long KEY_BASE_BYTES = 128L; + private static final long TABLE_BASE_BYTES = 16L * 1024L; + private static final long SCHEMA_VERSION_BYTES = 512L; + private static final long SCHEMA_FIELD_BYTES = 512L; + private static final long NESTED_SCHEMA_FIELD_BYTES = 512L; + private static final long PARTITION_SPEC_BYTES = 256L; + private static final long PARTITION_SPEC_FIELD_BYTES = 384L; + private static final long SORT_ORDER_BYTES = 256L; + private static final long SORT_FIELD_BYTES = 256L; + private static final long TABLE_PROPERTY_BYTES = 256L; + private static final long CURRENT_SNAPSHOT_BYTES = 512L; + private static final long PARTITION_BYTES = 512L; + private static final long PARTITION_ALIAS_BYTES = 256L; + private static final long NAME_MAPPING_ENTRY_BYTES = 256L; + private static final long MANIFEST_ENTRY_BASE_BYTES = 256L; + private static final long DATA_FILE_BYTES = 16L * 1024L; + private static final long DELETE_FILE_BYTES = 18L * 1024L; + private static final long FILE_METRIC_ENTRY_BYTES = 160L; + + private IcebergCacheSizeEstimator() { + } + + static MetaCacheSizeEstimate estimateTableEntry(NameMapping key, IcebergTableCacheValue value) { + Table table = value.getRetainedIcebergTable(); + MetaCacheSizeEstimate support = checkSupportedTable(table); + if (!support.isComplete()) { + return support; + } + long bytes = MetaCacheWeightUtils.saturatedAdd( + KEY_BASE_BYTES, MetaCacheWeightUtils.estimatedNameMappingBytes(key)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedTablePayloadBytes()); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getRetainedCurrentSnapshotPayloadBytes()); + return MetaCacheSizeEstimate.complete(bytes); + } + + static MetaCacheSizeEstimate estimateSnapshotEntry( + IcebergSnapshotEntryKey key, IcebergSnapshotCacheValue value) { + long bytes = KEY_BASE_BYTES; + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(key.getTableUuid())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(key.getMetadataFileLocation())); + + IcebergPartitionInfo partitionInfo = value.getPartitionInfo(); + bytes = addCount(bytes, partitionInfo.getNameToPartitionItem().size(), PARTITION_BYTES); + bytes = addCount(bytes, partitionInfo.getNameToIcebergPartition().size(), PARTITION_BYTES); + bytes = addCount(bytes, partitionInfo.getNameToIcebergPartitionNames().size(), PARTITION_ALIAS_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, partitionInfo.getRetainedPayloadBytes()); + bytes = addCount(bytes, value.getNameMapping().map(java.util.Map::size).orElse(0), + NAME_MAPPING_ENTRY_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getRetainedNameMappingPayloadBytes()); + + if (value.getRetainedIcebergTable().isPresent()) { + Table table = value.getRetainedIcebergTable().get(); + MetaCacheSizeEstimate support = checkSupportedTable(table); + if (!support.isComplete()) { + return support; + } + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table)); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getRetainedTablePayloadBytes()); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getRetainedCurrentSnapshotPayloadBytes()); + } + return MetaCacheSizeEstimate.complete(bytes); + } + + static MetaCacheSizeEstimate estimateManifestEntry( + IcebergManifestEntryKey key, ManifestCacheValue value) { + long bytes = MetaCacheWeightUtils.saturatedAdd( + MANIFEST_ENTRY_BASE_BYTES, + MetaCacheWeightUtils.estimatedStringBytes(key.getManifestPath())); + bytes = addCount(bytes, value.getDataFiles().size(), DATA_FILE_BYTES); + bytes = addCount(bytes, value.getDeleteFiles().size(), DELETE_FILE_BYTES); + bytes = addCount(bytes, value.getDataFileMetricEntryCount(), FILE_METRIC_ENTRY_BYTES); + bytes = addCount(bytes, value.getDeleteFileMetricEntryCount(), FILE_METRIC_ENTRY_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedPayloadBytes()); + return MetaCacheSizeEstimate.complete(bytes); + } + + private static MetaCacheSizeEstimate checkSupportedTable(Table table) { + if (table == null) { + return MetaCacheSizeEstimate.incomplete("missing_iceberg_table"); + } + if (!(table instanceof HasTableOperations)) { + return MetaCacheSizeEstimate.incomplete( + "unsupported_iceberg_table:" + table.getClass().getName()); + } + TableMetadata metadata = ((HasTableOperations) table).operations().current(); + if (metadata == null) { + return MetaCacheSizeEstimate.incomplete("missing_iceberg_table_metadata"); + } + if (metadata.metadataFileLocation() == null + || metadata.metadataFileLocation().isEmpty()) { + return MetaCacheSizeEstimate.incomplete("missing_iceberg_metadata_location"); + } + return MetaCacheSizeEstimate.complete(1L); + } + + /** Reads only metadata collection sizes and a constant number of strings; no FileIO is used. */ + private static long estimateTable(Table table) { + TableMetadata metadata = ((HasTableOperations) table).operations().current(); + long bytes = MetaCacheWeightUtils.saturatedAdd( + TABLE_BASE_BYTES, MetaCacheWeightUtils.estimatedStringBytes(table.name())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(metadata.location())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(metadata.metadataFileLocation())); + + bytes = addCount(bytes, metadata.properties().size(), TABLE_PROPERTY_BYTES); + if (metadata.currentSnapshot() != null) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, CURRENT_SNAPSHOT_BYTES); + } + return bytes; + } + + /** Captures exact historical cardinalities and skew-sensitive payload once before admission. */ + static long retainedTablePayloadBytes(Table table) { + if (!(table instanceof HasTableOperations)) { + return 0L; + } + TableMetadata metadata = ((HasTableOperations) table).operations().current(); + if (metadata == null) { + return 0L; + } + + long bytes = 0L; + for (Schema schema : metadata.schemas()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SCHEMA_VERSION_BYTES); + for (Types.NestedField field : schema.columns()) { + bytes = addFieldPayload(bytes, field, false); + } + } + for (PartitionSpec spec : metadata.specs()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, PARTITION_SPEC_BYTES); + for (org.apache.iceberg.PartitionField field : spec.fields()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, PARTITION_SPEC_FIELD_BYTES); + bytes = addString(bytes, field.name()); + } + } + for (SortOrder sortOrder : metadata.sortOrders()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SORT_ORDER_BYTES); + bytes = addCount(bytes, sortOrder.fields().size(), SORT_FIELD_BYTES); + } + for (Map.Entry property : metadata.properties().entrySet()) { + bytes = addString(bytes, property.getKey()); + bytes = addString(bytes, property.getValue()); + } + bytes = addString(bytes, metadata.uuid()); + return bytes; + } + + private static long addFieldPayload(long bytes, Types.NestedField field, boolean nested) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + nested ? NESTED_SCHEMA_FIELD_BYTES : SCHEMA_FIELD_BYTES); + bytes = addString(bytes, field.name()); + bytes = addString(bytes, field.doc()); + bytes = addDefaultPayload(bytes, field.initialDefault()); + bytes = addDefaultPayload(bytes, field.writeDefault()); + return addTypePayload(bytes, field.type()); + } + + private static long addTypePayload(long bytes, Type type) { + if (type.isStructType()) { + for (Types.NestedField field : type.asStructType().fields()) { + bytes = addFieldPayload(bytes, field, true); + } + } else if (type.isListType()) { + bytes = addTypePayload(bytes, type.asListType().elementType()); + } else if (type.isMapType()) { + bytes = addTypePayload(bytes, type.asMapType().keyType()); + bytes = addTypePayload(bytes, type.asMapType().valueType()); + } + return bytes; + } + + private static long addDefaultPayload(long bytes, Object value) { + if (value instanceof CharSequence) { + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedCharSequenceBytes((CharSequence) value)); + } else if (value instanceof ByteBuffer) { + return MetaCacheWeightUtils.saturatedAdd(bytes, ((ByteBuffer) value).capacity()); + } else if (value instanceof byte[]) { + return MetaCacheWeightUtils.saturatedAdd(bytes, ((byte[]) value).length); + } + return bytes; + } + + private static long addString(long bytes, String value) { + return MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedStringBytes(value)); + } + + private static long addStringMap(long bytes, Map values, long entryBytes) { + if (values == null) { + return bytes; + } + bytes = addCount(bytes, values.size(), entryBytes); + for (Map.Entry entry : values.entrySet()) { + bytes = addString(bytes, entry.getKey()); + bytes = addString(bytes, entry.getValue()); + } + return bytes; + } + + private static long addCount(long bytes, long count, long bytesPerItem) { + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply(count, bytesPerItem)); + } + +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index 8407a29d8908a9..e6402dd234839c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -17,7 +17,6 @@ package org.apache.doris.datasource.iceberg; -import org.apache.doris.catalog.Env; import org.apache.doris.common.AnalysisException; import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.CatalogIf; @@ -29,9 +28,12 @@ import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.commons.lang3.exception.ExceptionUtils; @@ -47,6 +49,8 @@ import java.io.IOException; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.function.Consumer; @@ -55,8 +59,8 @@ * *

Registered entries: *

    - *
  • {@code table}: loaded Iceberg {@link Table} instances per Doris table mapping, each - * memoizing its latest snapshot runtime projection
    • + *
  • {@code table}: loaded Iceberg {@link Table} instances per Doris table mapping
    • + *
  • {@code snapshot}: immutable snapshot projections keyed by a stable metadata generation
    • *
  • {@code view}: loaded Iceberg {@link View} instances
    • *
  • {@code manifest}: parsed manifest payload ({@link ManifestCacheValue}) keyed by * manifest path and content type
    • @@ -69,7 +73,7 @@ *

    Invalidation behavior: *

      *
    • catalog invalidation clears all entries and drops Iceberg {@link ManifestFiles} IO cache
      • - *
    • db/table invalidation clears table/view/schema entries, while keeping manifest entries
      • + *
    • db/table invalidation clears table/snapshot/view/schema entries, while keeping manifest entries
      • *
    • partition-level invalidation falls back to table-level invalidation
      • *
    */ @@ -78,26 +82,40 @@ public class IcebergExternalMetaCache extends AbstractExternalMetaCache { public static final String ENGINE = "iceberg"; public static final String ENTRY_TABLE = "table"; + public static final String ENTRY_SNAPSHOT = "snapshot"; public static final String ENTRY_VIEW = "view"; public static final String ENTRY_MANIFEST = "manifest"; public static final String ENTRY_SCHEMA = "schema"; private static final long DEFAULT_MANIFEST_CACHE_CAPACITY = 100_000L; private final EntryHandle tableEntry; + private final EntryHandle snapshotEntry; private final EntryHandle viewEntry; private final EntryHandle manifestEntry; private final EntryHandle schemaEntry; public IcebergExternalMetaCache(ExecutorService refreshExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public IcebergExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); tableEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_TABLE, NameMapping.class, IcebergTableCacheValue.class, this::loadTableCacheValue, defaultEntryCacheSpec(), - MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping))); + MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping)) + .withSizeEstimator(this::prepareTableForCachePublication)); + snapshotEntry = registerEntry(MetaCacheEntryDef.contextualOnly(ENTRY_SNAPSHOT, + IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class, defaultEntryCacheSpec(), + MetaCacheEntryInvalidation.forNameMapping(IcebergSnapshotEntryKey::getNameMapping)) + .withSizeEstimator((key, value) -> value.prepareForCachePublication(key))); viewEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_VIEW, NameMapping.class, View.class, this::loadView, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping))); manifestEntry = registerEntry(MetaCacheEntryDef.contextualOnly(ENTRY_MANIFEST, IcebergManifestEntryKey.class, ManifestCacheValue.class, - CacheSpec.of(false, CacheSpec.CACHE_NO_TTL, DEFAULT_MANIFEST_CACHE_CAPACITY))); + CacheSpec.of(false, CacheSpec.CACHE_NO_TTL, DEFAULT_MANIFEST_CACHE_CAPACITY)) + .withSizeEstimator((key, value) -> MetaCacheSizeEstimator.estimateSafely( + "iceberg_manifest_preparation_failed", + () -> IcebergCacheSizeEstimator.estimateManifestEntry(key, value)))); schemaEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_SCHEMA, IcebergSchemaCacheKey.class, SchemaCacheValue.class, this::loadSchemaCacheValue, defaultSchemaCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(IcebergSchemaCacheKey::getNameMapping))); @@ -108,13 +126,95 @@ public Table getIcebergTable(ExternalTable dorisTable) { return tableEntry.get(nameMapping.getCtlId()).get(nameMapping).getIcebergTable(); } + public Table getWritableIcebergTable(ExternalTable dorisTable) { + NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); + IcebergTableCacheValue tableValue = + tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + CatalogIf catalog = getCatalog(nameMapping.getCtlId()); + if (catalog == null) { + throw new RuntimeException("Cannot find catalog " + nameMapping.getCtlId() + + " when loading a writable Iceberg table"); + } + IcebergMetadataOps ops = resolveMetadataOps(catalog); + Table liveTable = executeAuthenticated(catalog, () -> ops.loadTable( + nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName())); + try { + return tableValue.getWritableIcebergTable(liveTable); + } catch (IcebergSnapshotCacheValue.StaleMetadataException e) { + MetaCacheEntry entry = + tableEntry.get(nameMapping.getCtlId()); + entry.invalidateKeyIfSame(nameMapping, tableValue); + IcebergTableCacheValue refreshedValue = entry.get(nameMapping); + Table refreshedLiveTable = executeAuthenticated(catalog, () -> ops.loadTable( + nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName())); + return refreshedValue.getWritableIcebergTable(refreshedLiveTable); + } + } + + Table getQueryScopedIcebergTable(ExternalTable dorisTable) { + NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); + MetaCacheEntry entry = + tableEntry.get(nameMapping.getCtlId()); + IcebergTableCacheValue tableValue = + entry.get(nameMapping); + try { + return createQueryTable(nameMapping, tableValue); + } catch (IcebergSnapshotCacheValue.StaleMetadataException e) { + entry.invalidateKeyIfSame(nameMapping, tableValue); + return createQueryTable(nameMapping, entry.get(nameMapping)); + } + } + + private Table createQueryTable( + NameMapping nameMapping, IcebergTableCacheValue tableValue) { + boolean isolateForQueries = tableValue.isQueryIsolationPrepared() + || snapshotEntry.get(nameMapping.getCtlId()).isWeightBounded(); + if (!isolateForQueries) { + return tableValue.getIcebergTable(); + } + Table queryTable = tableValue.newQueryScopedTable(); + IcebergSnapshotCacheValue.loadQueryMetadataForStatement(queryTable); + return queryTable; + } + public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - return tableEntry.get(nameMapping.getCtlId()).get(nameMapping).getLatestSnapshotCacheValue(); + IcebergTableCacheValue tableValue = + tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + Table retainedTable = tableValue.getRetainedIcebergTable(); + java.util.Optional optionalKey = + IcebergSnapshotEntryKey.tryCreate(nameMapping, retainedTable); + if (!optionalKey.isPresent()) { + boolean isolateForQueries = tableValue.isQueryIsolationPrepared(); + return executeAuthenticated(nameMapping.getCtlId(), + () -> loadSnapshotProjection( + dorisTable, + isolateForQueries ? tableValue.newQueryScopedTable() + : tableValue.getIcebergTable(), + tableValue.getRetainedIcebergTable(), + tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries)); + } + IcebergSnapshotEntryKey key = optionalKey.get(); + MetaCacheEntry entry = + snapshotEntry.get(nameMapping.getCtlId()); + boolean isolateForQueries = tableValue.isQueryIsolationPrepared() + || entry.isWeightBounded(); + return entry.get(key, ignored -> executeAuthenticated(nameMapping.getCtlId(), () -> { + Table projectionTable = isolateForQueries + ? tableValue.newQueryScopedTable() : tableValue.getIcebergTable(); + IcebergSnapshotCacheValue value = loadSnapshotProjection( + dorisTable, projectionTable, + tableValue.getRetainedIcebergTable(), + tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries); + if (entry.isWeightBounded()) { + value.prepareForCachePublication(key); + } + return value; + })); } public List getSnapshotList(ExternalTable dorisTable) { - Table icebergTable = getIcebergTable(dorisTable); + Table icebergTable = getQueryScopedIcebergTable(dorisTable); List snapshots = com.google.common.collect.Lists.newArrayList(); com.google.common.collect.Iterables.addAll(snapshots, icebergTable.snapshots()); return snapshots; @@ -139,11 +239,12 @@ public ManifestCacheValue getManifestCacheValue(ExternalTable dorisTable, MetaCacheEntry manifestEntry = this.manifestEntry.get(nameMapping.getCtlId()); IcebergManifestEntryKey key = IcebergManifestEntryKey.of(manifest); - boolean hit = manifestEntry.getIfPresent(key) != null; + boolean hit = manifestEntry.peekIfPresent(key) != null; if (cacheHitRecorder != null) { cacheHitRecorder.accept(hit); } - return manifestEntry.get(key, ignored -> loadManifestCacheValue(manifest, icebergTable, key.getContent())); + return manifestEntry.get(key, + ignored -> loadManifestCacheValue(manifest, icebergTable, key.getContent())); } @Override @@ -159,25 +260,33 @@ public void invalidateCatalogEntries(long catalogId) { } private IcebergTableCacheValue loadTableCacheValue(NameMapping nameMapping) { - CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(nameMapping.getCtlId()); + CatalogIf catalog = getCatalog(nameMapping.getCtlId()); if (catalog == null) { throw new RuntimeException(String.format("Cannot find catalog %d when loading table %s/%s.", nameMapping.getCtlId(), nameMapping.getLocalDbName(), nameMapping.getLocalTblName())); } IcebergMetadataOps ops = resolveMetadataOps(catalog); - try { - Table table = ((ExternalCatalog) catalog).getExecutionAuthenticator() - .execute(() -> ops.loadTable(nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName())); - ExternalTable dorisTable = findExternalTable(nameMapping, ENGINE); - return new IcebergTableCacheValue(table, () -> loadSnapshotProjection(dorisTable, table)); - } catch (Exception e) { - throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); - } + return executeAuthenticated(catalog, () -> { + Table table = ops.loadTable(nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName()); + IcebergTableCacheValue value = new IcebergTableCacheValue( + table, ((ExternalCatalog) catalog).getExecutionAuthenticator()); + MetaCacheEntry currentEntry = + tableEntry.getIfInitialized(nameMapping.getCtlId()); + if (currentEntry != null && currentEntry.isWeightBounded()) { + prepareTableForCachePublication(nameMapping, value); + } + return value; + }); + } + + MetaCacheSizeEstimate prepareTableForCachePublication( + NameMapping nameMapping, IcebergTableCacheValue value) { + return value.prepareForCachePublication(nameMapping); } private View loadView(NameMapping nameMapping) { - CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(nameMapping.getCtlId()); + CatalogIf catalog = getCatalog(nameMapping.getCtlId()); if (!(catalog instanceof IcebergExternalCatalog)) { return null; } @@ -199,10 +308,9 @@ private ManifestCacheValue loadManifestCacheValue(org.apache.iceberg.ManifestFil } try { if (content == ManifestContent.DELETES) { - return ManifestCacheValue.forDeleteFiles( - loadDeleteFiles(manifest, icebergTable)); + return loadDeleteFiles(manifest, icebergTable); } - return ManifestCacheValue.forDataFiles(loadDataFiles(manifest, icebergTable)); + return loadDataFiles(manifest, icebergTable); } catch (IOException e) { throw new CacheException("Failed to read manifest %s", e, manifest.path()); } @@ -216,27 +324,32 @@ private SchemaCacheValue loadSchemaCacheValue(IcebergSchemaCacheKey key) { key.getNameMapping().getLocalTblName(), key.getSchemaId())); } - private IcebergSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTable, Table icebergTable) { + private IcebergSnapshotCacheValue loadSnapshotProjection( + ExternalTable dorisTable, Table projectionTable, Table retainedTable, + String retainedCurrentSnapshotJson, boolean isolateForQueries) { if (!(dorisTable instanceof MTMVRelatedTableIf)) { throw new RuntimeException(String.format("Table %s.%s is not a valid MTMV related table.", dorisTable.getDbName(), dorisTable.getName())); } try { - // Freeze before deriving snapshot, partitions, and aliases; BaseTable accessors share - // refreshable operations and otherwise could mix two concurrent metadata generations. - Table retainedTable = IcebergSnapshotCacheValue.retainTableGeneration(icebergTable); MTMVRelatedTableIf table = (MTMVRelatedTableIf) dorisTable; - IcebergSnapshot latestIcebergSnapshot = IcebergUtils.getLatestIcebergSnapshot(retainedTable); + IcebergSnapshot latestIcebergSnapshot = IcebergUtils.getLatestIcebergSnapshot(projectionTable); IcebergPartitionInfo icebergPartitionInfo; if (!table.isValidRelatedTable()) { icebergPartitionInfo = IcebergPartitionInfo.empty(); } else { - icebergPartitionInfo = IcebergUtils.loadPartitionInfo(dorisTable, retainedTable, + icebergPartitionInfo = IcebergUtils.loadPartitionInfo(dorisTable, projectionTable, latestIcebergSnapshot.getSnapshotId(), latestIcebergSnapshot.getSchemaId()); } - return new IcebergSnapshotCacheValue( - icebergPartitionInfo, latestIcebergSnapshot, IcebergUtils.getNameMapping(retainedTable), - retainedTable); + Optional>> nameMapping = + IcebergUtils.getNameMapping(projectionTable); + return isolateForQueries + ? new IcebergSnapshotCacheValue( + icebergPartitionInfo, latestIcebergSnapshot, nameMapping, + retainedTable, retainedCurrentSnapshotJson) + : new IcebergSnapshotCacheValue( + icebergPartitionInfo, latestIcebergSnapshot, nameMapping, + retainedTable); } catch (AnalysisException e) { throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); } @@ -251,32 +364,56 @@ private IcebergMetadataOps resolveMetadataOps(CatalogIf catalog) { throw new RuntimeException("Only support 'hms' and 'iceberg' type for iceberg table"); } + private T executeAuthenticated(long catalogId, Callable task) { + CatalogIf catalog = getCatalog(catalogId); + if (catalog == null) { + throw new RuntimeException("Cannot find catalog " + catalogId + " when loading Iceberg metadata."); + } + return executeAuthenticated(catalog, task); + } + + private T executeAuthenticated(CatalogIf catalog, Callable task) { + if (!(catalog instanceof ExternalCatalog)) { + throw new RuntimeException("Iceberg metadata cache requires an external catalog"); + } + try { + return ((ExternalCatalog) catalog).getExecutionAuthenticator().execute(task); + } catch (Exception e) { + throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); + } + } + @Override protected Map catalogPropertyCompatibilityMap() { - return singleCompatibilityMap(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND, ENTRY_SCHEMA); + Map compatibility = new java.util.HashMap<>( + singleCompatibilityMap(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND, ENTRY_SCHEMA)); + compatibility.put("meta.cache.iceberg.table.enable", "meta.cache.iceberg.snapshot.enable"); + compatibility.put("meta.cache.iceberg.table.ttl-second", "meta.cache.iceberg.snapshot.ttl-second"); + compatibility.put("meta.cache.iceberg.table.capacity", "meta.cache.iceberg.snapshot.capacity"); + return compatibility; } - private List loadDataFiles(org.apache.iceberg.ManifestFile manifest, Table table) + private ManifestCacheValue loadDataFiles(org.apache.iceberg.ManifestFile manifest, Table table) throws IOException { - List dataFiles = com.google.common.collect.Lists.newArrayList(); + ManifestCacheValue.Builder builder = ManifestCacheValue.dataFilesBuilder(); try (ManifestReader reader = ManifestFiles.read(manifest, table.io())) { for (org.apache.iceberg.DataFile dataFile : reader) { - dataFiles.add(dataFile.copy()); + builder.addDataFile(dataFile.copy()); } } - return dataFiles; + return builder.build(); } - private List loadDeleteFiles(org.apache.iceberg.ManifestFile manifest, Table table) + private ManifestCacheValue loadDeleteFiles(org.apache.iceberg.ManifestFile manifest, Table table) throws IOException { - List deleteFiles = com.google.common.collect.Lists.newArrayList(); + ManifestCacheValue.Builder builder = ManifestCacheValue.deleteFilesBuilder(); try (ManifestReader reader = ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { for (org.apache.iceberg.DeleteFile deleteFile : reader) { - deleteFiles.add(deleteFile.copy()); + builder.addDeleteFile(deleteFile.copy()); } } - return deleteFiles; + return builder.build(); } private void dropManifestFileIoCacheForCatalog(long catalogId) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java index 5d59440a5a62b1..9c9ee6d53b6416 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java @@ -148,6 +148,10 @@ public Table getIcebergTable() { return IcebergUtils.getIcebergTable(this); } + public Table getWritableIcebergTable() { + return IcebergUtils.getWritableIcebergTable(this); + } + @Override public String getComment() { return properties().getOrDefault(TABLE_COMMENT_PROP, ""); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index bcbe03de6e6d01..d996f80754a37f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -483,7 +483,7 @@ public void truncateTableImpl(ExternalTable dorisTable, List partitions) @Override public void createOrReplaceBranchImpl(ExternalTable dorisTable, CreateOrReplaceBranchInfo branchInfo) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); BranchOptions branchOptions = branchInfo.getBranchOptions(); Long snapshotId = branchOptions.getSnapshotId() @@ -571,7 +571,7 @@ public void afterOperateOnBranchOrTag(String dbName, String tblName) { @Override public void createOrReplaceTagImpl(ExternalTable dorisTable, CreateOrReplaceTagInfo tagInfo) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); TagOptions tagOptions = tagInfo.getTagOptions(); Long snapshotId = tagOptions.getSnapshotId() .orElse( @@ -623,7 +623,7 @@ public void createOrReplaceTagImpl(ExternalTable dorisTable, CreateOrReplaceTagI public void dropTagImpl(ExternalTable dorisTable, DropTagInfo tagInfo) throws UserException { String tagName = tagInfo.getTagName(); boolean ifExists = tagInfo.getIfExists(); - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); SnapshotRef snapshotRef = icebergTable.refs().get(tagName); if (snapshotRef != null || !ifExists) { @@ -644,7 +644,7 @@ public void dropTagImpl(ExternalTable dorisTable, DropTagInfo tagInfo) throws Us public void dropBranchImpl(ExternalTable dorisTable, DropBranchInfo branchInfo) throws UserException { String branchName = branchInfo.getBranchName(); boolean ifExists = branchInfo.getIfExists(); - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); SnapshotRef snapshotRef = icebergTable.refs().get(branchName); if (snapshotRef != null || !ifExists) { @@ -747,7 +747,7 @@ private void refreshTable(ExternalTable dorisTable, long updateTime) { public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { validateAddColumnMetadata(column, true); - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); validateVariantSchema(icebergTable, column.getType(), column.getName(), false, true); validateRowLineageColumnMutation(icebergTable, column.getName(), "add"); Schema schema = icebergTable.schema(); @@ -778,7 +778,7 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co if (!column.isAllowNull()) { throw new UserException("New nested field '" + columnPath.getFullPath() + "' must be nullable"); } - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); validateVariantSchema(icebergTable, column.getType(), columnPath.getFullPath(), true, true); ResolvedColumnPath parentPath = resolveColumnPath(icebergTable.schema(), columnPath.getParentPath(), "add"); if (!parentPath.getType().isStructType()) { @@ -808,7 +808,7 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co @Override public void addColumns(ExternalTable dorisTable, List columns, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); for (Column column : columns) { validateAddColumnMetadata(column, true); validateVariantSchema(icebergTable, column.getType(), column.getName(), false, true); @@ -831,7 +831,7 @@ public void addColumns(ExternalTable dorisTable, List columns, long upda @Override public void dropColumn(ExternalTable dorisTable, String columnName, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); validateRowLineageColumnMutation(icebergTable, columnName, "drop"); ResolvedColumnPath columnPath = resolveColumnPath(icebergTable.schema(), ColumnPath.of(columnName), "drop"); UpdateSchema updateSchema = icebergTable.updateSchema(); @@ -851,7 +851,7 @@ public void dropColumn(ExternalTable dorisTable, ColumnPath columnPath, long upd dropColumn(dorisTable, columnPath.getTopLevelName(), updateTime); return; } - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); ResolvedColumnPath resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "drop"); UpdateSchema updateSchema = icebergTable.updateSchema(); @@ -868,7 +868,7 @@ public void dropColumn(ExternalTable dorisTable, ColumnPath columnPath, long upd @Override public void renameColumn(ExternalTable dorisTable, String oldName, String newName, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); validateRowLineageColumnMutation(icebergTable, oldName, "rename"); validateRowLineageColumnMutation(icebergTable, newName, "rename to"); Schema schema = icebergTable.schema(); @@ -893,7 +893,7 @@ public void renameColumn(ExternalTable dorisTable, ColumnPath columnPath, String renameColumn(dorisTable, columnPath.getTopLevelName(), newName, updateTime); return; } - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); ResolvedColumnPath resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "rename"); ResolvedColumnPath parentPath = resolveColumnPath(icebergTable.schema(), columnPath.getParentPath(), "rename"); validateNoCaseInsensitiveSiblingCollision(parentPath.getType().asStructType(), @@ -955,7 +955,7 @@ public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition private void modifyTopLevelColumn(ExternalTable dorisTable, ColumnPath columnPath, Column column, ColumnPosition position, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); validateRowLineageColumnMutation(icebergTable, columnPath.getTopLevelName(), "modify"); NestedField currentCol = icebergTable.schema().asStruct() .caseInsensitiveField(columnPath.getTopLevelName()); @@ -1024,7 +1024,7 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column return; } - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); ResolvedColumnPath resolvedPath = resolveColumnPath(icebergTable.schema(), columnPath, "modify"); NestedField currentCol = resolvedPath.getField(); validateCollectionPseudoFieldComment( @@ -1075,7 +1075,7 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column @Override public void modifyColumnComment(ExternalTable dorisTable, ColumnPath columnPath, String comment, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); if (!columnPath.isNested()) { validateRowLineageColumnMutation(icebergTable, columnPath.getTopLevelName(), "modify comment for"); } @@ -1642,7 +1642,7 @@ public void reorderColumns(ExternalTable dorisTable, List newOrder, long if (newOrder == null || newOrder.isEmpty()) { throw new UserException("Reorder column failed, new order is empty."); } - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); List canonicalOrder = new ArrayList<>(newOrder.size()); Set canonicalNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); for (String columnName : newOrder) { @@ -1709,7 +1709,7 @@ private Term getTransform(String transformName, String columnName, Integer trans */ public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldClause clause, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); String transformName = clause.getTransformName(); @@ -1738,7 +1738,7 @@ public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldClause */ public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldClause clause, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); if (clause.getPartitionFieldName() != null) { @@ -1765,7 +1765,7 @@ public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldClaus */ public void replacePartitionField(ExternalTable dorisTable, ReplacePartitionFieldClause clause, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); // remove old partition field diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartition.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartition.java index cccc6244a0d0cc..96ed0a0dcc1d8a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartition.java @@ -17,6 +17,8 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + import java.util.List; public class IcebergPartition { @@ -29,10 +31,19 @@ public class IcebergPartition { private final long lastUpdateTime; private final long lastSnapshotId; private final List transforms; + private final long retainedPayloadBytes; public IcebergPartition(String partitionName, int specId, long recordCount, long fileSizeInBytes, long fileCount, long lastUpdateTime, long lastSnapshotId, List partitionValues, List transforms) { + this(partitionName, specId, recordCount, fileSizeInBytes, fileCount, lastUpdateTime, + lastSnapshotId, partitionValues, transforms, + estimateRetainedPayloadBytes(partitionName, partitionValues, transforms)); + } + + public IcebergPartition(String partitionName, int specId, long recordCount, long fileSizeInBytes, long fileCount, + long lastUpdateTime, long lastSnapshotId, List partitionValues, + List transforms, long retainedPayloadBytes) { this.partitionName = partitionName; this.specId = specId; this.recordCount = recordCount; @@ -42,6 +53,7 @@ public IcebergPartition(String partitionName, int specId, long recordCount, long this.lastSnapshotId = lastSnapshotId; this.partitionValues = partitionValues; this.transforms = transforms; + this.retainedPayloadBytes = retainedPayloadBytes; } public String getPartitionName() { @@ -79,4 +91,26 @@ public List getPartitionValues() { public List getTransforms() { return transforms; } + + public long getRetainedPayloadBytes() { + return retainedPayloadBytes; + } + + private static long estimateRetainedPayloadBytes( + String partitionName, List partitionValues, List transforms) { + long bytes = MetaCacheWeightUtils.estimatedStringBytes(partitionName); + bytes = addStrings(bytes, partitionValues); + return addStrings(bytes, transforms); + } + + private static long addStrings(long bytes, List values) { + if (values == null) { + return bytes; + } + for (String value : values) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(value)); + } + return bytes; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java index de36f0855ddd14..5c43cc56f8bbf5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java @@ -18,9 +18,9 @@ package org.apache.doris.datasource.iceberg; import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; -import com.google.common.collect.Maps; - +import java.util.Collections; import java.util.Map; import java.util.Set; @@ -28,21 +28,32 @@ public class IcebergPartitionInfo { private final Map nameToPartitionItem; private final Map nameToIcebergPartition; private final Map> nameToIcebergPartitionNames; + private final long retainedPayloadBytes; private static final IcebergPartitionInfo EMPTY = new IcebergPartitionInfo(); private IcebergPartitionInfo() { - this.nameToPartitionItem = Maps.newHashMap(); - this.nameToIcebergPartition = Maps.newHashMap(); - this.nameToIcebergPartitionNames = Maps.newHashMap(); + this.nameToPartitionItem = Collections.emptyMap(); + this.nameToIcebergPartition = Collections.emptyMap(); + this.nameToIcebergPartitionNames = Collections.emptyMap(); + this.retainedPayloadBytes = 0L; } public IcebergPartitionInfo(Map nameToPartitionItem, Map nameToIcebergPartition, Map> nameToIcebergPartitionNames) { + this(nameToPartitionItem, nameToIcebergPartition, nameToIcebergPartitionNames, + retainedPayloadBytes(nameToIcebergPartition)); + } + + public IcebergPartitionInfo(Map nameToPartitionItem, + Map nameToIcebergPartition, + Map> nameToIcebergPartitionNames, + long retainedPayloadBytes) { this.nameToPartitionItem = nameToPartitionItem; this.nameToIcebergPartition = nameToIcebergPartition; this.nameToIcebergPartitionNames = nameToIcebergPartitionNames; + this.retainedPayloadBytes = retainedPayloadBytes; } static IcebergPartitionInfo empty() { @@ -57,6 +68,28 @@ public Map getNameToIcebergPartition() { return nameToIcebergPartition; } + Map> getNameToIcebergPartitionNames() { + return nameToIcebergPartitionNames; + } + + public long getRetainedPayloadBytes() { + return retainedPayloadBytes; + } + + private static long retainedPayloadBytes(Map partitions) { + if (partitions == null) { + return 0L; + } + long bytes = 0L; + for (IcebergPartition partition : partitions.values()) { + if (partition != null) { + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, partition.getRetainedPayloadBytes()); + } + } + return bytes; + } + public long getLatestSnapshotId(String partitionName) { Set icebergPartitionNames = nameToIcebergPartitionNames.get(partitionName); if (icebergPartitionNames == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index 30cf64fcfc6bfa..406d4ee11ab003 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -17,17 +17,33 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import com.google.common.collect.ImmutableList; import org.apache.iceberg.BaseTable; +import org.apache.iceberg.DataFile; +import org.apache.iceberg.DeleteFile; import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.HistoryEntry; +import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotParser; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.SortOrder; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.TableOperations; import org.apache.iceberg.encryption.EncryptionManager; import org.apache.iceberg.exceptions.CommitFailedException; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.LocationProvider; -import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -40,36 +56,65 @@ public class IcebergSnapshotCacheValue { private final IcebergPartitionInfo partitionInfo; private final IcebergSnapshot snapshot; private final Optional>> nameMapping; - private final Optional icebergTable; + private Optional
    icebergTable; + private final long retainedNameMappingPayloadBytes; + private String retainedCurrentSnapshotJson; + private boolean queryIsolationPrepared; + private long retainedTablePayloadBytes; + private MetaCacheSizeEstimate sizeEstimate; public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot) { - this(partitionInfo, snapshot, Optional.empty(), Optional.empty()); + this(partitionInfo, snapshot, Optional.empty(), Optional.empty(), null, false); } public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, Optional>> nameMapping) { - this(partitionInfo, snapshot, nameMapping, Optional.empty()); + this(partitionInfo, snapshot, nameMapping, Optional.empty(), null, false); } public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, Optional>> nameMapping, Table icebergTable) { - this(partitionInfo, snapshot, nameMapping, Optional.of(icebergTable)); + this(partitionInfo, snapshot, nameMapping, Optional.of(icebergTable), null, false); + } + + IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, + Optional>> nameMapping, Table retainedTable, + String retainedCurrentSnapshotJson) { + this(partitionInfo, snapshot, nameMapping, Optional.of(retainedTable), + retainedCurrentSnapshotJson, true); } private IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot, - Optional>> nameMapping, Optional
    icebergTable) { + Optional>> nameMapping, Optional
    icebergTable, + String retainedCurrentSnapshotJson, boolean isolateForQueries) { this.partitionInfo = partitionInfo; this.snapshot = snapshot; // A cached BaseTable shares live TableOperations; retain a metadata-only generation so a // later commit through that same Table cannot move an already bound statement forward. this.icebergTable = icebergTable.map(IcebergSnapshotCacheValue::retainTableGeneration); - this.nameMapping = nameMapping.map(mapping -> { + this.retainedCurrentSnapshotJson = retainedCurrentSnapshotJson; + if (isolateForQueries) { + this.icebergTable = this.icebergTable.map( + IcebergSnapshotCacheValue::retainNonGrowingGeneration); + this.queryIsolationPrepared = true; + } + if (nameMapping.isPresent()) { Map> copy = new HashMap<>(); - // Preserve the immutable snapshot contract while remaining compatible with branch-4.1's Java target. - mapping.forEach((id, names) -> copy.put(id, - Collections.unmodifiableList(new ArrayList<>(names)))); - return Collections.unmodifiableMap(copy); - }); + long payloadBytes = 0L; + for (Map.Entry> entry : nameMapping.get().entrySet()) { + List names = ImmutableList.copyOf(entry.getValue()); + copy.put(entry.getKey(), names); + for (String name : names) { + payloadBytes = MetaCacheWeightUtils.saturatedAdd(payloadBytes, + MetaCacheWeightUtils.estimatedStringBytes(name)); + } + } + this.nameMapping = Optional.of(Collections.unmodifiableMap(copy)); + this.retainedNameMappingPayloadBytes = payloadBytes; + } else { + this.nameMapping = Optional.empty(); + this.retainedNameMappingPayloadBytes = 0L; + } } public IcebergPartitionInfo getPartitionInfo() { @@ -85,26 +130,153 @@ public Optional>> getNameMapping() { } public Optional
    getIcebergTable() { + return queryIsolationPrepared + ? icebergTable.map(table -> createQueryScopedTable( + table, retainedCurrentSnapshotJson)) + : icebergTable; + } + + MetaCacheSizeEstimate prepareForCachePublication(IcebergSnapshotEntryKey key) { + if (sizeEstimate == null) { + if (retainedCurrentSnapshotJson == null) { + retainedCurrentSnapshotJson = icebergTable + .map(IcebergSnapshotCacheValue::retainCurrentSnapshotJson).orElse(null); + } + retainedTablePayloadBytes = icebergTable + .map(IcebergCacheSizeEstimator::retainedTablePayloadBytes).orElse(0L); + sizeEstimate = MetaCacheSizeEstimator.estimateSafely("iceberg_snapshot_preparation_failed", + () -> IcebergCacheSizeEstimator.estimateSnapshotEntry(key, this)); + if (sizeEstimate.isComplete()) { + icebergTable = icebergTable.map( + IcebergSnapshotCacheValue::retainNonGrowingGeneration); + queryIsolationPrepared = true; + } + } + return sizeEstimate; + } + + public MetaCacheSizeEstimate getSizeEstimate() { + return sizeEstimate == null + ? MetaCacheSizeEstimate.incomplete("not_prepared") : sizeEstimate; + } + + long getRetainedNameMappingPayloadBytes() { + return retainedNameMappingPayloadBytes; + } + + long getRetainedTablePayloadBytes() { + return retainedTablePayloadBytes; + } + + long getRetainedCurrentSnapshotPayloadBytes() { + return retainedSnapshotJsonBytes(retainedCurrentSnapshotJson); + } + + Optional
    getRetainedIcebergTable() { return icebergTable; } static Table retainTableGeneration(Table table) { + return retainTableGeneration(table, null); + } + + static Table retainTableGeneration(Table table, ExecutionAuthenticator authenticator) { if (!(table instanceof HasTableOperations) || isFrozenGeneration(table)) { return table; } TableOperations operations = ((HasTableOperations) table).operations(); // Capture current() exactly once so every projection derived from the returned table sees // one metadata generation even when the shared catalog handle refreshes concurrently. - TableOperations frozenOperations = new FrozenTableOperations(operations, operations.current()); + TableOperations frozenOperations = new FrozenTableOperations( + operations, operations.current(), authenticator); return tableWithOperations(table, frozenOperations); } + static Table retainNonGrowingGeneration(Table table) { + if (!isFrozenGeneration(table)) { + return table; + } + TableOperations retainedOperations = ((HasTableOperations) table).operations(); + TableMetadata source = retainedOperations.current(); + if (source.schemas().isEmpty() || source.specs().isEmpty() + || source.sortOrders().isEmpty()) { + return table; + } + TableMetadata.Builder builder = TableMetadata.buildFromEmpty(source.formatVersion()); + if (source.uuid() != null) { + builder.assignUUID(source.uuid()); + } + for (Schema schema : source.schemas()) { + builder.addSchema(schema); + } + builder.setCurrentSchema(source.currentSchemaId()); + for (PartitionSpec spec : source.specs()) { + builder.addPartitionSpec(spec); + } + builder.setDefaultPartitionSpec(source.defaultSpecId()); + for (SortOrder sortOrder : source.sortOrders()) { + builder.addSortOrder(sortOrder); + } + builder.setDefaultSortOrder(source.defaultSortOrderId()); + builder.setLocation(source.location()); + builder.setProperties(source.properties()); + if (source.currentSnapshot() != null) { + builder.setBranchSnapshot( + new NonGrowingSnapshot(source.currentSnapshot()), SnapshotRef.MAIN_BRANCH); + } + TableMetadata retainedMetadata = builder.discardChanges() + .withMetadataLocation(source.metadataFileLocation()).build(); + return tableWithOperations(table, new FrozenTableOperations( + retainedOperations, retainedMetadata)); + } + + static String retainCurrentSnapshotJson(Table table) { + if (!(table instanceof HasTableOperations)) { + return null; + } + TableMetadata metadata = ((HasTableOperations) table).operations().current(); + Snapshot snapshot = metadata == null ? null : metadata.currentSnapshot(); + return snapshot == null ? null : SnapshotParser.toJson(snapshot, false); + } + + static long retainedSnapshotJsonBytes(String snapshotJson) { + return MetaCacheWeightUtils.estimatedStringBytes(snapshotJson); + } + + static Table createQueryScopedTable(Table retainedTable, String currentSnapshotJson) { + if (!isFrozenGeneration(retainedTable)) { + return retainedTable; + } + TableOperations retainedOperations = ((HasTableOperations) retainedTable).operations(); + if (retainedTable instanceof BaseTable) { + return new QueryScopedTable(retainedOperations, retainedTable.name(), + ((BaseTable) retainedTable).reporter(), currentSnapshotJson); + } + return new QueryScopedTable(retainedOperations, retainedTable.name(), null, + currentSnapshotJson); + } + + static void loadQueryMetadataForStatement(Table table) { + if (table instanceof QueryScopedTable) { + ((QueryScopedTable) table).queryMetadata(); + } + } + static boolean isFrozenGeneration(Table table) { return table instanceof HasTableOperations && ((HasTableOperations) table).operations() instanceof FrozenTableOperations; } - static Table createWritableTable(Table retainedTable, Table liveTable) { + static TableOperations unwrapRetainedTableOperations(TableOperations operations) { + TableOperations current = Objects.requireNonNull(operations, "operations can not be null"); + while (current instanceof RetainedTableOperations) { + current = ((RetainedTableOperations) current).delegate; + } + return current; + } + + static Table createWritableTable( + Table retainedTable, Table liveTable, boolean reloadRetainedMetadata) { if (!isFrozenGeneration(retainedTable)) { return retainedTable; } @@ -113,12 +285,46 @@ static Table createWritableTable(Table retainedTable, Table liveTable) { throw new IllegalArgumentException( "Iceberg commit table must provide writable table operations"); } - TableMetadata retainedMetadata = ((HasTableOperations) retainedTable).operations().current(); - TableOperations liveOperations = ((HasTableOperations) liveTable).operations(); + TableOperations retainedOperations = ((HasTableOperations) retainedTable).operations(); + TableMetadata retainedMetadata = reloadRetainedMetadata + ? loadQueryMetadata(retainedOperations) : retainedOperations.current(); + TableOperations liveOperations = unwrapRetainedTableOperations( + ((HasTableOperations) liveTable).operations()); return tableWithOperations(retainedTable, new WritableTableOperations(liveOperations, retainedMetadata)); } + static Table createWritableTable(Table retainedTable, Table liveTable) { + return createWritableTable(retainedTable, liveTable, + isNonGrowingGeneration(retainedTable)); + } + + private static boolean isNonGrowingGeneration(Table table) { + return isFrozenGeneration(table) + && ((FrozenTableOperations) ((HasTableOperations) table).operations()).nonGrowing; + } + + private static TableMetadata loadQueryMetadata(TableOperations retainedOperations) { + TableMetadata retainedMetadata = retainedOperations.current(); + TableOperations serviceOperations = unwrapRetainedTableOperations(retainedOperations); + String metadataLocation = retainedMetadata.metadataFileLocation(); + if (metadataLocation != null && !metadataLocation.isEmpty() && serviceOperations.io() != null) { + ExecutionAuthenticator authenticator = retainedOperations instanceof FrozenTableOperations + ? ((FrozenTableOperations) retainedOperations).authenticator : null; + try { + return authenticator == null + ? TableMetadataParser.read(serviceOperations.io(), metadataLocation) + : authenticator.execute( + () -> TableMetadataParser.read(serviceOperations.io(), metadataLocation)); + } catch (Exception e) { + throw new StaleMetadataException( + "Iceberg metadata generation is no longer readable: " + metadataLocation, e); + } + } + throw new IllegalStateException( + "Iceberg query-local metadata requires a stable metadata location and FileIO"); + } + private static Table tableWithOperations(Table table, TableOperations operations) { if (table instanceof BaseTable) { return new BaseTable(operations, table.name(), ((BaseTable) table).reporter()); @@ -166,15 +372,75 @@ public LocationProvider locationProvider() { } } - private static class FrozenTableOperations extends RetainedTableOperations { - private FrozenTableOperations(TableOperations delegate, TableMetadata metadata) { - super(delegate, metadata); + private static class FrozenTableOperations implements TableOperations { + private final TableMetadata metadata; + private final FileIO fileIO; + private final EncryptionManager encryptionManager; + private final LocationProvider locationProvider; + private final ExecutionAuthenticator authenticator; + private final boolean nonGrowing; + + private FrozenTableOperations(TableOperations source, TableMetadata metadata) { + this(source, metadata, source instanceof FrozenTableOperations + ? ((FrozenTableOperations) source).authenticator : null, true); + } + + private FrozenTableOperations(TableOperations source, TableMetadata metadata, + ExecutionAuthenticator authenticator) { + this(source, metadata, authenticator, false); + } + + private FrozenTableOperations(TableOperations source, TableMetadata metadata, + ExecutionAuthenticator authenticator, boolean nonGrowing) { + this.metadata = metadata; + this.fileIO = source.io(); + this.encryptionManager = source.encryption(); + this.locationProvider = source.locationProvider(); + this.authenticator = authenticator; + this.nonGrowing = nonGrowing; + } + + @Override + public TableMetadata current() { + return metadata; + } + + @Override + public TableMetadata refresh() { + return metadata; } @Override public void commit(TableMetadata base, TableMetadata newMetadata) { throw new UnsupportedOperationException("Frozen Iceberg table generation is read-only"); } + + @Override + public FileIO io() { + return fileIO; + } + + @Override + public EncryptionManager encryption() { + return encryptionManager; + } + + @Override + public String metadataFileLocation(String fileName) { + String metadataLocation = metadata.metadataFileLocation(); + if (metadataLocation == null) { + throw new UnsupportedOperationException( + "Frozen Iceberg table has no metadata directory"); + } + int separator = metadataLocation.lastIndexOf('/'); + return separator < 0 ? fileName + : metadataLocation.substring(0, separator + 1) + fileName; + } + + @Override + public LocationProvider locationProvider() { + return locationProvider; + } } private static class WritableTableOperations extends RetainedTableOperations { @@ -208,8 +474,9 @@ public TableMetadata refresh() { @Override public void commit(TableMetadata base, TableMetadata newMetadata) { - delegate.commit(base, newMetadata); - currentMetadata = newMetadata; + TableMetadata delegateBase = prepareDelegateCommit(delegate, base, currentMetadata); + delegate.commit(delegateBase, newMetadata); + currentMetadata = delegate.current(); } private boolean isWriterCompatible(TableMetadata refreshedMetadata) { @@ -221,4 +488,221 @@ private boolean isWriterCompatible(TableMetadata refreshedMetadata) { && Objects.equals(retainedMetadata.properties(), refreshedMetadata.properties()); } } + + /** A per-caller view whose Iceberg lazy snapshot state is never written into the cache value. */ + private static final class QueryScopedTable extends BaseTable { + private final TableOperations retainedOperations; + private final Snapshot currentSnapshot; + private TableMetadata queryMetadata; + + private QueryScopedTable(TableOperations retainedOperations, String name, + org.apache.iceberg.metrics.MetricsReporter reporter, String currentSnapshotJson) { + super(retainedOperations, name, reporter == null + ? org.apache.iceberg.metrics.LoggingMetricsReporter.instance() : reporter); + this.retainedOperations = retainedOperations; + this.currentSnapshot = currentSnapshotJson == null + ? null : SnapshotParser.fromJson(currentSnapshotJson); + } + + @Override + public Snapshot currentSnapshot() { + return currentSnapshot; + } + + @Override + public Snapshot snapshot(long snapshotId) { + if (currentSnapshot != null && currentSnapshot.snapshotId() == snapshotId) { + return currentSnapshot; + } + return queryMetadata().snapshot(snapshotId); + } + + @Override + public Iterable snapshots() { + return queryMetadata().snapshots(); + } + + @Override + public List history() { + return queryMetadata().snapshotLog(); + } + + @Override + public Map refs() { + return queryMetadata().refs(); + } + + @Override + public List statisticsFiles() { + return queryMetadata().statisticsFiles(); + } + + @Override + public List partitionStatisticsFiles() { + return queryMetadata().partitionStatisticsFiles(); + } + + private synchronized TableMetadata queryMetadata() { + if (queryMetadata == null) { + queryMetadata = loadQueryMetadata(retainedOperations); + } + return queryMetadata; + } + } + + static final class StaleMetadataException extends RuntimeException { + private StaleMetadataException(String message, Throwable cause) { + super(message, cause); + } + } + + /** Scalar-only snapshot retained by the cache-owned metadata generation. */ + private static final class NonGrowingSnapshot implements Snapshot { + private final long sequenceNumber; + private final long snapshotId; + private final Long parentId; + private final long timestampMillis; + private final Integer schemaId; + private final Long firstRowId; + private final Long addedRows; + + private NonGrowingSnapshot(Snapshot snapshot) { + this.sequenceNumber = snapshot.sequenceNumber(); + this.snapshotId = snapshot.snapshotId(); + this.parentId = snapshot.parentId(); + this.timestampMillis = snapshot.timestampMillis(); + this.schemaId = snapshot.schemaId(); + this.firstRowId = snapshot.firstRowId(); + this.addedRows = snapshot.addedRows(); + } + + @Override + public long sequenceNumber() { + return sequenceNumber; + } + + @Override + public long snapshotId() { + return snapshotId; + } + + @Override + public Long parentId() { + return parentId; + } + + @Override + public long timestampMillis() { + return timestampMillis; + } + + @Override + public List allManifests(FileIO fileIO) { + throw queryScopedSnapshotRequired(); + } + + @Override + public List dataManifests(FileIO fileIO) { + throw queryScopedSnapshotRequired(); + } + + @Override + public List deleteManifests(FileIO fileIO) { + throw queryScopedSnapshotRequired(); + } + + @Override + public String operation() { + return null; + } + + @Override + public Map summary() { + return Collections.emptyMap(); + } + + @Override + public Iterable addedDataFiles(FileIO fileIO) { + throw queryScopedSnapshotRequired(); + } + + @Override + public Iterable removedDataFiles(FileIO fileIO) { + throw queryScopedSnapshotRequired(); + } + + @Override + public Iterable addedDeleteFiles(FileIO fileIO) { + throw queryScopedSnapshotRequired(); + } + + @Override + public Iterable removedDeleteFiles(FileIO fileIO) { + throw queryScopedSnapshotRequired(); + } + + @Override + public String manifestListLocation() { + return null; + } + + @Override + public Integer schemaId() { + return schemaId; + } + + @Override + public Long firstRowId() { + return firstRowId; + } + + @Override + public Long addedRows() { + return addedRows; + } + + @Override + public String keyId() { + return null; + } + + private UnsupportedOperationException queryScopedSnapshotRequired() { + return new UnsupportedOperationException( + "Cache-owned Iceberg snapshots cannot materialize manifests or files"); + } + } + + private static TableMetadata prepareDelegateCommit(TableOperations delegate, + TableMetadata base, TableMetadata wrapperCurrent) { + if (base != wrapperCurrent) { + throw new CommitFailedException("Cannot commit from a stale Iceberg table view"); + } + TableMetadata delegateCurrent = delegate.current(); + if (!isSameGeneration(base, delegateCurrent)) { + throw new CommitFailedException("Cannot commit from a stale Iceberg metadata generation"); + } + return delegateCurrent; + } + + private static boolean isSameGeneration(TableMetadata retained, TableMetadata live) { + if (retained == live) { + return true; + } + if (retained == null || live == null) { + return false; + } + if (!Objects.equals(retained.uuid(), live.uuid())) { + return false; + } + if (retained.metadataFileLocation() != null || live.metadataFileLocation() != null) { + return Objects.equals(retained.metadataFileLocation(), live.metadataFileLocation()); + } + return retained.lastUpdatedMillis() == live.lastUpdatedMillis() + && retained.lastSequenceNumber() == live.lastSequenceNumber() + && retained.currentSchemaId() == live.currentSchemaId() + && retained.defaultSpecId() == live.defaultSpecId() + && retained.defaultSortOrderId() == live.defaultSortOrderId() + && Objects.equals(retained.location(), live.location()) + && Objects.equals(retained.properties(), live.properties()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotEntryKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotEntryKey.java new file mode 100644 index 00000000000000..782a51d5f135db --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotEntryKey.java @@ -0,0 +1,115 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.iceberg; + +import org.apache.doris.datasource.NameMapping; + +import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; + +import java.util.Objects; +import java.util.Optional; + +/** Stable identity for an Iceberg snapshot projection built from one frozen metadata generation. */ +public final class IcebergSnapshotEntryKey { + private final NameMapping nameMapping; + private final String tableUuid; + private final String metadataFileLocation; + private final long snapshotId; + private final int schemaId; + private final int defaultSpecId; + + private IcebergSnapshotEntryKey(NameMapping nameMapping, String tableUuid, String metadataFileLocation, + long snapshotId, int schemaId, int defaultSpecId) { + this.nameMapping = Objects.requireNonNull(nameMapping, "nameMapping can not be null"); + this.tableUuid = Objects.requireNonNull(tableUuid, "tableUuid can not be null"); + this.metadataFileLocation = Objects.requireNonNull( + metadataFileLocation, "metadataFileLocation can not be null"); + this.snapshotId = snapshotId; + this.schemaId = schemaId; + this.defaultSpecId = defaultSpecId; + } + + /** + * Build a key from the same retained table generation that will be used by the value loader. + * Tables without a stable metadata location intentionally bypass the snapshot cache. + */ + public static Optional tryCreate(NameMapping nameMapping, Table retainedTable) { + if (!(retainedTable instanceof HasTableOperations)) { + return Optional.empty(); + } + TableMetadata metadata = ((HasTableOperations) retainedTable).operations().current(); + if (metadata == null || metadata.uuid() == null || metadata.uuid().isEmpty() + || metadata.metadataFileLocation() == null + || metadata.metadataFileLocation().isEmpty()) { + return Optional.empty(); + } + Snapshot snapshot = metadata.currentSnapshot(); + long snapshotId = snapshot == null ? IcebergUtils.UNKNOWN_SNAPSHOT_ID : snapshot.snapshotId(); + return Optional.of(new IcebergSnapshotEntryKey(nameMapping, metadata.uuid(), metadata.metadataFileLocation(), + snapshotId, metadata.currentSchemaId(), metadata.defaultSpecId())); + } + + public NameMapping getNameMapping() { + return nameMapping; + } + + public String getMetadataFileLocation() { + return metadataFileLocation; + } + + public String getTableUuid() { + return tableUuid; + } + + public long getSnapshotId() { + return snapshotId; + } + + public int getSchemaId() { + return schemaId; + } + + public int getDefaultSpecId() { + return defaultSpecId; + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (!(object instanceof IcebergSnapshotEntryKey)) { + return false; + } + IcebergSnapshotEntryKey that = (IcebergSnapshotEntryKey) object; + return snapshotId == that.snapshotId + && schemaId == that.schemaId + && defaultSpecId == that.defaultSpecId + && nameMapping.equals(that.nameMapping) + && tableUuid.equals(that.tableUuid) + && metadataFileLocation.equals(that.metadataFileLocation); + } + + @Override + public int hashCode() { + return Objects.hash(nameMapping, tableUuid, metadataFileLocation, snapshotId, schemaId, defaultSpecId); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java index cdef77346ade27..6403e6bfdbafa8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java @@ -17,25 +17,90 @@ package org.apache.doris.datasource.iceberg; -import com.google.common.base.Suppliers; -import org.apache.iceberg.Table; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; -import java.util.function.Supplier; +import org.apache.iceberg.Table; public class IcebergTableCacheValue { - private final Table icebergTable; - private final Supplier latestSnapshotCacheValue; + private Table icebergTable; + private String retainedCurrentSnapshotJson; + private volatile boolean queryIsolationPrepared; + private long retainedTablePayloadBytes; + private MetaCacheSizeEstimate sizeEstimate; - public IcebergTableCacheValue(Table icebergTable, Supplier latestSnapshotCacheValue) { - this.icebergTable = icebergTable; - this.latestSnapshotCacheValue = Suppliers.memoize(latestSnapshotCacheValue::get); + public IcebergTableCacheValue(Table icebergTable) { + this.icebergTable = IcebergSnapshotCacheValue.retainTableGeneration(icebergTable); + } + + IcebergTableCacheValue(Table icebergTable, ExecutionAuthenticator authenticator) { + this.icebergTable = IcebergSnapshotCacheValue.retainTableGeneration( + icebergTable, authenticator); } public Table getIcebergTable() { + return queryIsolationPrepared + ? IcebergSnapshotCacheValue.createQueryScopedTable( + icebergTable, retainedCurrentSnapshotJson) + : icebergTable; + } + + public Table getWritableIcebergTable(Table liveTable) { + return IcebergSnapshotCacheValue.createWritableTable( + icebergTable, liveTable, queryIsolationPrepared); + } + + MetaCacheSizeEstimate prepareForCachePublication(NameMapping key) { + if (sizeEstimate == null) { + retainedCurrentSnapshotJson = + IcebergSnapshotCacheValue.retainCurrentSnapshotJson(icebergTable); + retainedTablePayloadBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes(icebergTable); + sizeEstimate = MetaCacheSizeEstimator.estimateSafely("iceberg_table_preparation_failed", + () -> IcebergCacheSizeEstimator.estimateTableEntry(key, this)); + if (sizeEstimate.isComplete()) { + icebergTable = IcebergSnapshotCacheValue.retainNonGrowingGeneration(icebergTable); + queryIsolationPrepared = true; + } + } + return sizeEstimate; + } + + public MetaCacheSizeEstimate getSizeEstimate() { + return sizeEstimate == null + ? MetaCacheSizeEstimate.incomplete("not_prepared") : sizeEstimate; + } + + Table getRetainedIcebergTable() { return icebergTable; } - public IcebergSnapshotCacheValue getLatestSnapshotCacheValue() { - return latestSnapshotCacheValue.get(); + synchronized Table newQueryScopedTable() { + if (!queryIsolationPrepared) { + retainedCurrentSnapshotJson = + IcebergSnapshotCacheValue.retainCurrentSnapshotJson(icebergTable); + icebergTable = IcebergSnapshotCacheValue.retainNonGrowingGeneration(icebergTable); + queryIsolationPrepared = true; + } + return IcebergSnapshotCacheValue.createQueryScopedTable( + icebergTable, retainedCurrentSnapshotJson); + } + + String getRetainedCurrentSnapshotJson() { + return retainedCurrentSnapshotJson; + } + + boolean isQueryIsolationPrepared() { + return queryIsolationPrepared; + } + + long getRetainedTablePayloadBytes() { + return retainedTablePayloadBytes; + } + + long getRetainedCurrentSnapshotPayloadBytes() { + return IcebergSnapshotCacheValue.retainedSnapshotJsonBytes( + retainedCurrentSnapshotJson); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java index 71935cbd88157b..138e4fc4b1289c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java @@ -299,7 +299,7 @@ private Table createTransactionTable(ExternalTable dorisTable, Table retainedTab // Reads stay on the retained generation; commit refreshes may follow data-only snapshots, // while writer-contract changes still invalidate files produced for the retained metadata. return IcebergSnapshotCacheValue.createWritableTable( - retainedTable, IcebergUtils.getIcebergTable(dorisTable)); + retainedTable, IcebergUtils.getWritableIcebergTable(dorisTable)); } /** Begin UPDATE/MERGE against the metadata generation retained by the merge sink. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 0b375c70d6791e..1c370222d71a83 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -56,6 +56,7 @@ import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.iceberg.source.IcebergTableQueryInfo; import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.doris.datasource.mvcc.MvccSnapshot; import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.datasource.property.metastore.HMSBaseProperties; @@ -1057,6 +1058,10 @@ public static Table getIcebergTable(ExternalTable dorisTable) { return icebergExternalMetaCache(dorisTable).getIcebergTable(dorisTable); } + public static Table getWritableIcebergTable(ExternalTable dorisTable) { + return icebergExternalMetaCache(dorisTable).getWritableIcebergTable(dorisTable); + } + private static IcebergExternalMetaCache icebergExternalMetaCache(ExternalCatalog catalog) { Preconditions.checkNotNull(catalog, "catalog can not be null"); return Env.getCurrentEnv().getExtMetaCacheMgr().iceberg(catalog.getId()); @@ -1751,10 +1756,13 @@ public static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, T } Map nameToPartition = Maps.newHashMap(); Map nameToPartitionItem = Maps.newHashMap(); + long retainedPayloadBytes = 0L; List partitionColumns = IcebergUtils.getSchemaCacheValue(dorisTable, schemaId).getPartitionColumns(); for (IcebergPartition partition : icebergPartitions) { nameToPartition.put(partition.getPartitionName(), partition); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, partition.getRetainedPayloadBytes()); String transform = table.specs().get(partition.getSpecId()).fields().get(0).transform().toString(); Range partitionRange = getPartitionRange( partition.getPartitionValues().get(0), transform, partitionColumns); @@ -1762,7 +1770,8 @@ public static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, T nameToPartitionItem.put(partition.getPartitionName(), item); } Map> partitionNameMap = mergeOverlapPartitions(nameToPartitionItem); - return new IcebergPartitionInfo(nameToPartitionItem, nameToPartition, partitionNameMap); + return new IcebergPartitionInfo( + nameToPartitionItem, nameToPartition, partitionNameMap, retainedPayloadBytes); } private static List loadIcebergPartition(Table table, long snapshotId) { @@ -1802,6 +1811,7 @@ private static IcebergPartition generateIcebergPartition(Table table, StructLike StringBuilder sb = new StringBuilder(); List partitionValues = Lists.newArrayList(); List transforms = Lists.newArrayList(); + long retainedPayloadBytes = 0L; for (int i = 0; i < partitionSpec.fields().size(); ++i) { PartitionField partitionField = partitionSpec.fields().get(i); Class fieldClass = partitionSpec.javaClasses()[i]; @@ -1817,12 +1827,19 @@ private static IcebergPartition generateIcebergPartition(Table table, StructLike sb.append(fieldValue); sb.append("/"); partitionValues.add(fieldValue); - transforms.add(partitionField.transform().toString()); + String transform = partitionField.transform().toString(); + transforms.add(transform); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd(retainedPayloadBytes, + MetaCacheWeightUtils.estimatedStringBytes(fieldValue)); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd(retainedPayloadBytes, + MetaCacheWeightUtils.estimatedStringBytes(transform)); } if (sb.length() > 0) { sb.delete(sb.length() - 1, sb.length()); } String partitionName = sb.toString(); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd(retainedPayloadBytes, + MetaCacheWeightUtils.estimatedStringBytes(partitionName)); long recordCount = row.get(2, Long.class); long fileCount = row.get(3, Integer.class); long fileSizeInBytes = row.get(4, Long.class); @@ -1841,7 +1858,7 @@ private static IcebergPartition generateIcebergPartition(Table table, StructLike lastUpdateSnapShotId = UNKNOWN_SNAPSHOT_ID; } return new IcebergPartition(partitionName, specId, recordCount, fileSizeInBytes, fileCount, - lastUpdateTime, lastUpdateSnapShotId, partitionValues, transforms); + lastUpdateTime, lastUpdateSnapShotId, partitionValues, transforms, retainedPayloadBytes); } @VisibleForTesting @@ -2000,7 +2017,8 @@ public static IcebergSnapshotCacheValue getSnapshotCacheValue( Optional scanParams) { if (tableSnapshot.isPresent() || IcebergUtils.isIcebergBranchOrTag(scanParams)) { // If a snapshot is specified, use the specified snapshot and the corresponding schema (not latest). - Table icebergTable = IcebergSnapshotCacheValue.retainTableGeneration(getIcebergTable(dorisTable)); + IcebergExternalMetaCache metaCache = icebergExternalMetaCache(dorisTable); + Table icebergTable = metaCache.getQueryScopedIcebergTable(dorisTable); IcebergTableQueryInfo info; try { info = getQuerySpecSnapshot(icebergTable, tableSnapshot, scanParams); @@ -2010,8 +2028,7 @@ public static IcebergSnapshotCacheValue getSnapshotCacheValue( return new IcebergSnapshotCacheValue( IcebergPartitionInfo.empty(), new IcebergSnapshot(info.getSnapshotId(), info.getSchemaId()), - getNameMapping(icebergTable), - icebergTable); + getNameMapping(icebergTable), icebergTable); } return getLatestSnapshotCacheValue(dorisTable); } @@ -2027,11 +2044,12 @@ public static List getIcebergSchema(ExternalTable dorisTable, Optional getIcebergPartitionColumns(Optional snapshot, ExternalTable dorisTable) { IcebergSnapshotCacheValue snapshotValue = getSnapshotCacheValue(snapshot, dorisTable); - if (snapshotValue.getIcebergTable().isPresent()) { + Optional
    snapshotTable = snapshotValue.getIcebergTable(); + if (snapshotTable.isPresent()) { // Schema ID alone cannot identify the partition spec; metadata-only evolution may keep // the same schema and snapshot IDs while changing spec(), so derive both from T0. return buildTableSchemaCacheValue(dorisTable, snapshotValue.getSnapshot().getSchemaId(), - snapshotValue.getIcebergTable().get()).getPartitionColumns(); + snapshotTable.get()).getPartitionColumns(); } return getSchemaCacheValue(dorisTable, snapshotValue).getPartitionColumns(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergCherrypickSnapshotAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergCherrypickSnapshotAction.java index e303f0e9111486..94924514a58a48 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergCherrypickSnapshotAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergCherrypickSnapshotAction.java @@ -70,7 +70,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); Long sourceSnapshotId = namedArguments.getLong(SNAPSHOT_ID); try { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExpireSnapshotsAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExpireSnapshotsAction.java index 0937af8ba4cac4..82a93022354067 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExpireSnapshotsAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergExpireSnapshotsAction.java @@ -149,7 +149,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); // Parse parameters String olderThan = namedArguments.getString(OLDER_THAN); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergFastForwardAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergFastForwardAction.java index a5560db65520e4..cd746a7dbe6959 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergFastForwardAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergFastForwardAction.java @@ -70,7 +70,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); String sourceBranch = namedArguments.getString(BRANCH); String desBranch = namedArguments.getString(TO); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergPublishChangesAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergPublishChangesAction.java index e1bf8cbdad4472..bf3f116d1cba81 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergPublishChangesAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergPublishChangesAction.java @@ -66,7 +66,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); String targetWapId = namedArguments.getString(WAP_ID); // Find the target WAP snapshot diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteManifestsAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteManifestsAction.java index 430e9fe9d5e22d..dce45c2729693b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteManifestsAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRewriteManifestsAction.java @@ -68,7 +68,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { try { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); Snapshot current = icebergTable.currentSnapshot(); if (current == null) { // No current snapshot means the table is empty, no manifests to rewrite diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToSnapshotAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToSnapshotAction.java index 8d6b3842a9dc80..a5609f83439d45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToSnapshotAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToSnapshotAction.java @@ -68,7 +68,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); Long targetSnapshotId = namedArguments.getLong(SNAPSHOT_ID); Snapshot targetSnapshot = icebergTable.snapshot(targetSnapshotId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToTimestampAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToTimestampAction.java index 6957c563512657..de7e2a680791c2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToTimestampAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergRollbackToTimestampAction.java @@ -96,7 +96,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); String timestampStr = namedArguments.getString(TIMESTAMP); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergSetCurrentSnapshotAction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergSetCurrentSnapshotAction.java index 44df40f8f492b9..5b2c5bd220eb7f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergSetCurrentSnapshotAction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/action/IcebergSetCurrentSnapshotAction.java @@ -87,7 +87,7 @@ protected void validateIcebergAction() throws UserException { @Override protected List executeAction(TableIf table) throws UserException { - Table icebergTable = ((IcebergExternalTable) table).getIcebergTable(); + Table icebergTable = ((IcebergExternalTable) table).getWritableIcebergTable(); Snapshot previousSnapshot = icebergTable.currentSnapshot(); Long previousSnapshotId = previousSnapshot != null ? previousSnapshot.snapshotId() : null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java index e98ca6b2fb2808..47f8c8cde3e7df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java @@ -17,30 +17,62 @@ package org.apache.doris.datasource.iceberg.cache; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import com.google.common.collect.ImmutableList; +import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.StructLike; -import java.util.Collections; +import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.List; +import java.util.Map; /** * Cached manifest payload containing parsed files. */ public class ManifestCacheValue { + private static final long AUXILIARY_LIST_ENTRY_BYTES = 32L; + private final List dataFiles; private final List deleteFiles; + private final long dataFileMetricEntryCount; + private final long deleteFileMetricEntryCount; + private final long retainedPayloadBytes; - private ManifestCacheValue(List dataFiles, List deleteFiles) { - this.dataFiles = dataFiles == null ? Collections.emptyList() : dataFiles; - this.deleteFiles = deleteFiles == null ? Collections.emptyList() : deleteFiles; + private ManifestCacheValue(List dataFiles, List deleteFiles, + long dataFileMetricEntryCount, long deleteFileMetricEntryCount, long retainedPayloadBytes) { + this.dataFiles = ImmutableList.copyOf(dataFiles); + this.deleteFiles = ImmutableList.copyOf(deleteFiles); + this.dataFileMetricEntryCount = dataFileMetricEntryCount; + this.deleteFileMetricEntryCount = deleteFileMetricEntryCount; + this.retainedPayloadBytes = retainedPayloadBytes; } public static ManifestCacheValue forDataFiles(List dataFiles) { - return new ManifestCacheValue(dataFiles, Collections.emptyList()); + Builder builder = dataFilesBuilder(); + if (dataFiles != null) { + dataFiles.forEach(builder::addDataFile); + } + return builder.build(); } public static ManifestCacheValue forDeleteFiles(List deleteFiles) { - return new ManifestCacheValue(Collections.emptyList(), deleteFiles); + Builder builder = deleteFilesBuilder(); + if (deleteFiles != null) { + deleteFiles.forEach(builder::addDeleteFile); + } + return builder.build(); + } + + public static Builder dataFilesBuilder() { + return new Builder(true); + } + + public static Builder deleteFilesBuilder() { + return new Builder(false); } public List getDataFiles() { @@ -50,4 +82,134 @@ public List getDataFiles() { public List getDeleteFiles() { return deleteFiles; } + + public long getDataFileMetricEntryCount() { + return dataFileMetricEntryCount; + } + + public long getDeleteFileMetricEntryCount() { + return deleteFileMetricEntryCount; + } + + public long getRetainedPayloadBytes() { + return retainedPayloadBytes; + } + + /** Accumulates retained-size counters in the manifest reader's existing file loop. */ + public static final class Builder { + private final boolean dataContent; + private final List dataFiles = new ArrayList<>(); + private final List deleteFiles = new ArrayList<>(); + private long metricEntryCount; + private long retainedPayloadBytes; + + private Builder(boolean dataContent) { + this.dataContent = dataContent; + } + + public void addDataFile(DataFile file) { + if (!dataContent) { + throw new IllegalStateException("delete manifest builder cannot accept a data file"); + } + dataFiles.add(file); + account(file); + } + + public void addDeleteFile(DeleteFile file) { + if (dataContent) { + throw new IllegalStateException("data manifest builder cannot accept a delete file"); + } + deleteFiles.add(file); + account(file); + } + + public ManifestCacheValue build() { + return new ManifestCacheValue(dataFiles, deleteFiles, + dataContent ? metricEntryCount : 0L, + dataContent ? 0L : metricEntryCount, + retainedPayloadBytes); + } + + private void account(ContentFile file) { + metricEntryCount = MetaCacheWeightUtils.saturatedAdd( + metricEntryCount, metricEntryCount(file)); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, retainedPayloadBytes(file)); + } + } + + private static long metricEntryCount(ContentFile file) { + long count = mapSize(file.columnSizes()); + count = MetaCacheWeightUtils.saturatedAdd(count, mapSize(file.valueCounts())); + count = MetaCacheWeightUtils.saturatedAdd(count, mapSize(file.nullValueCounts())); + count = MetaCacheWeightUtils.saturatedAdd(count, mapSize(file.nanValueCounts())); + count = MetaCacheWeightUtils.saturatedAdd(count, mapSize(file.lowerBounds())); + return MetaCacheWeightUtils.saturatedAdd(count, mapSize(file.upperBounds())); + } + + private static long retainedPayloadBytes(ContentFile file) { + long bytes = MetaCacheWeightUtils.estimatedCharSequenceBytes(file.path()); + bytes = addBuffer(bytes, file.keyMetadata()); + bytes = addBuffers(bytes, file.lowerBounds()); + bytes = addBuffers(bytes, file.upperBounds()); + bytes = addListEntries(bytes, file.splitOffsets()); + bytes = addListEntries(bytes, file.equalityFieldIds()); + if (file instanceof DeleteFile) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes( + ((DeleteFile) file).referencedDataFile())); + } + return addPartitionPayload(bytes, file.partition()); + } + + private static long addBuffers(long bytes, Map buffers) { + if (buffers == null) { + return bytes; + } + for (ByteBuffer buffer : buffers.values()) { + bytes = addBuffer(bytes, buffer); + } + return bytes; + } + + private static long addBuffer(long bytes, ByteBuffer buffer) { + return buffer == null ? bytes : MetaCacheWeightUtils.saturatedAdd(bytes, buffer.capacity()); + } + + private static long addListEntries(long bytes, List values) { + if (values == null) { + return bytes; + } + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply( + values.size(), AUXILIARY_LIST_ENTRY_BYTES)); + } + + private static long addPartitionPayload(long bytes, StructLike partition) { + if (partition == null) { + return bytes; + } + try { + for (int index = 0; index < partition.size(); index++) { + Object value = partition.get(index, Object.class); + if (value instanceof CharSequence) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedCharSequenceBytes((CharSequence) value)); + } else if (value instanceof ByteBuffer) { + bytes = addBuffer(bytes, (ByteBuffer) value); + } else if (value instanceof byte[]) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ((byte[]) value).length); + } + } + } catch (RuntimeException ignored) { + // A third-party StructLike may reject Object.class. The fixed per-file allowance + // remains conservative, and cache accounting must never fail manifest loading. + } + return bytes; + } + + private static int mapSize(Map map) { + return map == null ? 0 : map.size(); + } + } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index 957ab6ed55e193..570652f245ff71 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -781,8 +781,9 @@ private Table useFrozenTableGeneration(Table currentTable) { if (snapshot.filter(IcebergMvccSnapshot.class::isInstance).isPresent()) { IcebergSnapshotCacheValue cacheValue = ((IcebergMvccSnapshot) snapshot.get()).getSnapshotCacheValue(); - if (cacheValue.getIcebergTable().isPresent()) { - Table frozenBaseTable = cacheValue.getIcebergTable().get(); + Optional
    frozenTable = cacheValue.getIcebergTable(); + if (frozenTable.isPresent()) { + Table frozenBaseTable = frozenTable.get(); if (isSystemTable && source.getTargetTable() instanceof IcebergSysExternalTable) { IcebergSysExternalTable systemTable = (IcebergSysExternalTable) source.getTargetTable(); if (systemTable.supportsSnapshotSelection()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCache.java index 46e58f1e380081..da2980f7185ca7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/MaxComputeExternalMetaCache.java @@ -27,6 +27,7 @@ import org.apache.doris.datasource.TablePartitionValues; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; @@ -52,7 +53,12 @@ public class MaxComputeExternalMetaCache extends AbstractExternalMetaCache { private final EntryHandle schemaEntry; public MaxComputeExternalMetaCache(ExecutorService refreshExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public MaxComputeExternalMetaCache(ExecutorService refreshExecutor, + ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); partitionValuesEntry = registerEntry(MetaCacheEntryDef.contextualOnly( ENTRY_PARTITION_VALUES, NameMapping.class, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java index a3a44151e45e2f..4cf326b9110b8d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java @@ -33,6 +33,7 @@ import java.util.List; import java.util.Map; import java.util.Objects; +import java.util.OptionalLong; import java.util.concurrent.ExecutorService; import java.util.function.Function; import java.util.function.Predicate; @@ -62,12 +63,19 @@ protected static CacheSpec defaultSchemaCacheSpec() { private final String engine; private final ExecutorService refreshExecutor; + private final ExternalMetaCacheBudgetManager budgetManager; private final Map catalogEntries = Maps.newConcurrentMap(); private final Map> metaCacheEntryDefs = Maps.newConcurrentMap(); protected AbstractExternalMetaCache(String engine, ExecutorService refreshExecutor) { + this(engine, refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.empty())); + } + + protected AbstractExternalMetaCache(String engine, ExecutorService refreshExecutor, + ExternalMetaCacheBudgetManager budgetManager) { this.engine = engine; this.refreshExecutor = Objects.requireNonNull(refreshExecutor, "refreshExecutor can not be null"); + this.budgetManager = Objects.requireNonNull(budgetManager, "budgetManager can not be null"); } @Override @@ -81,10 +89,43 @@ public Collection aliases() { } @Override - public void initCatalog(long catalogId, Map catalogProperties) { + public void validateCatalogProperties(Map catalogProperties) { Map safeCatalogProperties = CacheSpec.applyCompatibilityMap( catalogProperties, catalogPropertyCompatibilityMap()); - catalogEntries.computeIfAbsent(catalogId, id -> buildCatalogEntryGroup(safeCatalogProperties)); + validateMappedCatalogProperties(safeCatalogProperties, true); + } + + @Override + public void initCatalog(long catalogId, Map catalogProperties) { + if (catalogEntries.containsKey(catalogId)) { + return; + } + synchronized (this) { + if (catalogEntries.containsKey(catalogId)) { + return; + } + Map safeCatalogProperties = CacheSpec.applyCompatibilityMap( + catalogProperties, catalogPropertyCompatibilityMap()); + validateMappedCatalogProperties(safeCatalogProperties, false); + catalogEntries.put(catalogId, buildCatalogEntryGroup(catalogId, safeCatalogProperties)); + } + } + + private void validateMappedCatalogProperties( + Map catalogProperties, boolean validateAgainstLocalGlobalLimit) { + CacheSpec.validateEngineProperties(catalogProperties, engine, metaCacheEntryDefs); + OptionalLong catalogMaxWeight = budgetManager.parseCatalogMaxWeight(catalogProperties); + metaCacheEntryDefs.values().stream() + .filter(entryDef -> entryDef.getSizeEstimator() != null) + .map(entryDef -> CacheSpec.fromProperties( + catalogProperties, engine, entryDef.getName(), entryDef.getDefaultCacheSpec())) + .forEach(cacheSpec -> { + if (validateAgainstLocalGlobalLimit) { + budgetManager.validateHierarchy(catalogMaxWeight, cacheSpec.getMaxWeight()); + } else { + budgetManager.validateCatalogEntryHierarchy(catalogMaxWeight, cacheSpec.getMaxWeight()); + } + }); } @Override @@ -114,6 +155,7 @@ public MetaCacheEntry entry(long catalogId, String entryName, Class MetaCacheEntryDef def = requireMetaCacheEntryDef(entryName); ensureTypeCompatible(def, keyType, valueType); + beforeCatalogEntryLookupForTest(catalogId, entryName); MetaCacheEntry cacheEntry = group.get(entryName); if (cacheEntry == null) { throw new IllegalStateException(String.format( @@ -124,10 +166,10 @@ public MetaCacheEntry entry(long catalogId, String entryName, Class } @Override - public void invalidateCatalog(long catalogId) { + public synchronized void invalidateCatalog(long catalogId) { CatalogEntryGroup removed = catalogEntries.remove(catalogId); if (removed != null) { - removed.invalidateAll(); + removed.close(); } } @@ -162,8 +204,8 @@ public Map stats(long catalogId) { } @Override - public void close() { - catalogEntries.values().forEach(CatalogEntryGroup::invalidateAll); + public synchronized void close() { + catalogEntries.values().forEach(CatalogEntryGroup::close); catalogEntries.clear(); } @@ -191,6 +233,10 @@ protected final MetaCacheEntry entry(long catalogId, MetaCacheEntry return entry(catalogId, entryDef.getName(), entryDef.getKeyType(), entryDef.getValueType()); } + // Let tests pause after capturing a group and before looking up its entry. + void beforeCatalogEntryLookupForTest(long catalogId, String entryName) { + } + protected final String metaCacheTtlKey(String entryName) { return "meta.cache." + engine + "." + entryName + ".ttl-second"; } @@ -283,23 +329,52 @@ private void invalidateEntryIfMatched(CatalogEntryGroup group, MetaCacheE } } - private CatalogEntryGroup buildCatalogEntryGroup(Map catalogProperties) { + private CatalogEntryGroup buildCatalogEntryGroup(long catalogId, Map catalogProperties) { CatalogEntryGroup group = new CatalogEntryGroup(); - metaCacheEntryDefs.values() - .forEach(entryDef -> group.put(entryDef.getName(), newMetaCacheEntry(entryDef, catalogProperties))); - return group; + try { + metaCacheEntryDefs.values().forEach(entryDef -> group.put( + entryDef.getName(), newMetaCacheEntry(catalogId, entryDef, catalogProperties))); + return group; + } catch (RuntimeException | Error e) { + group.close(); + throw e; + } } @SuppressWarnings("unchecked") private MetaCacheEntry newMetaCacheEntry( - MetaCacheEntryDef rawEntryDef, Map catalogProperties) { + long catalogId, MetaCacheEntryDef rawEntryDef, Map catalogProperties) { MetaCacheEntryDef entryDef = (MetaCacheEntryDef) rawEntryDef; CacheSpec cacheSpec = CacheSpec.fromProperties( catalogProperties, engine, entryDef.getName(), entryDef.getDefaultCacheSpec()); - return new MetaCacheEntry<>(entryDef.getName(), - wrapSchemaValidator(entryDef.getLoader(), entryDef.getValueType()), - cacheSpec, - refreshExecutor, entryDef.isAutoRefresh(), entryDef.isContextualOnly()); + OptionalLong catalogMaxWeight = budgetManager.parseCatalogMaxWeight(catalogProperties); + if (cacheSpec.isWeightBounded() && entryDef.getSizeEstimator() == null) { + throw new IllegalArgumentException(String.format( + "Entry '%s' for engine '%s' configures max-weight but has no estimator.", + entryDef.getName(), engine)); + } + boolean enableWeight = entryDef.getSizeEstimator() != null + && (cacheSpec.isWeightBounded() + || catalogMaxWeight.isPresent() + || budgetManager.getGlobalMaxWeight().isPresent()); + ExternalMetaCacheBudgetManager.EntryBudget entryBudget = null; + if (enableWeight) { + entryBudget = budgetManager.createEntryBudget( + catalogId, engine, entryDef.getName(), catalogMaxWeight, cacheSpec.getMaxWeight()); + cacheSpec = cacheSpec.withMaxWeight(entryBudget.getEffectiveMaxWeight()); + } + try { + return new MetaCacheEntry<>(entryDef.getName(), + wrapSchemaValidator(entryDef.getLoader(), entryDef.getValueType()), + cacheSpec, + refreshExecutor, entryDef.isAutoRefresh(), entryDef.isContextualOnly(), + entryDef.getSizeEstimator(), entryBudget); + } catch (RuntimeException | Error e) { + if (entryBudget != null) { + entryBudget.close(); + } + throw e; + } } private Function wrapSchemaValidator(Function loader, Class valueType) { @@ -327,8 +402,12 @@ public MetaCacheEntry get(long catalogId) { return entry(catalogId, entryDef); } + @SuppressWarnings("unchecked") public MetaCacheEntry getIfInitialized(long catalogId) { - return isCatalogInitialized(catalogId) ? get(catalogId) : null; + // Read the group once. A concurrent invalidation may close that captured entry, which + // is safe; looking the group up a second time could instead throw after the first check. + CatalogEntryGroup group = catalogEntries.get(catalogId); + return group == null ? null : (MetaCacheEntry) group.get(entryDef.getName()); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java index 0bb640ad0d753c..1cea5d20a3a591 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java @@ -21,10 +21,16 @@ import org.apache.commons.lang3.math.NumberUtils; +import java.math.BigDecimal; +import java.math.BigInteger; import java.util.HashMap; +import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.OptionalLong; +import java.util.Set; +import java.util.regex.Matcher; +import java.util.regex.Pattern; /** * Common cache specification for external metadata caches. @@ -43,19 +49,36 @@ public final class CacheSpec { private static final String KEY_ENABLE = ".enable"; private static final String KEY_TTL_SECOND = ".ttl-second"; private static final String KEY_CAPACITY = ".capacity"; + private static final String KEY_MAX_WEIGHT = ".max-weight"; + private static final Pattern DATA_VOLUME_PATTERN = Pattern.compile("^([0-9]+)\\s*(B|KB|MB|GB|TB|PB)?$", + Pattern.CASE_INSENSITIVE); + private static final BigInteger LONG_MAX = BigInteger.valueOf(Long.MAX_VALUE); private final boolean enable; private final long ttlSecond; private final long capacity; + private final OptionalLong maxWeight; - private CacheSpec(boolean enable, long ttlSecond, long capacity) { + private CacheSpec(boolean enable, long ttlSecond, long capacity, OptionalLong maxWeight) { this.enable = enable; this.ttlSecond = ttlSecond; this.capacity = capacity; + this.maxWeight = Objects.requireNonNull(maxWeight, "maxWeight"); } public static CacheSpec of(boolean enable, long ttlSecond, long capacity) { - return new CacheSpec(enable, ttlSecond, capacity); + return new CacheSpec(enable, ttlSecond, capacity, OptionalLong.empty()); + } + + public static CacheSpec ofWeight(boolean enable, long ttlSecond, long capacity, long maxWeight) { + if (maxWeight < 0) { + throw new IllegalArgumentException("maxWeight can not be negative: " + maxWeight); + } + return new CacheSpec(enable, ttlSecond, capacity, OptionalLong.of(maxWeight)); + } + + public CacheSpec withMaxWeight(long effectiveMaxWeight) { + return ofWeight(enable, ttlSecond, capacity, effectiveMaxWeight); } public static PropertySpec.Builder propertySpecBuilder() { @@ -77,7 +100,8 @@ public static CacheSpec fromProperties(Map properties, PropertyS boolean enable = getBooleanProperty(properties, propertySpec.getEnableKey(), propertySpec.isDefaultEnable()); long ttlSecond = getLongProperty(properties, propertySpec.getTtlKey(), propertySpec.getDefaultTtlSecond()); long capacity = getLongProperty(properties, propertySpec.getCapacityKey(), propertySpec.getDefaultCapacity()); - return of(enable, ttlSecond, capacity); + OptionalLong maxWeight = getWeightProperty(properties, propertySpec.getMaxWeightKey()); + return new CacheSpec(enable, ttlSecond, capacity, maxWeight); } /** @@ -95,6 +119,7 @@ public static PropertySpec metaCachePropertySpec(String engine, String entryName .enable(cacheKeyPrefix + KEY_ENABLE, defaultSpec.isEnable()) .ttl(cacheKeyPrefix + KEY_TTL_SECOND, defaultSpec.getTtlSecond()) .capacity(cacheKeyPrefix + KEY_CAPACITY, defaultSpec.getCapacity()) + .maxWeight(cacheKeyPrefix + KEY_MAX_WEIGHT) .build(); } @@ -151,6 +176,68 @@ public static boolean isCacheEnabled(boolean enable, long ttlSecond, long capaci return enable && ttlSecond != 0 && capacity != 0; } + /** + * Parse an exact byte value with an optional binary unit. Percentages are accepted only + * when {@code allowPercent} is true and are resolved against {@code maxHeapBytes}. + */ + public static long parseWeight(String value, String key, boolean allowPercent, long maxHeapBytes) { + String normalized = Objects.requireNonNull(value, "value").trim(); + if (normalized.isEmpty()) { + throw invalidWeight(key, value); + } + if (normalized.endsWith("%")) { + if (!allowPercent || maxHeapBytes <= 0) { + throw invalidWeight(key, value); + } + String percentageText = normalized.substring(0, normalized.length() - 1).trim(); + try { + BigDecimal percentage = new BigDecimal(percentageText); + if (percentage.signum() < 0 || percentage.compareTo(BigDecimal.valueOf(100L)) > 0) { + throw invalidWeight(key, value); + } + BigInteger bytes = BigDecimal.valueOf(maxHeapBytes) + .multiply(percentage) + .divide(BigDecimal.valueOf(100L)) + .toBigInteger(); + return checkedLong(bytes, key, value); + } catch (NumberFormatException e) { + throw invalidWeight(key, value); + } + } + + Matcher matcher = DATA_VOLUME_PATTERN.matcher(normalized); + if (!matcher.matches()) { + throw invalidWeight(key, value); + } + BigInteger amount = new BigInteger(matcher.group(1)); + String rawUnit = matcher.group(2); + String unit = rawUnit == null ? "B" : rawUnit.toUpperCase(Locale.ROOT); + int power; + switch (unit) { + case "B": + power = 0; + break; + case "KB": + power = 1; + break; + case "MB": + power = 2; + break; + case "GB": + power = 3; + break; + case "TB": + power = 4; + break; + case "PB": + power = 5; + break; + default: + throw invalidWeight(key, value); + } + return checkedLong(amount.multiply(BigInteger.valueOf(1024L).pow(power)), key, value); + } + /** * Build standard external meta cache key prefix for one engine. * Example: {@code meta.cache.iceberg.} @@ -166,6 +253,85 @@ public static boolean isMetaCacheKeyForEngine(String key, String engine) { return key != null && engine != null && key.startsWith(metaCacheKeyPrefix(engine)); } + /** + * Strictly validate one engine namespace so misspelled entries/options cannot be silently ignored. + * The catalog-wide {@code meta.cache.max-weight} key is validated by the budget manager. + */ + static void validateEngineProperties(Map properties, String engine, + Map> entryDefs) { + Set weightedEntries = new java.util.HashSet<>(); + for (MetaCacheEntryDef entryDef : entryDefs.values()) { + if (entryDef.getSizeEstimator() != null) { + weightedEntries.add(entryDef.getName()); + } + } + validateEngineProperties(properties, engine, entryDefs.keySet(), weightedEntries); + } + + public static void validateEngineProperties(Map properties, String engine, + Set entryNames, Set weightedEntryNames) { + if (properties == null || properties.isEmpty()) { + return; + } + String enginePrefix = metaCacheKeyPrefix(engine); + for (Map.Entry property : properties.entrySet()) { + String key = property.getKey(); + if (key == null || !key.startsWith(enginePrefix)) { + continue; + } + String remainder = key.substring(enginePrefix.length()); + int optionSeparator = remainder.lastIndexOf('.'); + if (optionSeparator <= 0 || optionSeparator == remainder.length() - 1) { + throw new IllegalArgumentException("Unknown external meta cache property: " + key); + } + String entryName = remainder.substring(0, optionSeparator); + String option = remainder.substring(optionSeparator + 1); + if (!entryNames.contains(entryName)) { + throw new IllegalArgumentException("Unknown external meta cache entry property: " + key); + } + String value = property.getValue(); + switch (option) { + case "enable": + requireStrictBoolean(key, value); + break; + case "ttl-second": + requireLongAtLeast(key, value, CACHE_NO_TTL); + break; + case "capacity": + requireLongAtLeast(key, value, 0L); + break; + case "max-weight": + if (!weightedEntryNames.contains(entryName)) { + throw new IllegalArgumentException( + "External meta cache entry does not support max-weight: " + key); + } + parseWeight(value, key, false, 0L); + break; + default: + throw new IllegalArgumentException("Unknown external meta cache property: " + key); + } + } + } + + private static void requireStrictBoolean(String key, String value) { + if (value == null || (!"true".equalsIgnoreCase(value) && !"false".equalsIgnoreCase(value))) { + throw new IllegalArgumentException("Invalid boolean cache property '" + key + "': " + value); + } + } + + private static void requireLongAtLeast(String key, String value, long minimum) { + final long parsed; + try { + parsed = Long.parseLong(value); + } catch (RuntimeException e) { + throw new IllegalArgumentException("Invalid integer cache property '" + key + "': " + value, e); + } + if (parsed < minimum) { + throw new IllegalArgumentException("Cache property '" + key + "' must be >= " + minimum + + ", but was " + value); + } + } + /** * Convert ttlSecond to OptionalLong for CacheFactory. * ttlSecond=-1 means no expiration; ttlSecond=0 disables cache. @@ -193,6 +359,27 @@ private static long getLongProperty(Map properties, String key, return NumberUtils.toLong(value, defaultValue); } + private static OptionalLong getWeightProperty(Map properties, String key) { + if (key == null) { + return OptionalLong.empty(); + } + String value = properties.get(key); + return value == null + ? OptionalLong.empty() + : OptionalLong.of(parseWeight(value, key, false, 0L)); + } + + private static long checkedLong(BigInteger value, String key, String rawValue) { + if (value.signum() < 0 || value.compareTo(LONG_MAX) > 0) { + throw invalidWeight(key, rawValue); + } + return value.longValue(); + } + + private static IllegalArgumentException invalidWeight(String key, String value) { + return new IllegalArgumentException("Invalid cache weight for '" + key + "': " + value); + } + public boolean isEnable() { return enable; } @@ -205,6 +392,19 @@ public long getCapacity() { return capacity; } + public OptionalLong getMaxWeight() { + return maxWeight; + } + + public boolean isWeightBounded() { + return maxWeight.isPresent(); + } + + public boolean isCacheEnabled() { + return isCacheEnabled(enable, ttlSecond, capacity) + && (!maxWeight.isPresent() || maxWeight.getAsLong() != 0L); + } + public static final class PropertySpec { private final String enableKey; private final boolean defaultEnable; @@ -212,15 +412,17 @@ public static final class PropertySpec { private final long defaultTtlSecond; private final String capacityKey; private final long defaultCapacity; + private final String maxWeightKey; private PropertySpec(String enableKey, boolean defaultEnable, String ttlKey, - long defaultTtlSecond, String capacityKey, long defaultCapacity) { + long defaultTtlSecond, String capacityKey, long defaultCapacity, String maxWeightKey) { this.enableKey = enableKey; this.defaultEnable = defaultEnable; this.ttlKey = ttlKey; this.defaultTtlSecond = defaultTtlSecond; this.capacityKey = capacityKey; this.defaultCapacity = defaultCapacity; + this.maxWeightKey = maxWeightKey; } public String getEnableKey() { @@ -247,6 +449,10 @@ public long getDefaultCapacity() { return defaultCapacity; } + public String getMaxWeightKey() { + return maxWeightKey; + } + public static final class Builder { private String enableKey; private boolean defaultEnable; @@ -254,6 +460,7 @@ public static final class Builder { private long defaultTtlSecond; private String capacityKey; private long defaultCapacity; + private String maxWeightKey; public Builder enable(String key, boolean defaultValue) { this.enableKey = key; @@ -273,6 +480,11 @@ public Builder capacity(String key, long defaultValue) { return this; } + public Builder maxWeight(String key) { + this.maxWeightKey = Objects.requireNonNull(key, "key"); + return this; + } + public PropertySpec build() { return new PropertySpec( Objects.requireNonNull(enableKey, "enableKey is required"), @@ -280,7 +492,8 @@ public PropertySpec build() { Objects.requireNonNull(ttlKey, "ttlKey is required"), defaultTtlSecond, Objects.requireNonNull(capacityKey, "capacityKey is required"), - defaultCapacity); + defaultCapacity, + maxWeightKey); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java index c195087f415bfc..09376a1a129051 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java @@ -46,4 +46,12 @@ public Map stats() { public void invalidateAll() { entries.values().forEach(MetaCacheEntry::invalidateAll); } + + public void close() { + entries.values().forEach(MetaCacheEntry::close); + // Keep the closed entries reachable from this retired group. A query may have captured the + // group immediately before its catalog is removed; returning a closed entry lets that query + // serve an uncached load instead of spuriously observing an uninitialized entry. The group + // is already absent from the owner map and is reclaimed with the last concurrent reader. + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java index 1a067726ec9136..8b874fac8f659e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java @@ -41,6 +41,10 @@ public interface ExternalMetaCache { */ Collection aliases(); + /** Validate cache properties in this engine's canonical namespace. */ + default void validateCatalogProperties(Map catalogProperties) { + } + /** * Initialize all registered entries for one catalog under current engine. * Entry instances are created eagerly at this stage. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java new file mode 100644 index 00000000000000..a41e53bfb35b95 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java @@ -0,0 +1,414 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.apache.doris.common.Config; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.atomic.AtomicLong; + +/** + * FE-wide admission accounting for managed external metadata caches. + * + *

    All changes are serialized by one short critical section. Cache loads and + * estimators run outside it, so the lock only protects a few arithmetic and map + * operations while making global/catalog/entry reservation atomic. + */ +public final class ExternalMetaCacheBudgetManager { + public static final String CATALOG_MAX_WEIGHT_PROPERTY = "meta.cache.max-weight"; + + private final Object lock = new Object(); + private final OptionalLong globalMaxWeight; + private final Map catalogBuckets = new HashMap<>(); + private final Map entryBuckets = new HashMap<>(); + private long globalUsedWeight; + private final AtomicLong globalRejectedCount = new AtomicLong(); + + public ExternalMetaCacheBudgetManager(OptionalLong globalMaxWeight) { + this.globalMaxWeight = Objects.requireNonNull(globalMaxWeight, "globalMaxWeight"); + if (globalMaxWeight.isPresent() && globalMaxWeight.getAsLong() <= 0) { + throw new IllegalArgumentException("global max weight must be positive when enabled"); + } + } + + public static ExternalMetaCacheBudgetManager fromConfig() { + String configured = Config.external_meta_cache_max_weight; + long parsed = CacheSpec.parseWeight( + configured, + "external_meta_cache_max_weight", + true, + Runtime.getRuntime().maxMemory()); + if (configured.trim().endsWith("%") && parsed == 0L) { + throw new IllegalArgumentException( + "external_meta_cache_max_weight percentage must be greater than 0%"); + } + return new ExternalMetaCacheBudgetManager(parsed == 0L ? OptionalLong.empty() : OptionalLong.of(parsed)); + } + + public OptionalLong parseCatalogMaxWeight(Map catalogProperties) { + String configured = catalogProperties.get(CATALOG_MAX_WEIGHT_PROPERTY); + if (configured == null) { + return OptionalLong.empty(); + } + long parsed = CacheSpec.parseWeight(configured, CATALOG_MAX_WEIGHT_PROPERTY, false, 0L); + if (parsed <= 0) { + throw new IllegalArgumentException(CATALOG_MAX_WEIGHT_PROPERTY + " must be positive"); + } + return OptionalLong.of(parsed); + } + + /** Validate a catalog limit at DDL time against this FE's configured global bound. */ + public OptionalLong validateCatalogMaxWeight(Map catalogProperties) { + OptionalLong catalogMaxWeight = parseCatalogMaxWeight(catalogProperties); + validateHierarchy(catalogMaxWeight, OptionalLong.empty()); + return catalogMaxWeight; + } + + /** + * Create the budget handle used by one physical per-catalog cache entry. + */ + public EntryBudget createEntryBudget(long catalogId, String engine, String entryName, + OptionalLong catalogMaxWeight, OptionalLong entryMaxWeight) { + Objects.requireNonNull(engine, "engine"); + Objects.requireNonNull(entryName, "entryName"); + Objects.requireNonNull(catalogMaxWeight, "catalogMaxWeight"); + Objects.requireNonNull(entryMaxWeight, "entryMaxWeight"); + validateCatalogEntryHierarchy(catalogMaxWeight, entryMaxWeight); + + OptionalLong effectiveMax = minimumPresent(globalMaxWeight, catalogMaxWeight, entryMaxWeight); + if (!effectiveMax.isPresent()) { + throw new IllegalArgumentException("entry budget requires at least one configured weight bound"); + } + + EntryScope scope = new EntryScope(catalogId, engine, entryName); + synchronized (lock) { + Bucket catalogBucket = catalogBuckets.get(catalogId); + long catalogLimit = minimumLimit(globalMaxWeight, catalogMaxWeight); + if (catalogBucket == null) { + catalogBucket = new Bucket(catalogLimit); + catalogBuckets.put(catalogId, catalogBucket); + } else if (catalogBucket.maxWeight != catalogLimit) { + throw new IllegalStateException("Conflicting catalog cache max weight for catalog " + catalogId); + } + + if (entryBuckets.containsKey(scope)) { + throw new IllegalStateException("Duplicated external meta cache budget: " + scope); + } + Bucket entryBucket = new Bucket(effectiveMax.getAsLong()); + entryBuckets.put(scope, entryBucket); + return new EntryBudget(this, scope, catalogBucket, entryBucket, effectiveMax.getAsLong()); + } + } + + public OptionalLong getGlobalMaxWeight() { + return globalMaxWeight; + } + + public long getGlobalUsedWeight() { + synchronized (lock) { + return globalUsedWeight; + } + } + + public long getGlobalRejectedCount() { + return globalRejectedCount.get(); + } + + public void validateHierarchy(OptionalLong catalogMaxWeight, OptionalLong entryMaxWeight) { + if (globalMaxWeight.isPresent() && catalogMaxWeight.isPresent() + && catalogMaxWeight.getAsLong() > globalMaxWeight.getAsLong()) { + throw new IllegalArgumentException(CATALOG_MAX_WEIGHT_PROPERTY + " can not exceed FE global max weight"); + } + OptionalLong parent = catalogMaxWeight.isPresent() ? catalogMaxWeight : globalMaxWeight; + if (parent.isPresent() && entryMaxWeight.isPresent() + && entryMaxWeight.getAsLong() > parent.getAsLong()) { + throw new IllegalArgumentException("entry max weight can not exceed its parent max weight"); + } + } + + /** + * Validate persisted catalog-to-entry hierarchy without comparing it with this FE's local + * global bound. Catalog properties are validated on the master, while the global percentage + * is resolved independently from each FE's heap. Runtime admission therefore clamps to the + * local global limit instead of rejecting a catalog accepted on a larger master. + */ + public void validateCatalogEntryHierarchy(OptionalLong catalogMaxWeight, OptionalLong entryMaxWeight) { + OptionalLong parent = catalogMaxWeight; + if (parent.isPresent() && entryMaxWeight.isPresent() + && entryMaxWeight.getAsLong() > parent.getAsLong()) { + throw new IllegalArgumentException("entry max weight can not exceed its parent max weight"); + } + } + + private Optional tryReserve(EntryBudget entryBudget, long bytes) { + checkWeight(bytes); + synchronized (lock) { + if (entryBudget.closed) { + return Optional.empty(); + } + if (!fits(limitOf(globalMaxWeight), globalUsedWeight, bytes) + || !fits(entryBudget.catalogBucket.maxWeight, entryBudget.catalogBucket.usedWeight, bytes) + || !fits(entryBudget.entryBucket.maxWeight, entryBudget.entryBucket.usedWeight, bytes)) { + entryBudget.rejectedCount.incrementAndGet(); + globalRejectedCount.incrementAndGet(); + return Optional.empty(); + } + addUsed(entryBudget, bytes); + return Optional.of(new AdmissionReservation(this, entryBudget, bytes)); + } + } + + private boolean resize(AdmissionReservation reservation, long newBytes) { + checkWeight(newBytes); + synchronized (lock) { + if (!reservation.active || reservation.entryBudget.closed) { + return false; + } + long delta = newBytes - reservation.bytes; + if (delta > 0 && (!fits(limitOf(globalMaxWeight), globalUsedWeight, delta) + || !fits(reservation.entryBudget.catalogBucket.maxWeight, + reservation.entryBudget.catalogBucket.usedWeight, delta) + || !fits(reservation.entryBudget.entryBucket.maxWeight, + reservation.entryBudget.entryBucket.usedWeight, delta))) { + reservation.entryBudget.rejectedCount.incrementAndGet(); + globalRejectedCount.incrementAndGet(); + return false; + } + if (delta >= 0) { + addUsed(reservation.entryBudget, delta); + } else { + subtractUsed(reservation.entryBudget, -delta); + } + reservation.bytes = newBytes; + return true; + } + } + + private void release(AdmissionReservation reservation) { + synchronized (lock) { + if (!reservation.active) { + return; + } + subtractUsed(reservation.entryBudget, reservation.bytes); + reservation.bytes = 0L; + reservation.active = false; + } + } + + private void close(EntryBudget entryBudget) { + synchronized (lock) { + if (entryBudget.closed) { + return; + } + if (entryBudget.entryBucket.usedWeight != 0L) { + throw new IllegalStateException("entry budget closed with active reservations: " + entryBudget.scope); + } + entryBudget.closed = true; + entryBuckets.remove(entryBudget.scope, entryBudget.entryBucket); + Bucket catalogBucket = entryBudget.catalogBucket; + boolean catalogStillReferenced = entryBuckets.keySet().stream() + .anyMatch(scope -> scope.catalogId == entryBudget.scope.catalogId); + if (!catalogStillReferenced && catalogBucket.usedWeight == 0L) { + catalogBuckets.remove(entryBudget.scope.catalogId, catalogBucket); + } + } + } + + private void addUsed(EntryBudget entryBudget, long bytes) { + globalUsedWeight += bytes; + entryBudget.catalogBucket.usedWeight += bytes; + entryBudget.entryBucket.usedWeight += bytes; + } + + private void subtractUsed(EntryBudget entryBudget, long bytes) { + if (bytes > globalUsedWeight + || bytes > entryBudget.catalogBucket.usedWeight + || bytes > entryBudget.entryBucket.usedWeight) { + throw new IllegalStateException("external meta cache budget accounting underflow"); + } + globalUsedWeight -= bytes; + entryBudget.catalogBucket.usedWeight -= bytes; + entryBudget.entryBucket.usedWeight -= bytes; + } + + private static boolean fits(long maxWeight, long usedWeight, long delta) { + return delta >= 0 && usedWeight <= maxWeight && delta <= maxWeight - usedWeight; + } + + private static long limitOf(OptionalLong configured) { + return configured.isPresent() ? configured.getAsLong() : Long.MAX_VALUE; + } + + private static long minimumLimit(OptionalLong first, OptionalLong second) { + return Math.min(limitOf(first), limitOf(second)); + } + + private static OptionalLong minimumPresent(OptionalLong first, OptionalLong second, OptionalLong third) { + if (!first.isPresent() && !second.isPresent() && !third.isPresent()) { + return OptionalLong.empty(); + } + long minimum = Math.min(limitOf(first), Math.min(limitOf(second), limitOf(third))); + return OptionalLong.of(minimum); + } + + private static void checkWeight(long bytes) { + if (bytes < 0) { + throw new IllegalArgumentException("cache reservation can not be negative: " + bytes); + } + } + + private static final class Bucket { + private final long maxWeight; + private long usedWeight; + + private Bucket(long maxWeight) { + this.maxWeight = maxWeight; + } + } + + private static final class EntryScope { + private final long catalogId; + private final String engine; + private final String entryName; + + private EntryScope(long catalogId, String engine, String entryName) { + this.catalogId = catalogId; + this.engine = engine; + this.entryName = entryName; + } + + @Override + public boolean equals(Object other) { + if (this == other) { + return true; + } + if (!(other instanceof EntryScope)) { + return false; + } + EntryScope that = (EntryScope) other; + return catalogId == that.catalogId && engine.equals(that.engine) && entryName.equals(that.entryName); + } + + @Override + public int hashCode() { + return Objects.hash(catalogId, engine, entryName); + } + + @Override + public String toString() { + return catalogId + "/" + engine + "/" + entryName; + } + } + + public static final class EntryBudget { + private final ExternalMetaCacheBudgetManager manager; + private final EntryScope scope; + private final Bucket catalogBucket; + private final Bucket entryBucket; + private final long effectiveMaxWeight; + private final AtomicLong rejectedCount = new AtomicLong(); + // Guarded by manager.lock. A closed handle must never re-enter accounting. + private boolean closed; + + private EntryBudget(ExternalMetaCacheBudgetManager manager, EntryScope scope, + Bucket catalogBucket, Bucket entryBucket, long effectiveMaxWeight) { + this.manager = manager; + this.scope = scope; + this.catalogBucket = catalogBucket; + this.entryBucket = entryBucket; + this.effectiveMaxWeight = effectiveMaxWeight; + } + + public Optional tryReserve(long bytes) { + return manager.tryReserve(this, bytes); + } + + public long getEffectiveMaxWeight() { + return effectiveMaxWeight; + } + + public long getUsedWeight() { + synchronized (manager.lock) { + return entryBucket.usedWeight; + } + } + + public long getCatalogUsedWeight() { + synchronized (manager.lock) { + return catalogBucket.usedWeight; + } + } + + public long getCatalogMaxWeight() { + return catalogBucket.maxWeight == Long.MAX_VALUE ? -1L : catalogBucket.maxWeight; + } + + public long getRejectedCount() { + return rejectedCount.get(); + } + + public long getGlobalUsedWeight() { + return manager.getGlobalUsedWeight(); + } + + public long getGlobalMaxWeight() { + return manager.globalMaxWeight.isPresent() ? manager.globalMaxWeight.getAsLong() : -1L; + } + + public void close() { + manager.close(this); + } + } + + public static final class AdmissionReservation { + private final ExternalMetaCacheBudgetManager manager; + private final EntryBudget entryBudget; + private long bytes; + private boolean active = true; + + private AdmissionReservation(ExternalMetaCacheBudgetManager manager, EntryBudget entryBudget, long bytes) { + this.manager = manager; + this.entryBudget = entryBudget; + this.bytes = bytes; + } + + public boolean tryResize(long newBytes) { + return manager.resize(this, newBytes); + } + + public void release() { + manager.release(this); + } + + public long getBytes() { + synchronized (manager.lock) { + return bytes; + } + } + + public boolean isActive() { + synchronized (manager.lock) { + return active; + } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java index 30668163539d3b..01998d16f2d3ca 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java @@ -19,14 +19,29 @@ import org.apache.doris.common.CacheFactory; import org.apache.doris.common.Config; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.AdmissionReservation; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.EntryBudget; import com.github.benmanes.caffeine.cache.Cache; import com.github.benmanes.caffeine.cache.LoadingCache; +import com.github.benmanes.caffeine.cache.RemovalCause; +import com.github.benmanes.caffeine.cache.Weigher; import com.github.benmanes.caffeine.cache.stats.CacheStats; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import java.util.HashSet; +import java.util.Map; import java.util.Objects; +import java.util.Optional; import java.util.OptionalLong; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; @@ -40,8 +55,22 @@ * key/predicate/full invalidation, and lightweight runtime stats. */ public class MetaCacheEntry { + private static final Logger LOG = LogManager.getLogger(MetaCacheEntry.class); // Use striped locks to deduplicate slow external loads without managing per-key lock lifecycle. private static final int LOAD_LOCK_STRIPES = 128; + private static final int LOCAL_EVICTION_BATCH_SIZE = 16; + private static final int REMOVAL_CLEANUP_BATCH_SIZE = 256; + // Direct Caffeine callbacks must not wait for admissionLock. A daemon drains one coalesced + // generation map per physical entry after callbacks return; cleanup tasks never capture values. + private static final ExecutorService REMOVAL_CLEANUP_EXECUTOR = Executors.newSingleThreadExecutor(runnable -> { + Thread thread = new Thread(runnable, "external-meta-cache-removal-cleanup"); + thread.setDaemon(true); + return thread; + }); + // Conservative retained cost outside the estimator-owned key/value graph: Caffeine's data + // node and policy links plus the reservation ConcurrentHashMap node, record and token. This + // deliberately overestimates common compressed-oops layouts; calibrate downward only with JOL. + static final long FIXED_ENTRY_ACCOUNTING_OVERHEAD_BYTES = 512L; private final String name; @Nullable @@ -49,15 +78,39 @@ public class MetaCacheEntry { private final CacheSpec cacheSpec; private final boolean effectiveEnabled; private final boolean autoRefresh; + private final ExecutorService refreshExecutor; + @Nullable + private final MetaCacheSizeEstimator sizeEstimator; + @Nullable + private final EntryBudget entryBudget; + private final boolean weightBounded; + // Estimator-backed entries use the same generation-fenced refresh protocol even before a + // max-weight is configured. This keeps event ordering stable when weight governance is toggled. + private final boolean generationFencedRefresh; // Keep the loading cache for refreshAfterWrite and the legacy sync-load path when the feature is disabled. private final LoadingCache loadingData; // Use the plain cache view for manual miss load so slow I/O does not happen in Caffeine's sync load path. private final Cache data; // Protect one key stripe at a time to deduplicate concurrent miss loads with bounded lock count. private final Object[] loadLocks = new Object[LOAD_LOCK_STRIPES]; + // Serialize weighted cache mutation with reservation ownership changes. + private final Object admissionLock = new Object(); + // Ownership records deliberately contain no V reference. A weighted cache's Caffeine soft + // reference must be the only cache-owned path to its value, while generation fencing keeps + // delayed removal callbacks from releasing a replacement reservation. + private final Map reservations = new ConcurrentHashMap<>(); + private final Map refreshRecords = new ConcurrentHashMap<>(); + private final Map pendingRemovalGenerations = new ConcurrentHashMap<>(); + private final AtomicBoolean removalCleanupScheduled = new AtomicBoolean(false); + private final Map refreshesInFlight = new ConcurrentHashMap<>(); + // A state exists only while a miss/refresh for the key is in flight. Mutations advance that + // state's epoch, fencing stale publication without retaining every key ever observed. + private final Map keyMutationStates = new ConcurrentHashMap<>(); private final AtomicLong invalidateCount = new AtomicLong(0); - // Bump generation before invalidation so in-flight manual loads do not repopulate stale values. - private final AtomicLong invalidateGeneration = new AtomicLong(0); + // Full invalidation is the only cross-key fence. Ordinary mutations use the per-key state. + private final AtomicLong fullInvalidationGeneration = new AtomicLong(0); + // Primitive owner id lets queued refresh work fence a reservation without retaining its value. + private final AtomicLong reservationGeneration = new AtomicLong(0); // Track load statistics outside Caffeine because manual miss loads bypass the built-in load counters. private final AtomicLong loadSuccessCount = new AtomicLong(0); private final AtomicLong loadFailureCount = new AtomicLong(0); @@ -65,6 +118,11 @@ public class MetaCacheEntry { private final AtomicLong lastLoadSuccessTimeMs = new AtomicLong(-1L); private final AtomicLong lastLoadFailureTimeMs = new AtomicLong(-1L); private final AtomicReference lastError = new AtomicReference<>(""); + private final AtomicLong weightAdmissionRejectedCount = new AtomicLong(0L); + private final AtomicLong localEvictionCount = new AtomicLong(0L); + private final AtomicLong localEvictionWeight = new AtomicLong(0L); + private final AtomicReference lastWeightRejectReason = new AtomicReference<>(""); + private final AtomicBoolean closed = new AtomicBoolean(false); public MetaCacheEntry(String name, Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor) { this(name, loader, cacheSpec, refreshExecutor, true, false); @@ -77,6 +135,12 @@ public MetaCacheEntry(String name, Function loader, CacheSpec cacheSpec, E public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly) { + this(name, loader, cacheSpec, refreshExecutor, autoRefresh, contextualOnly, null, null); + } + + public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, + ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, + @Nullable MetaCacheSizeEstimator sizeEstimator, @Nullable EntryBudget entryBudget) { this.name = name; if (contextualOnly) { if (loader != null) { @@ -91,23 +155,43 @@ public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec ca this.loader = loader; this.cacheSpec = Objects.requireNonNull(cacheSpec, "cacheSpec can not be null"); this.autoRefresh = autoRefresh; - Objects.requireNonNull(refreshExecutor, "refreshExecutor can not be null"); - this.effectiveEnabled = CacheSpec.isCacheEnabled( - this.cacheSpec.isEnable(), this.cacheSpec.getTtlSecond(), this.cacheSpec.getCapacity()); + this.refreshExecutor = Objects.requireNonNull(refreshExecutor, "refreshExecutor can not be null"); + this.sizeEstimator = sizeEstimator; + this.entryBudget = entryBudget; + this.weightBounded = this.cacheSpec.isWeightBounded(); + this.generationFencedRefresh = autoRefresh && sizeEstimator != null; + if (weightBounded && (sizeEstimator == null || entryBudget == null)) { + throw new IllegalArgumentException("weighted cache entry requires both estimator and budget: " + name); + } + this.effectiveEnabled = this.cacheSpec.isCacheEnabled(); OptionalLong expireAfterAccessSec = effectiveEnabled ? CacheSpec.toExpireAfterAccess(this.cacheSpec.getTtlSecond()) : OptionalLong.empty(); OptionalLong refreshAfterWriteSec = - effectiveEnabled && autoRefresh + effectiveEnabled && autoRefresh && !weightBounded && !generationFencedRefresh ? OptionalLong.of(Config.external_cache_refresh_time_minutes * 60) : OptionalLong.empty(); long maxSize = effectiveEnabled ? this.cacheSpec.getCapacity() : 0L; + Weigher cacheWeigher = weightBounded ? this::weigh : null; CacheFactory cacheFactory = new CacheFactory( expireAfterAccessSec, refreshAfterWriteSec, maxSize, + weightBounded ? OptionalLong.of(effectiveEnabled ? this.cacheSpec.getMaxWeight().getAsLong() : 0L) + : OptionalLong.empty(), + cacheWeigher, true, null); - this.loadingData = cacheFactory.buildCache(this::loadFromDefaultLoader, refreshExecutor); + if (weightBounded) { + cacheFactory.withSoftValues(); + } + if (weightBounded || generationFencedRefresh) { + // Direct notification avoids queuing REPLACED values. The listener itself is lock-free + // and delegates only current-owner cleanup, so it is safe under Caffeine's eviction lock. + this.loadingData = cacheFactory.buildCacheWithSyncRemovalListener( + this::loadFromDefaultLoader, this::onRemoval); + } else { + this.loadingData = cacheFactory.buildCache(this::loadFromDefaultLoader, refreshExecutor); + } this.data = loadingData; // Initialize striped locks eagerly to keep the hot path allocation-free. for (int i = 0; i < loadLocks.length; i++) { @@ -120,6 +204,9 @@ public String name() { } public V get(K key) { + if (closed.get()) { + return loadAndTrack(key, this::applyDefaultLoader); + } if (!isManualMissLoadEnabled()) { return loadingData.get(key); } @@ -128,6 +215,9 @@ public V get(K key) { public V get(K key, Function missLoader) { Function loadFunction = Objects.requireNonNull(missLoader, "missLoader can not be null"); + if (closed.get()) { + return loadAndTrack(key, loadFunction); + } if (!isManualMissLoadEnabled()) { return loadingData.get(key, typedKey -> loadAndTrack(typedKey, loadFunction)); } @@ -135,42 +225,232 @@ public V get(K key, Function missLoader) { } public V getIfPresent(K key) { - if (!effectiveEnabled) { + if (!effectiveEnabled || closed.get()) { return null; } - return data.getIfPresent(key); + V value = data.getIfPresent(key); + if (value != null) { + maybeRefreshManagedValue(key, value); + } + return value; + } + + /** Return the current value without recording a user-visible cache request. */ + public V peekIfPresent(K key) { + if (!effectiveEnabled || closed.get()) { + return null; + } + return data.asMap().get(key); + } + + /** + * Fence loads and refreshes that started before an event, but only while the expected value is + * still current. Estimator-backed entries retain the known-good value and advance the key's + * mutation epoch. Other count-based entries must invalidate because their legacy Caffeine-managed + * refresh path does not participate in that generation protocol. + */ + public boolean fenceInFlightLoadIfSame(K key, V expectedCurrent) { + Objects.requireNonNull(expectedCurrent, "expectedCurrent can not be null"); + synchronized (admissionLock) { + if (!effectiveEnabled || closed.get() || data.asMap().get(key) != expectedCurrent) { + return false; + } + advanceKeyMutation(key); + if (!weightBounded && !generationFencedRefresh) { + if (!data.asMap().remove(key, expectedCurrent)) { + return false; + } + invalidateCount.incrementAndGet(); + } + return true; + } } public void put(K key, V value) { - if (!effectiveEnabled) { + if (!effectiveEnabled || closed.get()) { return; } - data.put(key, value); + if (weightBounded) { + admitWeightedValue(key, value, null, false, null, -1L, true); + } else { + synchronized (admissionLock) { + if (!closed.get()) { + advanceKeyMutation(key); + putNonWeightedValue(key, value); + } + } + } } - public void invalidateKey(K key) { - invalidateGeneration.incrementAndGet(); - if (data.asMap().remove(key) != null) { + /** Result of an atomic compare-and-replace operation. */ + public enum ReplaceResult { + REPLACED, + NOT_CURRENT, + REJECTED, + DISABLED + } + + /** + * Replace one cached value only when it is still the expected identity. + * + *

    Weighted entries perform the identity check, budget resize and Caffeine write under the + * same admission lock. Callers can therefore distinguish a concurrent update from admission + * rejection and avoid retaining a value they already know is stale. + */ + public ReplaceResult tryReplace(K key, V expectedCurrent, V newValue) { + Objects.requireNonNull(expectedCurrent, "expectedCurrent can not be null"); + Objects.requireNonNull(newValue, "newValue can not be null"); + if (!effectiveEnabled || closed.get()) { + return ReplaceResult.DISABLED; + } + if (weightBounded) { + return toReplaceResult(admitWeightedValue( + key, newValue, expectedCurrent, true, null, -1L, true)); + } + synchronized (admissionLock) { + AtomicReference result = new AtomicReference<>(ReplaceResult.NOT_CURRENT); + AtomicReference published = new AtomicReference<>(); + data.asMap().computeIfPresent(key, (ignored, current) -> { + if (closed.get()) { + result.set(ReplaceResult.DISABLED); + return current; + } + if (current != expectedCurrent) { + return current; + } + advanceKeyMutation(key); + published.set(publishRefreshRecord(key)); + result.set(ReplaceResult.REPLACED); + return newValue; + }); + RefreshRecord record = published.get(); + if (record != null && refreshRecords.get(key) == record + && data.asMap().get(key) == newValue) { + record.published = true; + } + return result.get(); + } + } + + /** Remove a key only if it still maps to the expected value identity. */ + public boolean invalidateKeyIfSame(K key, V expectedCurrent) { + Objects.requireNonNull(expectedCurrent, "expectedCurrent can not be null"); + if (!weightBounded) { + synchronized (admissionLock) { + AtomicBoolean removed = new AtomicBoolean(false); + data.asMap().computeIfPresent(key, (ignored, current) -> { + if (current != expectedCurrent) { + return current; + } + advanceKeyMutation(key); + invalidateCount.incrementAndGet(); + refreshRecords.remove(key); + removed.set(true); + return null; + }); + return removed.get(); + } + } + synchronized (admissionLock) { + V current = data.asMap().get(key); + ReservationRecord record = reservations.get(key); + if (current != expectedCurrent || record == null || !record.published) { + return false; + } + advanceKeyMutation(key); + if (!data.asMap().remove(key, current)) { + return false; + } + releaseReservation(key, record.generation); invalidateCount.incrementAndGet(); + return true; + } + } + + public void invalidateKey(K key) { + synchronized (admissionLock) { + advanceKeyMutation(key); + if (weightBounded) { + ReservationRecord record = reservations.get(key); + V removed = data.asMap().remove(key); + if (removed != null) { + invalidateCount.incrementAndGet(); + } + if (record != null && data.asMap().get(key) == null) { + releaseReservation(key, record.generation); + } + } else { + V removed = data.asMap().remove(key); + if (removed != null) { + refreshRecords.remove(key); + invalidateCount.incrementAndGet(); + } + } } } public void invalidateIf(Predicate predicate) { - invalidateGeneration.incrementAndGet(); - data.asMap().keySet().removeIf(key -> { - if (predicate.test(key)) { - invalidateCount.incrementAndGet(); - return true; + synchronized (admissionLock) { + Set candidates = new HashSet<>(data.asMap().keySet()); + candidates.addAll(keyMutationStates.keySet()); + if (weightBounded) { + candidates.addAll(reservations.keySet()); + } else if (generationFencedRefresh) { + candidates.addAll(refreshRecords.keySet()); } - return false; - }); + for (K key : candidates) { + if (predicate.test(key)) { + advanceKeyMutation(key); + if (weightBounded) { + ReservationRecord record = reservations.get(key); + V removed = data.asMap().remove(key); + if (removed != null) { + invalidateCount.incrementAndGet(); + } + if (record != null && data.asMap().get(key) == null) { + releaseReservation(key, record.generation); + } + } else { + V removed = data.asMap().remove(key); + refreshRecords.remove(key); + if (removed != null) { + invalidateCount.incrementAndGet(); + } + } + } + } + } } public void invalidateAll() { - invalidateGeneration.incrementAndGet(); - long size = data.estimatedSize(); - data.invalidateAll(); - invalidateCount.addAndGet(size); + synchronized (admissionLock) { + fullInvalidationGeneration.incrementAndGet(); + if (weightBounded) { + long size = data.estimatedSize(); + beforeWeightedInvalidateAllForTest(); + data.invalidateAll(); + reservations.values().forEach(record -> record.reservation.release()); + reservations.clear(); + pendingRemovalGenerations.clear(); + invalidateCount.addAndGet(size); + } else { + long size = data.estimatedSize(); + data.invalidateAll(); + refreshRecords.clear(); + pendingRemovalGenerations.clear(); + invalidateCount.addAndGet(size); + } + } + } + + public void close() { + if (!closed.compareAndSet(false, true)) { + return; + } + invalidateAll(); + if (entryBudget != null) { + entryBudget.close(); + } } public void forEach(BiConsumer consumer) { @@ -198,62 +478,581 @@ public MetaCacheEntryStats stats() { failureCount, totalLoadTime, totalLoadCount == 0 ? 0D : (double) totalLoadTime / totalLoadCount, - cacheStats.evictionCount(), + saturatedAdd(cacheStats.evictionCount(), localEvictionCount.get()), invalidateCount.get(), lastLoadSuccessTimeMs.get(), lastLoadFailureTimeMs.get(), - lastError.get()); + lastError.get(), + weightBounded, + weightBounded ? cacheSpec.getMaxWeight().getAsLong() : -1L, + weightBounded ? entryBudget.getUsedWeight() : -1L, + weightBounded ? saturatedAdd(cacheStats.evictionWeight(), localEvictionWeight.get()) : -1L, + weightBounded ? weightAdmissionRejectedCount.get() : -1L, + weightBounded ? entryBudget.getCatalogMaxWeight() : -1L, + weightBounded ? entryBudget.getCatalogUsedWeight() : -1L, + weightBounded ? entryBudget.getGlobalMaxWeight() : -1L, + weightBounded ? entryBudget.getGlobalUsedWeight() : -1L, + weightBounded ? lastWeightRejectReason.get() : ""); + } + + public boolean isWeightBounded() { + return weightBounded; + } + + private AdmissionResult admitWeightedValue( + K key, V value, @Nullable V expectedCurrent, boolean requireExpected, + @Nullable KeyMutationToken expectedMutation, long expectedReservationGeneration, + boolean advanceMutationOnAdmission) { + if (closed.get()) { + return AdmissionResult.DISABLED; + } + MetaCacheSizeEstimate estimate; + try { + estimate = Objects.requireNonNull(sizeEstimator.estimate(key, value), "size estimate"); + } catch (IllegalArgumentException e) { + throw e; + } catch (RuntimeException e) { + rejectWeight("invalid_estimate"); + return AdmissionResult.REJECTED; + } + if (!estimate.isComplete()) { + rejectWeight(estimate.getIncompleteReason()); + return AdmissionResult.REJECTED; + } + + long estimatedPayloadBytes = estimate.getBytes(); + // A retained non-null key/value plus Caffeine node can never consume zero bytes. Treat a + // complete zero as an estimator contract violation so an omitted formula cannot bypass + // every quota and admit an unbounded number of zero-weight entries. + if (estimatedPayloadBytes == 0L) { + rejectWeight("invalid_estimate"); + return AdmissionResult.REJECTED; + } + long newWeight = MetaCacheWeightUtils.saturatedAdd( + estimatedPayloadBytes, FIXED_ENTRY_ACCOUNTING_OVERHEAD_BYTES); + synchronized (admissionLock) { + if (closed.get()) { + return AdmissionResult.DISABLED; + } + if (expectedMutation != null && !isKeyMutationCurrent(key, expectedMutation)) { + return AdmissionResult.NOT_CURRENT; + } + V oldValue = data.asMap().get(key); + ReservationRecord record = reservations.get(key); + if (expectedReservationGeneration >= 0L + && (record == null || record.generation != expectedReservationGeneration)) { + return AdmissionResult.NOT_CURRENT; + } + if (requireExpected && oldValue != expectedCurrent) { + return AdmissionResult.NOT_CURRENT; + } + if (oldValue == null && record != null) { + reservations.remove(key, record); + record.reservation.release(); + record = null; + } + if (oldValue != null && (record == null || !record.published)) { + rejectWeight("missing_reservation"); + return AdmissionResult.REJECTED; + } + + if (record == null) { + Optional reservation = reserveWithLocalEviction(key, newWeight); + if (!reservation.isPresent()) { + rejectWeight("budget_exceeded"); + return AdmissionResult.REJECTED; + } + ReservationRecord newRecord = new ReservationRecord( + newWeight, reservation.get(), nextReservationGeneration()); + if (advanceMutationOnAdmission) { + advanceKeyMutation(key); + } + reservations.put(key, newRecord); + try { + beforeWeightedCachePutForTest(key, value); + data.put(key, value); + if (reservations.get(key) == newRecord && data.asMap().get(key) == value) { + newRecord.published = true; + } + return AdmissionResult.ADMITTED; + } catch (RuntimeException | Error e) { + reservations.remove(key, newRecord); + newRecord.reservation.release(); + throw e; + } + } + + ReservationRecord previousRecord = record; + long reservedWeight = Math.max(previousRecord.weight, newWeight); + if (!resizeWithLocalEviction(key, previousRecord.reservation, reservedWeight)) { + rejectWeight("budget_exceeded"); + return AdmissionResult.REJECTED; + } + if (advanceMutationOnAdmission) { + advanceKeyMutation(key); + } + ReservationRecord newRecord = new ReservationRecord( + newWeight, previousRecord.reservation, nextReservationGeneration()); + reservations.put(key, newRecord); + try { + beforeWeightedCachePutForTest(key, value); + data.put(key, value); + boolean retained = reservations.get(key) == newRecord && data.asMap().get(key) == value; + if (retained) { + newRecord.published = true; + } + if (retained && reservedWeight != newWeight && !newRecord.reservation.tryResize(newWeight)) { + throw new IllegalStateException("failed to release cache replacement reservation delta"); + } + return AdmissionResult.ADMITTED; + } catch (RuntimeException | Error e) { + if (reservations.replace(key, newRecord, previousRecord)) { + if (data.asMap().get(key) == null) { + reservations.remove(key, previousRecord); + previousRecord.reservation.release(); + } else if (!previousRecord.reservation.tryResize(previousRecord.weight)) { + throw new IllegalStateException("failed to roll back cache replacement reservation", e); + } + } + throw e; + } + } + } + + private Optional reserveWithLocalEviction(K incomingKey, long bytes) { + if (bytes > entryBudget.getEffectiveMaxWeight()) { + return Optional.empty(); + } + Optional reservation = entryBudget.tryReserve(bytes); + while (!reservation.isPresent()) { + int evicted = evictLocalColdest(incomingKey, LOCAL_EVICTION_BATCH_SIZE); + if (evicted == 0) { + break; + } + reservation = entryBudget.tryReserve(bytes); + } + return reservation; + } + + private boolean resizeWithLocalEviction(K incomingKey, AdmissionReservation reservation, long newBytes) { + if (newBytes > entryBudget.getEffectiveMaxWeight()) { + return false; + } + if (reservation.tryResize(newBytes)) { + return true; + } + while (true) { + int evicted = evictLocalColdest(incomingKey, LOCAL_EVICTION_BATCH_SIZE); + if (evicted == 0) { + return false; + } + if (reservation.tryResize(newBytes)) { + return true; + } + } + } + + private int evictLocalColdest(K incomingKey, int limit) { + if (!data.policy().eviction().isPresent()) { + return 0; + } + Map coldest = data.policy().eviction().get().coldest(limit); + int evicted = 0; + for (Map.Entry candidate : coldest.entrySet()) { + if (Objects.equals(candidate.getKey(), incomingKey)) { + continue; + } + V current = data.asMap().get(candidate.getKey()); + ReservationRecord record = reservations.get(candidate.getKey()); + long evictedWeight = record != null && record.published && current != null ? record.weight : 0L; + if (current == candidate.getValue() && data.asMap().remove(candidate.getKey(), current)) { + if (record != null) { + releaseReservation(candidate.getKey(), record.generation); + } + localEvictionCount.incrementAndGet(); + localEvictionWeight.accumulateAndGet(evictedWeight, MetaCacheWeightUtils::saturatedAdd); + evicted++; + } + } + return evicted; + } + + private int weigh(K key, V value) { + ReservationRecord record = reservations.get(key); + // Every supported write path installs the reservation record before calling data.put. + // Missing ownership is an invariant violation, so fail closed without invoking an O(n) + // estimator from Caffeine's hot weigher callback. + long weight = record == null ? Integer.MAX_VALUE : record.weight; + return weight >= Integer.MAX_VALUE ? Integer.MAX_VALUE : (int) weight; + } + + private void onRemoval(@Nullable K key, @Nullable V value, RemovalCause cause) { + if (key == null) { + return; + } + if (!weightBounded && !generationFencedRefresh) { + return; + } + if (closed.get()) { + return; + } + // Replacement transfers the existing reservation to the newly published generation. A + // soft-value collection instead reports a null value with COLLECTED and must release it. + if (cause == RemovalCause.REPLACED) { + return; + } + if (Thread.holdsLock(admissionLock)) { + // Other removals have already removed the Caffeine mapping and can release their owner + // inline. A stale callback cannot release a replacement while its mapping is visible. + if (data.asMap().get(key) == null) { + if (weightBounded) { + ReservationRecord record = reservations.get(key); + if (record != null) { + releaseReservation(key, record.generation); + } + } else { + RefreshRecord record = refreshRecords.get(key); + if (record != null) { + releaseRefreshRecord(key, record.generation); + } + } + } + return; + } + beforeRemovalOwnerSnapshotForTest(key); + long ownerGeneration = currentOwnerGeneration(key); + if (ownerGeneration >= 0L) { + beforeRemovalReleaseForTest(key); + if (closed.get()) { + return; + } + pendingRemovalGenerations.merge(key, ownerGeneration, Math::max); + if (closed.get()) { + pendingRemovalGenerations.remove(key, ownerGeneration); + return; + } + scheduleRemovalCleanup(); + } + } + + private void scheduleRemovalCleanup() { + if (closed.get()) { + return; + } + if (removalCleanupScheduled.compareAndSet(false, true)) { + try { + REMOVAL_CLEANUP_EXECUTOR.execute(this::drainRemovalCleanups); + } catch (RejectedExecutionException e) { + removalCleanupScheduled.set(false); + LOG.warn("Failed to schedule removal cleanup for external metadata cache entry {}", name, e); + } + } + } + + private void drainRemovalCleanups() { + try { + int processed = 0; + for (Map.Entry cleanup : pendingRemovalGenerations.entrySet()) { + if (processed++ >= REMOVAL_CLEANUP_BATCH_SIZE) { + break; + } + K key = cleanup.getKey(); + long generation = cleanup.getValue(); + // Claim before cleanup. If the same generation is reported again while cleanup + // runs, its notification creates a new pending item instead of being lost when + // this worker finishes. + if (!pendingRemovalGenerations.remove(key, generation)) { + continue; + } + try { + cleanupRemovedReservation(key, generation); + } catch (RuntimeException e) { + // Restore the generation for retry. The finally block requeues one bounded + // drain instead of permanently wedging this entry's scheduled flag. + pendingRemovalGenerations.merge(key, generation, Math::max); + LOG.warn("Failed to clean a removal reservation for external metadata cache entry {}", + name, e); + } + } + } finally { + removalCleanupScheduled.set(false); + if (!closed.get() && !pendingRemovalGenerations.isEmpty()) { + // One bounded task per turn prevents a hot entry from monopolizing the process-wide + // cleanup executor; a later task is queued behind already scheduled catalogs. + scheduleRemovalCleanup(); + } + } + } + + private void cleanupRemovedReservation(K key, long expectedReservationGeneration) { + beforeRemovalCleanupLockForTest(key); + synchronized (admissionLock) { + if (weightBounded) { + ReservationRecord record = reservations.get(key); + if (record != null && record.generation == expectedReservationGeneration + && data.asMap().get(key) == null + && reservations.remove(key, record)) { + record.reservation.release(); + } + } else { + RefreshRecord record = refreshRecords.get(key); + if (record != null && record.generation == expectedReservationGeneration + && data.asMap().get(key) == null) { + refreshRecords.remove(key, record); + } + } + } + afterRemovalCleanupForTest(key); + } + + private void releaseReservation(K key, long expectedGeneration) { + ReservationRecord record = reservations.get(key); + if (record != null && record.generation == expectedGeneration && reservations.remove(key, record)) { + record.reservation.release(); + } + } + + private void releaseRefreshRecord(K key, long expectedGeneration) { + RefreshRecord record = refreshRecords.get(key); + if (record != null && record.generation == expectedGeneration) { + refreshRecords.remove(key, record); + } + } + + private long currentOwnerGeneration(K key) { + if (weightBounded) { + ReservationRecord record = reservations.get(key); + return record == null ? -1L : record.generation; + } + RefreshRecord record = refreshRecords.get(key); + return record == null ? -1L : record.generation; + } + + private void putNonWeightedValue(K key, V value) { + RefreshRecord previous = refreshRecords.get(key); + RefreshRecord next = publishRefreshRecord(key); + try { + beforeNonWeightedCachePutForTest(key, value); + data.put(key, value); + if (next != null && refreshRecords.get(key) == next && data.asMap().get(key) == value) { + next.published = true; + } + } catch (RuntimeException | Error e) { + if (next != null) { + if (previous == null) { + refreshRecords.remove(key, next); + } else { + refreshRecords.replace(key, next, previous); + } + } + throw e; + } + } + + @Nullable + private RefreshRecord publishRefreshRecord(K key) { + if (!generationFencedRefresh) { + return null; + } + RefreshRecord record = new RefreshRecord(nextReservationGeneration()); + refreshRecords.put(key, record); + return record; + } + + private long nextReservationGeneration() { + return reservationGeneration.incrementAndGet(); + } + + private void rejectWeight(String reason) { + weightAdmissionRejectedCount.incrementAndGet(); + lastWeightRejectReason.set(reason == null || reason.isEmpty() ? "unknown" : reason); + } + + private void maybeRefreshManagedValue(K key, V currentValue) { + if (closed.get() || !generationFencedRefresh || loader == null) { + return; + } + if (!weightBounded) { + maybeRefreshNonWeightedValue(key, currentValue); + return; + } + ReservationRecord record = reservations.get(key); + long refreshNanos = TimeUnit.MINUTES.toNanos(Config.external_cache_refresh_time_minutes); + if (record == null || !record.published || data.asMap().get(key) != currentValue || refreshNanos <= 0 + || System.nanoTime() - record.writeNanos < refreshNanos + || refreshesInFlight.putIfAbsent(key, Boolean.TRUE) != null) { + return; + } + submitWeightedRefresh(key, record.generation, beginKeyMutation(key)); + } + + private void maybeRefreshNonWeightedValue(K key, V currentValue) { + RefreshRecord record = refreshRecords.get(key); + long refreshNanos = TimeUnit.MINUTES.toNanos(Config.external_cache_refresh_time_minutes); + if (record == null || !record.published || data.asMap().get(key) != currentValue || refreshNanos <= 0 + || System.nanoTime() - record.writeNanos < refreshNanos + || refreshesInFlight.putIfAbsent(key, Boolean.TRUE) != null) { + return; + } + submitNonWeightedRefresh(key, record.generation, beginKeyMutation(key)); + } + + private void submitNonWeightedRefresh( + K key, long expectedRefreshGeneration, KeyMutationToken expectedMutation) { + try { + refreshExecutor.execute(() -> { + try { + if (!isRefreshRecordCurrent(key, expectedRefreshGeneration, expectedMutation)) { + return; + } + V refreshed = loadAndTrack(key, this::applyDefaultLoader); + if (refreshed == null) { + return; + } + synchronized (admissionLock) { + if (!isRefreshRecordCurrent(key, expectedRefreshGeneration, expectedMutation)) { + return; + } + advanceKeyMutation(key); + putNonWeightedValue(key, refreshed); + } + } finally { + endKeyMutation(key, expectedMutation); + refreshesInFlight.remove(key); + } + }); + } catch (RejectedExecutionException e) { + endKeyMutation(key, expectedMutation); + refreshesInFlight.remove(key); + } + } + + private boolean isRefreshRecordCurrent( + K key, long expectedRefreshGeneration, KeyMutationToken expectedMutation) { + if (closed.get() || !isKeyMutationCurrent(key, expectedMutation)) { + return false; + } + RefreshRecord record = refreshRecords.get(key); + return record != null && record.generation == expectedRefreshGeneration + && record.published && data.asMap().get(key) != null; + } + + private void submitWeightedRefresh( + K key, long expectedReservationGeneration, KeyMutationToken expectedMutation) { + try { + refreshExecutor.execute(() -> { + try { + if (!isReservationCurrent(key, expectedReservationGeneration, expectedMutation)) { + return; + } + V refreshed = loadAndTrack(key, this::applyDefaultLoader); + if (refreshed != null && isKeyMutationCurrent(key, expectedMutation)) { + AdmissionResult result = admitWeightedValue( + key, refreshed, null, false, expectedMutation, + expectedReservationGeneration, true); + if (result == AdmissionResult.REJECTED) { + invalidateKeyIfReservationGeneration(key, expectedReservationGeneration); + } + } + } finally { + endKeyMutation(key, expectedMutation); + refreshesInFlight.remove(key); + } + }); + } catch (RejectedExecutionException e) { + endKeyMutation(key, expectedMutation); + refreshesInFlight.remove(key); + } + } + + private boolean isReservationCurrent( + K key, long expectedReservationGeneration, KeyMutationToken expectedMutation) { + if (closed.get() || !isKeyMutationCurrent(key, expectedMutation)) { + return false; + } + ReservationRecord record = reservations.get(key); + return record != null && record.generation == expectedReservationGeneration + && record.published && data.asMap().get(key) != null; + } + + private void invalidateKeyIfReservationGeneration(K key, long expectedReservationGeneration) { + synchronized (admissionLock) { + ReservationRecord record = reservations.get(key); + if (record == null || record.generation != expectedReservationGeneration) { + return; + } + V current = data.asMap().get(key); + if (current != null && data.asMap().remove(key, current)) { + advanceKeyMutation(key); + releaseReservation(key, expectedReservationGeneration); + invalidateCount.incrementAndGet(); + } + } } // Read the config dynamically so existing cache entries follow runtime config updates. private boolean isManualMissLoadEnabled() { - return Config.enable_external_meta_cache_manual_miss_load; + return weightBounded || generationFencedRefresh || Config.enable_external_meta_cache_manual_miss_load; } // Execute slow miss loads outside Caffeine's sync load path and suppress stale write-back after invalidation. private V getWithManualLoad(K key, Function loadFunction) { - if (!effectiveEnabled) { - // Bypass cache entirely when the entry is disabled so manual miss load does not relax disable semantics. + if (!effectiveEnabled || closed.get()) { + // Disabled and closed entries may still serve the caller, but can not retain the loaded value. return loadAndTrack(key, loadFunction); } V value = data.getIfPresent(key); if (value != null) { + maybeRefreshManagedValue(key, value); return value; } synchronized (loadLock(key)) { + if (!effectiveEnabled || closed.get()) { + return loadAndTrack(key, loadFunction); + } value = data.asMap().get(key); if (value != null) { + maybeRefreshManagedValue(key, value); return value; } - long generation = invalidateGeneration.get(); - V loaded = loadAndTrack(key, loadFunction); - if (generation != invalidateGeneration.get()) { - return loaded; - } + KeyMutationToken mutation = beginKeyMutation(key); + try { + V loaded = loadAndTrack(key, loadFunction); + if (!isKeyMutationCurrent(key, mutation)) { + return loaded; + } - // Keep null results uncached so manual miss load matches LoadingCache null-return behavior. - if (loaded == null) { - return null; - } + // Keep null results uncached so manual miss load matches LoadingCache null-return behavior. + if (loaded == null) { + return null; + } - // Leave a narrow hook for tests to pause exactly before the cache put race window. - beforeManualCachePutForTest(key, loaded); - data.put(key, loaded); - if (generation != invalidateGeneration.get()) { - removeLoadedValue(key, loaded); + // Leave a narrow hook for tests to pause exactly before the cache put race window. + beforeManualCachePutForTest(key, loaded); + if (closed.get() || !isKeyMutationCurrent(key, mutation)) { + return loaded; + } + if (weightBounded) { + admitWeightedValue(key, loaded, null, false, mutation, -1L, false); + } else { + synchronized (admissionLock) { + if (closed.get() || !isKeyMutationCurrent(key, mutation)) { + return loaded; + } + beforeNonWeightedManualCachePutForTest(key, loaded); + putNonWeightedValue(key, loaded); + } + } + return loaded; + } finally { + endKeyMutation(key, mutation); } - return loaded; } } - // Remove only the value loaded by the current request and keep newer replacements intact. - private void removeLoadedValue(K key, V loaded) { - data.asMap().computeIfPresent(key, (ignored, currentValue) -> currentValue == loaded ? null : currentValue); - } - // Map keys to a fixed lock stripe set to bound memory usage while keeping same-key deduplication. private Object loadLock(K key) { int hash = key == null ? 0 : key.hashCode(); @@ -264,6 +1063,67 @@ private Object loadLock(K key) { void beforeManualCachePutForTest(K key, V loaded) { } + // Let tests pause after the final generation check while holding the admission lock. + void beforeNonWeightedManualCachePutForTest(K key, V loaded) { + } + + // Called inside Caffeine's direct removal callback; tests use it to force the eviction-lock race. + void beforeRemovalReleaseForTest(K key) { + } + + // Let tests pause a callback after Caffeine removal but before reservation-owner lookup. + void beforeRemovalOwnerSnapshotForTest(K key) { + } + + // Called after reservation ownership is published and before Caffeine receives the value. + void beforeWeightedCachePutForTest(K key, V value) { + } + + // Called after refresh ownership is published and before Caffeine receives a count-bounded value. + void beforeNonWeightedCachePutForTest(K key, V value) { + } + + // Called after one asynchronous generation-conditional removal cleanup has finished. + void afterRemovalCleanupForTest(K key) { + } + + // Called immediately before the asynchronous drain tries to acquire admissionLock. + void beforeRemovalCleanupLockForTest(K key) { + } + + // Let tests establish admissionLock -> Caffeine eviction-lock ordering deterministically. + void beforeWeightedInvalidateAllForTest() { + } + + // Invoke the direct-listener branch while holding the mutation lock. + void notifyRemovalUnderAdmissionLockForTest(K key, V value, RemovalCause cause) { + synchronized (admissionLock) { + onRemoval(key, value, cause); + } + } + + // Enqueue the same generation-only refresh task without waiting for the production interval. + void triggerRefreshForTest(K key) { + V current = data.asMap().get(key); + if (current == null || refreshesInFlight.putIfAbsent(key, Boolean.TRUE) != null) { + return; + } + if (weightBounded) { + ReservationRecord record = reservations.get(key); + if (record != null && record.published) { + submitWeightedRefresh(key, record.generation, beginKeyMutation(key)); + return; + } + } else if (generationFencedRefresh) { + RefreshRecord record = refreshRecords.get(key); + if (record != null && record.published) { + submitNonWeightedRefresh(key, record.generation, beginKeyMutation(key)); + return; + } + } + refreshesInFlight.remove(key); + } + private V loadFromDefaultLoader(K key) { return loadAndTrack(key, this::applyDefaultLoader); } @@ -294,4 +1154,104 @@ private V loadAndTrack(K key, Function loadFunction) { throw e; } } + + private KeyMutationToken beginKeyMutation(K key) { + synchronized (admissionLock) { + KeyMutationState state = keyMutationStates.computeIfAbsent(key, ignored -> new KeyMutationState()); + state.inFlight++; + return new KeyMutationToken(state, state.generation, fullInvalidationGeneration.get()); + } + } + + private void advanceKeyMutation(K key) { + // Callers already serialize cache mutation with admissionLock. Keeping the helper lock-free + // prevents accidental deadlock if it is used by a listener reached under that same lock. + KeyMutationState state = keyMutationStates.get(key); + if (state != null) { + state.generation++; + } + } + + private boolean isKeyMutationCurrent(K key, KeyMutationToken token) { + return token.fullInvalidationGeneration == fullInvalidationGeneration.get() + && token.state == keyMutationStates.get(key) + && token.generation == token.state.generation; + } + + private void endKeyMutation(K key, KeyMutationToken token) { + synchronized (admissionLock) { + if (--token.state.inFlight == 0) { + keyMutationStates.remove(key, token.state); + } + } + } + + private static final class ReservationRecord { + private final long weight; + private final long writeNanos; + private final AdmissionReservation reservation; + private final long generation; + private volatile boolean published; + + private ReservationRecord(long weight, AdmissionReservation reservation, long generation) { + this.weight = weight; + this.reservation = reservation; + this.writeNanos = System.nanoTime(); + this.generation = generation; + } + } + + private static final class RefreshRecord { + private final long writeNanos; + private final long generation; + private volatile boolean published; + + private RefreshRecord(long generation) { + this.writeNanos = System.nanoTime(); + this.generation = generation; + } + } + + private static final class KeyMutationState { + private volatile long generation; + private int inFlight; + } + + private static final class KeyMutationToken { + private final KeyMutationState state; + private final long generation; + private final long fullInvalidationGeneration; + + private KeyMutationToken( + KeyMutationState state, long generation, long fullInvalidationGeneration) { + this.state = state; + this.generation = generation; + this.fullInvalidationGeneration = fullInvalidationGeneration; + } + } + + private static ReplaceResult toReplaceResult(AdmissionResult result) { + switch (result) { + case ADMITTED: + return ReplaceResult.REPLACED; + case NOT_CURRENT: + return ReplaceResult.NOT_CURRENT; + case REJECTED: + return ReplaceResult.REJECTED; + case DISABLED: + default: + return ReplaceResult.DISABLED; + } + } + + private static long saturatedAdd(long left, long right) { + return left > Long.MAX_VALUE - right ? Long.MAX_VALUE : left + right; + } + + private enum AdmissionResult { + ADMITTED, + NOT_CURRENT, + REJECTED, + DISABLED + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java index 1f48057a44fc40..5963f3d8aa7b86 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java @@ -101,10 +101,12 @@ public final class MetaCacheEntryDef { private final boolean autoRefresh; private final boolean contextualOnly; private final MetaCacheEntryInvalidation invalidation; + @Nullable + private final MetaCacheSizeEstimator sizeEstimator; private MetaCacheEntryDef(String name, Class keyType, Class valueType, @Nullable Function loader, CacheSpec defaultCacheSpec, boolean autoRefresh, boolean contextualOnly, - MetaCacheEntryInvalidation invalidation) { + MetaCacheEntryInvalidation invalidation, @Nullable MetaCacheSizeEstimator sizeEstimator) { this.name = Objects.requireNonNull(name, "entry name is required"); this.keyType = Objects.requireNonNull(keyType, "entry key type is required"); this.valueType = Objects.requireNonNull(valueType, "entry value type is required"); @@ -123,6 +125,7 @@ private MetaCacheEntryDef(String name, Class keyType, Class valueType, this.autoRefresh = autoRefresh; this.contextualOnly = contextualOnly; this.invalidation = Objects.requireNonNull(invalidation, "entry invalidation is required"); + this.sizeEstimator = sizeEstimator; } /** @@ -142,7 +145,7 @@ public static MetaCacheEntryDef of(String name, Class keyType, C public static MetaCacheEntryDef of(String name, Class keyType, Class valueType, Function loader, CacheSpec defaultCacheSpec, MetaCacheEntryInvalidation invalidation) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, true, false, - invalidation); + invalidation, null); } /** @@ -164,7 +167,7 @@ public static MetaCacheEntryDef of(String name, Class keyType, C Function loader, CacheSpec defaultCacheSpec, boolean autoRefresh, MetaCacheEntryInvalidation invalidation) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, false, - invalidation); + invalidation, null); } /** @@ -179,7 +182,14 @@ public static MetaCacheEntryDef contextualOnly( String name, Class keyType, Class valueType, CacheSpec defaultCacheSpec, MetaCacheEntryInvalidation invalidation) { return new MetaCacheEntryDef<>(name, keyType, valueType, null, defaultCacheSpec, false, true, - invalidation); + invalidation, null); + } + + /** Return a definition with a publication-time size estimator. */ + public MetaCacheEntryDef withSizeEstimator(MetaCacheSizeEstimator estimator) { + return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, + autoRefresh, contextualOnly, invalidation, + Objects.requireNonNull(estimator, "estimator")); } /** @@ -232,4 +242,9 @@ public boolean isContextualOnly() { public MetaCacheEntryInvalidation getInvalidation() { return invalidation; } + + @Nullable + public MetaCacheSizeEstimator getSizeEstimator() { + return sizeEstimator; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java index 495fd011083bb0..433cc027515873 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryStats.java @@ -51,6 +51,16 @@ public final class MetaCacheEntryStats { private final long lastLoadSuccessTimeMs; private final long lastLoadFailureTimeMs; private final String lastError; + private final boolean weightBounded; + private final long maxWeight; + private final long estimatedWeight; + private final long evictionWeight; + private final long weightAdmissionRejectedCount; + private final long catalogMaxWeight; + private final long catalogEstimatedWeight; + private final long globalMaxWeight; + private final long globalEstimatedWeight; + private final String lastWeightRejectReason; /** * Build an immutable stats snapshot. @@ -74,7 +84,17 @@ public MetaCacheEntryStats( long invalidateCount, long lastLoadSuccessTimeMs, long lastLoadFailureTimeMs, - String lastError) { + String lastError, + boolean weightBounded, + long maxWeight, + long estimatedWeight, + long evictionWeight, + long weightAdmissionRejectedCount, + long catalogMaxWeight, + long catalogEstimatedWeight, + long globalMaxWeight, + long globalEstimatedWeight, + String lastWeightRejectReason) { this.configEnabled = configEnabled; this.effectiveEnabled = effectiveEnabled; this.autoRefresh = autoRefresh; @@ -94,6 +114,16 @@ public MetaCacheEntryStats( this.lastLoadSuccessTimeMs = lastLoadSuccessTimeMs; this.lastLoadFailureTimeMs = lastLoadFailureTimeMs; this.lastError = Objects.requireNonNull(lastError, "lastError"); + this.weightBounded = weightBounded; + this.maxWeight = maxWeight; + this.estimatedWeight = estimatedWeight; + this.evictionWeight = evictionWeight; + this.weightAdmissionRejectedCount = weightAdmissionRejectedCount; + this.catalogMaxWeight = catalogMaxWeight; + this.catalogEstimatedWeight = catalogEstimatedWeight; + this.globalMaxWeight = globalMaxWeight; + this.globalEstimatedWeight = globalEstimatedWeight; + this.lastWeightRejectReason = Objects.requireNonNull(lastWeightRejectReason, "lastWeightRejectReason"); } public boolean isConfigEnabled() { @@ -186,4 +216,44 @@ public long getLastLoadFailureTimeMs() { public String getLastError() { return lastError; } + + public boolean isWeightBounded() { + return weightBounded; + } + + public long getMaxWeight() { + return maxWeight; + } + + public long getEstimatedWeight() { + return estimatedWeight; + } + + public long getEvictionWeight() { + return evictionWeight; + } + + public long getWeightAdmissionRejectedCount() { + return weightAdmissionRejectedCount; + } + + public long getCatalogMaxWeight() { + return catalogMaxWeight; + } + + public long getCatalogEstimatedWeight() { + return catalogEstimatedWeight; + } + + public long getGlobalMaxWeight() { + return globalMaxWeight; + } + + public long getGlobalEstimatedWeight() { + return globalEstimatedWeight; + } + + public String getLastWeightRejectReason() { + return lastWeightRejectReason; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimate.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimate.java new file mode 100644 index 00000000000000..d44702ce559212 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimate.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import java.util.Objects; + +/** + * Immutable result value returned by {@link MetaCacheSizeEstimator}; this class is not an + * estimator implementation. An incomplete result carries no usable byte count and must fail + * cache admission closed. + */ +public final class MetaCacheSizeEstimate { + private final long bytes; + private final boolean complete; + private final String incompleteReason; + + private MetaCacheSizeEstimate(long bytes, boolean complete, String incompleteReason) { + this.bytes = bytes; + this.complete = complete; + this.incompleteReason = Objects.requireNonNull(incompleteReason, "incompleteReason"); + } + + public static MetaCacheSizeEstimate complete(long bytes) { + if (bytes < 0) { + throw new IllegalArgumentException("cache size estimate can not be negative: " + bytes); + } + return new MetaCacheSizeEstimate(bytes, true, ""); + } + + public static MetaCacheSizeEstimate incomplete(String reason) { + String safeReason = Objects.requireNonNull(reason, "reason").trim(); + if (safeReason.isEmpty()) { + throw new IllegalArgumentException("incomplete cache size estimate requires a reason"); + } + return new MetaCacheSizeEstimate(0L, false, safeReason); + } + + public long getBytes() { + return bytes; + } + + public boolean isComplete() { + return complete; + } + + public String getIncompleteReason() { + return incompleteReason; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java new file mode 100644 index 00000000000000..f78fd5bc734b39 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java @@ -0,0 +1,46 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import java.util.Objects; +import java.util.function.Supplier; + +/** + * Supplies the admission weight of one key/value pair. + * + *

    The callback runs after load and before admission. Implementations must use already available + * shape counters and constant-time collection sizes; they must not walk object graphs, perform IO, + * materialize lazy SDK state, or copy payloads. Caffeine's weigher reads only the admitted + * reservation record, so cache hits and eviction remain O(1). + */ +@FunctionalInterface +public interface MetaCacheSizeEstimator { + MetaCacheSizeEstimate estimate(K key, V value); + + /** Convert preparation failures into fail-closed incomplete estimates. */ + static MetaCacheSizeEstimate estimateSafely( + String failureReason, Supplier estimation) { + Objects.requireNonNull(failureReason, "failureReason"); + Objects.requireNonNull(estimation, "estimation"); + try { + return Objects.requireNonNull(estimation.get(), "size estimate"); + } catch (RuntimeException e) { + return MetaCacheSizeEstimate.incomplete(failureReason + ":" + e.getClass().getName()); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java new file mode 100644 index 00000000000000..7869ac5993f48e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java @@ -0,0 +1,70 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.apache.doris.datasource.NameMapping; + +/** Constant-time helpers for conservative external metadata cache weights. */ +public final class MetaCacheWeightUtils { + private static final long STRING_BASE_BYTES = 40L; + private static final long STRING_BYTES_PER_CHARACTER = 2L; + private static final long NAME_MAPPING_BASE_BYTES = 64L; + + private MetaCacheWeightUtils() { + } + + /** + * Estimate a String without inspecting its contents. Two bytes per character deliberately + * avoids depending on CompactStrings or VM-private layout details. + */ + public static long estimatedStringBytes(String value) { + return estimatedCharSequenceBytes(value); + } + + /** Estimate retained character data without materializing a String copy. */ + public static long estimatedCharSequenceBytes(CharSequence value) { + return value == null ? 0L : saturatedAdd( + STRING_BASE_BYTES, saturatedMultiply(value.length(), STRING_BYTES_PER_CHARACTER)); + } + + /** Estimate the fixed set of names retained by a cache key. */ + public static long estimatedNameMappingBytes(NameMapping nameMapping) { + if (nameMapping == null) { + return 0L; + } + long bytes = NAME_MAPPING_BASE_BYTES; + bytes = saturatedAdd(bytes, estimatedStringBytes(nameMapping.getLocalDbName())); + bytes = saturatedAdd(bytes, estimatedStringBytes(nameMapping.getLocalTblName())); + bytes = saturatedAdd(bytes, estimatedStringBytes(nameMapping.getRemoteDbName())); + return saturatedAdd(bytes, estimatedStringBytes(nameMapping.getRemoteTblName())); + } + + public static long saturatedAdd(long left, long right) { + if (left < 0L || right < 0L || Long.MAX_VALUE - left < right) { + return Long.MAX_VALUE; + } + return left + right; + } + + public static long saturatedMultiply(long left, long right) { + if (left < 0L || right < 0L || (left != 0L && right > Long.MAX_VALUE / left)) { + return Long.MAX_VALUE; + } + return left * right; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java new file mode 100644 index 00000000000000..b5a6178538a9c4 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java @@ -0,0 +1,193 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + +import org.apache.paimon.privilege.PrivilegedFileStoreTable; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.FallbackReadFileStoreTable; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.MapType; +import org.apache.paimon.types.MultisetType; +import org.apache.paimon.types.RowType; + +import java.util.List; +import java.util.Map; + +/** Constant-time retained-weight formula for Paimon snapshot projections. */ +final class PaimonCacheSizeEstimator { + private static final long KEY_BASE_BYTES = 128L; + private static final long SNAPSHOT_BASE_BYTES = 4L * 1024L; + private static final long TABLE_BASE_BYTES = 16L * 1024L; + private static final long TABLE_FIELD_BYTES = 3584L; + private static final long TABLE_OPTION_BYTES = 256L; + private static final long TABLE_KEY_BYTES = 128L; + private static final long NESTED_FIELD_BYTES = 512L; + private static final long PARTITION_BYTES = 1280L; + private static final long PARTITION_ITEM_BYTES = 1024L; + private static final long WRAPPER_BYTES = 512L; + + private PaimonCacheSizeEstimator() { + } + + static MetaCacheSizeEstimate estimateSnapshotEntry( + PaimonSnapshotEntryKey key, PaimonSnapshotCacheValue value) { + Table table = value.getSnapshot().getTable(); + if (!isSupportedTable(table)) { + return MetaCacheSizeEstimate.incomplete("unsupported_paimon_table:" + + (table == null ? "null" : table.getClass().getName())); + } + + long bytes = MetaCacheWeightUtils.saturatedAdd( + KEY_BASE_BYTES, MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SNAPSHOT_BASE_BYTES); + bytes = addCount(bytes, value.getPartitionInfo().getNameToPartition().size(), PARTITION_BYTES); + bytes = addCount(bytes, value.getPartitionInfo().getNameToPartitionItem().size(), PARTITION_ITEM_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, value.getPartitionInfo().getRetainedPayloadBytes()); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedTablePayloadBytes()); + return MetaCacheSizeEstimate.complete( + MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table))); + } + + private static boolean isSupportedTable(Table table) { + if (table instanceof PrivilegedFileStoreTable) { + return isSupportedTable(((PrivilegedFileStoreTable) table).wrapped()); + } + if (table instanceof FallbackReadFileStoreTable) { + FallbackReadFileStoreTable fallback = (FallbackReadFileStoreTable) table; + return isSupportedTable(fallback.wrapped()) && isSupportedTable(fallback.other()); + } + if (!(table instanceof FileStoreTable)) { + return false; + } + String className = table.getClass().getName(); + return "org.apache.paimon.table.AppendOnlyFileStoreTable".equals(className) + || "org.apache.paimon.table.PrimaryKeyFileStoreTable".equals(className); + } + + /** Uses TableSchema cardinalities only and deliberately never calls FileStoreTable.store(). */ + private static long estimateTable(Table table) { + if (table instanceof PrivilegedFileStoreTable) { + return MetaCacheWeightUtils.saturatedAdd(WRAPPER_BYTES, + estimateTable(((PrivilegedFileStoreTable) table).wrapped())); + } + if (table instanceof FallbackReadFileStoreTable) { + FallbackReadFileStoreTable fallback = (FallbackReadFileStoreTable) table; + long bytes = MetaCacheWeightUtils.saturatedAdd(WRAPPER_BYTES, estimateTable(fallback.wrapped())); + return MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(fallback.other())); + } + + FileStoreTable fileStoreTable = (FileStoreTable) table; + TableSchema schema = fileStoreTable.schema(); + long bytes = TABLE_BASE_BYTES; + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(table.name())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(fileStoreTable.location().toString())); + bytes = addCount(bytes, schema.fields().size(), TABLE_FIELD_BYTES); + bytes = addCount(bytes, schema.options().size(), TABLE_OPTION_BYTES); + bytes = addCount(bytes, schema.partitionKeys().size(), TABLE_KEY_BYTES); + bytes = addCount(bytes, schema.primaryKeys().size(), TABLE_KEY_BYTES); + return addCount(bytes, schema.bucketKeys().size(), TABLE_KEY_BYTES); + } + + /** + * Captures skew-sensitive schema text once when the snapshot cache value is constructed. + * All collections are already materialized in TableSchema; this never opens the table store. + */ + static long retainedTablePayloadBytes(Table table) { + if (table instanceof PrivilegedFileStoreTable) { + return retainedTablePayloadBytes(((PrivilegedFileStoreTable) table).wrapped()); + } + if (table instanceof FallbackReadFileStoreTable) { + FallbackReadFileStoreTable fallback = (FallbackReadFileStoreTable) table; + return MetaCacheWeightUtils.saturatedAdd( + retainedTablePayloadBytes(fallback.wrapped()), + retainedTablePayloadBytes(fallback.other())); + } + if (!(table instanceof FileStoreTable)) { + return 0L; + } + + TableSchema schema = ((FileStoreTable) table).schema(); + if (schema == null) { + return 0L; + } + long bytes = addString(0L, schema.comment()); + for (DataField field : schema.fields()) { + bytes = addFieldPayload(bytes, field, false); + } + for (Map.Entry option : schema.options().entrySet()) { + bytes = addString(bytes, option.getKey()); + bytes = addString(bytes, option.getValue()); + } + bytes = addStrings(bytes, schema.partitionKeys()); + bytes = addStrings(bytes, schema.primaryKeys()); + return addStrings(bytes, schema.bucketKeys()); + } + + private static long addStrings(long bytes, List values) { + for (String value : values) { + bytes = addString(bytes, value); + } + return bytes; + } + + private static long addFieldPayload(long bytes, DataField field, boolean nested) { + if (nested) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, NESTED_FIELD_BYTES); + } + bytes = addString(bytes, field.name()); + bytes = addString(bytes, field.description()); + bytes = addString(bytes, field.defaultValue()); + return addTypePayload(bytes, field.type()); + } + + private static long addTypePayload(long bytes, DataType type) { + if (type instanceof RowType) { + for (DataField field : ((RowType) type).getFields()) { + bytes = addFieldPayload(bytes, field, true); + } + } else if (type instanceof ArrayType) { + bytes = addTypePayload(bytes, ((ArrayType) type).getElementType()); + } else if (type instanceof MapType) { + bytes = addTypePayload(bytes, ((MapType) type).getKeyType()); + bytes = addTypePayload(bytes, ((MapType) type).getValueType()); + } else if (type instanceof MultisetType) { + bytes = addTypePayload(bytes, ((MultisetType) type).getElementType()); + } + return bytes; + } + + private static long addString(long bytes, String value) { + return MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedStringBytes(value)); + } + + private static long addCount(long bytes, long count, long bytesPerItem) { + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply(count, bytesPerItem)); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index cde2bbb31efd18..ee5fdeebb2f793 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -23,6 +23,8 @@ import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.metacache.AbstractExternalMetaCache; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; +import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryDef; import org.apache.doris.datasource.metacache.MetaCacheEntryInvalidation; import org.apache.doris.datasource.metacache.paimon.PaimonLatestSnapshotProjectionLoader; @@ -40,36 +42,48 @@ *

    Registered entries: *

      *
    • {@code table}: loaded Paimon table handle per table mapping
      • + *
    • {@code snapshot}: immutable partition projection keyed by a captured snapshot/schema fence
      • *
    • {@code schema}: schema cache keyed by table identity + schema id
      • *
    * - *

    Latest snapshot metadata is modeled as a runtime projection memoized inside the table cache - * value instead of as an independent cache entry. + *

    The latest main-branch snapshot is captured once as a fence and loaded through an independent + * contextual entry. Branch/tag/options projections remain statement-local and are not aliased to + * this main-snapshot key. * *

    Invalidation behavior: *

      - *
    • db/table invalidation clears table and schema entries by matching local names
      • + *
    • db/table invalidation clears table, snapshot and schema entries by matching local names
      • *
    • partition-level invalidation falls back to table-level invalidation
      • *
    */ public class PaimonExternalMetaCache extends AbstractExternalMetaCache { public static final String ENGINE = "paimon"; public static final String ENTRY_TABLE = "table"; + public static final String ENTRY_SNAPSHOT = "snapshot"; public static final String ENTRY_SCHEMA = "schema"; private final EntryHandle tableEntry; + private final EntryHandle snapshotEntry; private final EntryHandle schemaEntry; private final PaimonTableLoader tableLoader; private final PaimonLatestSnapshotProjectionLoader latestSnapshotProjectionLoader; public PaimonExternalMetaCache(ExecutorService refreshExecutor) { - super(ENGINE, refreshExecutor); + this(refreshExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); + } + + public PaimonExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { + super(ENGINE, refreshExecutor, budgetManager); tableLoader = new PaimonTableLoader(); latestSnapshotProjectionLoader = new PaimonLatestSnapshotProjectionLoader( new PaimonPartitionInfoLoader(), this::getPaimonSchemaCacheValue); tableEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_TABLE, NameMapping.class, PaimonTableCacheValue.class, this::loadTableCacheValue, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping))); + snapshotEntry = registerEntry(MetaCacheEntryDef.contextualOnly(ENTRY_SNAPSHOT, + PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class, defaultEntryCacheSpec(), + MetaCacheEntryInvalidation.forNameMapping(PaimonSnapshotEntryKey::getNameMapping)) + .withSizeEstimator((key, value) -> value.prepareForCachePublication(key))); schemaEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_SCHEMA, PaimonSchemaCacheKey.class, SchemaCacheValue.class, this::loadSchemaCacheValue, defaultSchemaCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(PaimonSchemaCacheKey::getNameMapping))); @@ -86,7 +100,13 @@ public Table getPaimonTable(NameMapping nameMapping) { public PaimonSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - return tableEntry.get(nameMapping.getCtlId()).get(nameMapping).getLatestSnapshotCacheValue(); + PaimonTableCacheValue tableValue = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + PaimonSnapshot fence = tableValue.getLatestSnapshotFence().getSnapshot(); + PaimonSnapshotEntryKey key = PaimonSnapshotEntryKey.of( + nameMapping, fence, tableValue.getGeneration()); + MetaCacheEntry entry = + snapshotEntry.get(nameMapping.getCtlId()); + return entry.get(key, ignored -> latestSnapshotProjectionLoader.loadAtFence(nameMapping, fence)); } public PaimonSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTable, Table effectiveTable) { @@ -95,8 +115,7 @@ public PaimonSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTable, public PaimonSnapshotCacheValue loadLatestSnapshotFence(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - Table table = tableEntry.get(nameMapping.getCtlId()).get(nameMapping).getPaimonTable(); - return latestSnapshotProjectionLoader.loadFence(nameMapping, table); + return tableEntry.get(nameMapping.getCtlId()).get(nameMapping).getLatestSnapshotFence(); } public PaimonSnapshotCacheValue loadSnapshotAtFence( @@ -119,8 +138,8 @@ public PaimonSchemaCacheValue getPaimonSchemaCacheValue(NameMapping nameMapping, private PaimonTableCacheValue loadTableCacheValue(NameMapping nameMapping) { Table paimonTable = tableLoader.load(nameMapping); - return new PaimonTableCacheValue(paimonTable, - () -> latestSnapshotProjectionLoader.load(nameMapping, paimonTable)); + PaimonSnapshotCacheValue fence = latestSnapshotProjectionLoader.loadFence(nameMapping, paimonTable); + return new PaimonTableCacheValue(paimonTable, fence); } private SchemaCacheValue loadSchemaCacheValue(PaimonSchemaCacheKey key) { @@ -133,6 +152,11 @@ private SchemaCacheValue loadSchemaCacheValue(PaimonSchemaCacheKey key) { @Override protected Map catalogPropertyCompatibilityMap() { - return singleCompatibilityMap(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND, ENTRY_SCHEMA); + Map compatibility = new java.util.HashMap<>( + singleCompatibilityMap(ExternalCatalog.SCHEMA_CACHE_TTL_SECOND, ENTRY_SCHEMA)); + compatibility.put("meta.cache.paimon.table.enable", "meta.cache.paimon.snapshot.enable"); + compatibility.put("meta.cache.paimon.table.ttl-second", "meta.cache.paimon.snapshot.ttl-second"); + compatibility.put("meta.cache.paimon.table.capacity", "meta.cache.paimon.snapshot.capacity"); + return compatibility; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java index 207810b66f5a68..004adc197061b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.paimon; import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.paimon.partition.Partition; @@ -49,18 +50,26 @@ public enum PruningStatus { private final PruningStatus pruningStatus; private final Map nameToPartitionItem; private final Map nameToPartition; + private final long retainedPayloadBytes; private PaimonPartitionInfo(PruningStatus pruningStatus) { this.pruningStatus = pruningStatus; this.nameToPartitionItem = Collections.emptyMap(); this.nameToPartition = Collections.emptyMap(); + this.retainedPayloadBytes = 0L; } public PaimonPartitionInfo(Map nameToPartitionItem, Map nameToPartition) { + this(nameToPartitionItem, nameToPartition, retainedPayloadBytes(nameToPartition)); + } + + public PaimonPartitionInfo(Map nameToPartitionItem, + Map nameToPartition, long retainedPayloadBytes) { this.pruningStatus = PruningStatus.PRUNABLE; this.nameToPartitionItem = nameToPartitionItem; this.nameToPartition = nameToPartition; + this.retainedPayloadBytes = retainedPayloadBytes; } public Map getNameToPartitionItem() { @@ -74,4 +83,47 @@ public Map getNameToPartition() { public PruningStatus getPruningStatus() { return pruningStatus; } + + public long getRetainedPayloadBytes() { + return retainedPayloadBytes; + } + + static long addRetainedStringPayload(long bytes, String value) { + return addString(bytes, value); + } + + private static long retainedPayloadBytes(Map partitions) { + if (partitions == null) { + return 0L; + } + long bytes = 0L; + for (Map.Entry entry : partitions.entrySet()) { + bytes = addString(bytes, entry.getKey()); + Partition partition = entry.getValue(); + if (partition == null) { + continue; + } + bytes = addStrings(bytes, partition.spec()); + bytes = addString(bytes, partition.createdBy()); + bytes = addString(bytes, partition.updatedBy()); + bytes = addStrings(bytes, partition.options()); + } + return bytes; + } + + private static long addStrings(long bytes, Map values) { + if (values == null) { + return bytes; + } + for (Map.Entry entry : values.entrySet()) { + bytes = addString(bytes, entry.getKey()); + bytes = addString(bytes, entry.getValue()); + } + return bytes; + } + + private static long addString(long bytes, String value) { + return MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedStringBytes(value)); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java index 37be7c6a5f3585..630db395afbfa3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java @@ -17,11 +17,16 @@ package org.apache.doris.datasource.paimon; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; + public class PaimonSnapshotCacheValue { private final PaimonPartitionInfo partitionInfo; private final PaimonSnapshot snapshot; private final boolean schemaFromSnapshotTable; + private long retainedTablePayloadBytes; + private MetaCacheSizeEstimate sizeEstimate; public PaimonSnapshotCacheValue(PaimonPartitionInfo partitionInfo, PaimonSnapshot snapshot) { this(partitionInfo, snapshot, false); @@ -45,4 +50,22 @@ public PaimonSnapshot getSnapshot() { public boolean isSchemaFromSnapshotTable() { return schemaFromSnapshotTable; } + + long getRetainedTablePayloadBytes() { + return retainedTablePayloadBytes; + } + + MetaCacheSizeEstimate prepareForCachePublication(PaimonSnapshotEntryKey key) { + if (sizeEstimate == null) { + retainedTablePayloadBytes = PaimonCacheSizeEstimator.retainedTablePayloadBytes(snapshot.getTable()); + sizeEstimate = MetaCacheSizeEstimator.estimateSafely("paimon_snapshot_preparation_failed", + () -> PaimonCacheSizeEstimator.estimateSnapshotEntry(key, this)); + } + return sizeEstimate; + } + + public MetaCacheSizeEstimate getSizeEstimate() { + return sizeEstimate == null + ? MetaCacheSizeEstimate.incomplete("not_prepared") : sizeEstimate; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotEntryKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotEntryKey.java new file mode 100644 index 00000000000000..d820d3c15e992b --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotEntryKey.java @@ -0,0 +1,80 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.datasource.NameMapping; + +import java.util.Objects; + +/** Stable identity for a Paimon projection hydrated from one captured snapshot/schema fence. */ +public final class PaimonSnapshotEntryKey { + private final NameMapping nameMapping; + private final long snapshotId; + private final long schemaId; + private final long tableGeneration; + + public PaimonSnapshotEntryKey( + NameMapping nameMapping, long snapshotId, long schemaId, long tableGeneration) { + this.nameMapping = Objects.requireNonNull(nameMapping, "nameMapping can not be null"); + this.snapshotId = snapshotId; + this.schemaId = schemaId; + this.tableGeneration = tableGeneration; + } + + public static PaimonSnapshotEntryKey of( + NameMapping nameMapping, PaimonSnapshot fence, long tableGeneration) { + return new PaimonSnapshotEntryKey( + nameMapping, fence.getSnapshotId(), fence.getSchemaId(), tableGeneration); + } + + public NameMapping getNameMapping() { + return nameMapping; + } + + public long getSnapshotId() { + return snapshotId; + } + + public long getSchemaId() { + return schemaId; + } + + public long getTableGeneration() { + return tableGeneration; + } + + @Override + public boolean equals(Object object) { + if (this == object) { + return true; + } + if (!(object instanceof PaimonSnapshotEntryKey)) { + return false; + } + PaimonSnapshotEntryKey that = (PaimonSnapshotEntryKey) object; + return snapshotId == that.snapshotId + && schemaId == that.schemaId + && tableGeneration == that.tableGeneration + && nameMapping.equals(that.nameMapping); + } + + @Override + public int hashCode() { + return Objects.hash(nameMapping, snapshotId, schemaId, tableGeneration); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java index 7539f28d770bf6..e5fba223ea92b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java @@ -17,28 +17,38 @@ package org.apache.doris.datasource.paimon; -import com.google.common.base.Suppliers; import org.apache.paimon.table.Table; -import java.util.function.Supplier; +import java.util.Objects; +import java.util.concurrent.atomic.AtomicLong; /** - * Cache value for Paimon table metadata and its latest runtime snapshot projection. + * Cache value for a Paimon table handle. Snapshot projections use a separate cache entry so this + * value cannot grow after admission through a memoized supplier. */ public class PaimonTableCacheValue { + private static final AtomicLong NEXT_GENERATION = new AtomicLong(); + private final Table paimonTable; - private final Supplier latestSnapshotCacheValue; + private final PaimonSnapshotCacheValue latestSnapshotFence; + private final long generation; - public PaimonTableCacheValue(Table paimonTable, Supplier latestSnapshotCacheValue) { + public PaimonTableCacheValue(Table paimonTable, PaimonSnapshotCacheValue latestSnapshotFence) { this.paimonTable = paimonTable; - this.latestSnapshotCacheValue = Suppliers.memoize(latestSnapshotCacheValue::get); + this.latestSnapshotFence = Objects.requireNonNull( + latestSnapshotFence, "latestSnapshotFence can not be null"); + this.generation = NEXT_GENERATION.incrementAndGet(); } public Table getPaimonTable() { return paimonTable; } - public PaimonSnapshotCacheValue getLatestSnapshotCacheValue() { - return latestSnapshotCacheValue.get(); + public long getGeneration() { + return generation; + } + + public PaimonSnapshotCacheValue getLatestSnapshotFence() { + return latestSnapshotFence; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index eaa59b28c2f4c1..a1eaf3100cdc28 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -176,6 +176,7 @@ public static PaimonPartitionInfo generatePartitionInfo(Table table, List candidates = Lists.newArrayListWithExpectedSize(partitionEntries.size()); Map> displayNameToTypedSpec = Maps.newHashMap(); + long retainedPayloadBytes = 0L; for (PartitionEntry partitionEntry : partitionEntries) { Map typedSpec = getPartitionInfoMap( @@ -193,6 +194,10 @@ public static PaimonPartitionInfo generatePartitionInfo(Table table, List previousTypedSpec = displayNameToTypedSpec.putIfAbsent( displayName, orderedTypedSpec); if (previousTypedSpec != null) { @@ -247,7 +254,7 @@ public static PaimonPartitionInfo generatePartitionInfo(Table table, List properties = new HashMap<>(); + + properties.put("meta.cache.hvie.partition_values.capacity", "10"); + Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.prepareCatalogByEngine(1L, "hive", properties)); + properties.clear(); + + properties.put("meta.cache.hms.partition_values.capacity", "10"); + Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.prepareCatalogByEngine(1L, "hive", properties)); + properties.clear(); + + properties.put("meta.cache.hive.partiton_values.capacity", "10"); + Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.prepareCatalogByEngine(1L, "hive", properties)); + } + + @Test + public void testCatalogCachePropertiesRejectEngineNotRoutedByCatalogType() { + ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); + Map properties = Collections.singletonMap( + "meta.cache.hive.partition_values.capacity", "10"); + PaimonExternalCatalog catalog = new PaimonExternalCatalog( + 1L, "paimon", null, Collections.emptyMap(), ""); + + IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.validateCatalogCacheProperties(catalog, properties)); + Assert.assertTrue(exception.getMessage().contains("not supported by catalog type")); + } + @Test public void testRouteByCatalogType() { ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); @@ -110,6 +149,60 @@ public void testPrepareCatalogByEngineSkipsMissingCatalog() throws Exception { Assert.assertEquals(0, hive.initCatalogCalls); } + @Test + public void testPreparedEngineUsesLockFreeFastPath() throws Exception { + RecordingExternalMetaCache hive = new RecordingExternalMetaCache( + "hive", Collections.singletonList("hms"), catalog -> true); + ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive); + long catalogId = 12L; + HMSExternalCatalog catalog = new HMSExternalCatalog( + catalogId, "hms", null, Collections.emptyMap(), ""); + mockCurrentCatalog(catalogId, catalog); + + metaCacheMgr.prepareCatalogByEngine(catalogId, "hive"); + metaCacheMgr.prepareCatalogByEngine(catalogId, "hive"); + + Assert.assertEquals(1, hive.initCatalogCalls); + } + + @Test + public void testCatalogRemovalFencesInFlightFirstInitialization() throws Exception { + CountDownLatch initializationEntered = new CountDownLatch(1); + CountDownLatch releaseInitialization = new CountDownLatch(1); + BlockingRecordingExternalMetaCache hive = new BlockingRecordingExternalMetaCache( + "hive", initializationEntered, releaseInitialization); + ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive); + ExecutorService workers = Executors.newFixedThreadPool(2); + long catalogId = 13L; + Map oldProperties = Collections.singletonMap("generation", "old"); + Map newProperties = Collections.singletonMap("generation", "new"); + try { + Future initialization = workers.submit( + () -> metaCacheMgr.prepareCatalogByEngine(catalogId, "hive", oldProperties)); + Assert.assertTrue(initializationEntered.await(3L, TimeUnit.SECONDS)); + + CountDownLatch removalStarted = new CountDownLatch(1); + Future removal = workers.submit(() -> { + removalStarted.countDown(); + metaCacheMgr.removeCatalogByEngine(catalogId, "hive"); + }); + Assert.assertTrue(removalStarted.await(3L, TimeUnit.SECONDS)); + Assert.assertFalse("removal must wait for the property snapshot publication", removal.isDone()); + + releaseInitialization.countDown(); + initialization.get(3L, TimeUnit.SECONDS); + removal.get(3L, TimeUnit.SECONDS); + Assert.assertFalse(hive.isCatalogInitialized(catalogId)); + + metaCacheMgr.prepareCatalogByEngine(catalogId, "hive", newProperties); + Assert.assertEquals("new", hive.lastCatalogProperties.get("generation")); + Assert.assertTrue(hive.isCatalogInitialized(catalogId)); + } finally { + releaseInitialization.countDown(); + workers.shutdownNow(); + } + } + @Test public void testGetSchemaCacheValueReturnsEmptyWhenCatalogMissing() throws Exception { MissingCatalogSchemaExternalMetaCache schemaCache = new MissingCatalogSchemaExternalMetaCache("default"); @@ -373,4 +466,35 @@ public MetaCacheEntry entry(long catalogId, String entryName, Class throw new IllegalStateException("catalog " + catalogId + " is not initialized"); } } + + private static final class BlockingRecordingExternalMetaCache extends RecordingExternalMetaCache { + private final CountDownLatch initializationEntered; + private final CountDownLatch releaseInitialization; + private final AtomicBoolean blockNextInitialization = new AtomicBoolean(true); + private Map lastCatalogProperties = Collections.emptyMap(); + + private BlockingRecordingExternalMetaCache(String engine, + CountDownLatch initializationEntered, CountDownLatch releaseInitialization) { + super(engine, Collections.emptyList(), catalog -> true); + this.initializationEntered = initializationEntered; + this.releaseInitialization = releaseInitialization; + } + + @Override + public void initCatalog(long catalogId, Map catalogProperties) { + if (blockNextInitialization.compareAndSet(true, false)) { + initializationEntered.countDown(); + try { + if (!releaseInitialization.await(3L, TimeUnit.SECONDS)) { + throw new AssertionError("timed out waiting to publish catalog initialization"); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + lastCatalogProperties = new HashMap<>(catalogProperties); + super.initCatalog(catalogId, catalogProperties); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java index 151d46252d9084..cafeaf915ddaff 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java @@ -17,23 +17,48 @@ package org.apache.doris.datasource.hive; +import org.apache.doris.analysis.PartitionValue; +import org.apache.doris.analysis.StringLiteral; +import org.apache.doris.catalog.ListPartitionItem; +import org.apache.doris.catalog.PartitionItem; +import org.apache.doris.catalog.PartitionKey; +import org.apache.doris.catalog.Type; import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.metacache.MetaCacheEntry; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import com.google.common.collect.HashBiMap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.atomic.AtomicLong; public class HiveMetaStoreCacheTest { + @Test + public void testPartitionValueWeightScalesLinearlyToOneHundredThousandPartitions() { + HiveExternalMetaCache.PartitionValueCacheKey key = new HiveExternalMetaCache.PartitionValueCacheKey( + NameMapping.createForTest("db", "tbl"), Collections.singletonList(Type.STRING)); + long base = partitionValueWeight(key, 0); + long oneThousand = partitionValueWeight(key, 1_000); + long tenThousand = partitionValueWeight(key, 10_000); + long oneHundredThousand = partitionValueWeight(key, 100_000); + + long oneThousandPayload = oneThousand - base; + Assertions.assertTrue(oneThousandPayload > 0L); + Assertions.assertEquals(oneThousandPayload * 10L, tenThousand - base); + Assertions.assertEquals(oneThousandPayload * 100L, oneHundredThousand - base); + } + @Test public void testInvalidateTableCache() { ThreadPoolExecutor executor = ThreadPoolManager.newDaemonFixedThreadPool( @@ -144,6 +169,82 @@ public void testInvalidatePartitionCacheClearsStaleFileCacheOnPartitionMiss() { } } + @Test + public void testPartitionValuesEstimateIsPreparedAgainAfterCopy() { + HiveExternalMetaCache.PartitionValueCacheKey key = new HiveExternalMetaCache.PartitionValueCacheKey( + NameMapping.createForTest("db", "tbl"), Collections.emptyList()); + PartitionKey partitionKey = new PartitionKey(); + ListPartitionItem partitionItem = new ListPartitionItem(Collections.singletonList(partitionKey)); + partitionItem.setDefaultPartition(true); + HashMap items = new HashMap<>(); + items.put(1L, partitionItem); + HashBiMap names = HashBiMap.create(); + names.put("p", 1L); + HashMap> partitionValues = new HashMap<>(); + partitionValues.put(1L, Collections.emptyList()); + HiveExternalMetaCache.HivePartitionValues values = new HiveExternalMetaCache.HivePartitionValues( + items, names, partitionValues); + + values.sealForPublication(); + values.prepareSizeEstimate(key); + Assertions.assertTrue(values.getSizeEstimate().isComplete()); + Assertions.assertTrue(values.getSizeEstimate().getBytes() > 0L); + + HiveExternalMetaCache.HivePartitionValues copy = values.mutableCopy(); + Assertions.assertFalse(copy.getSizeEstimate().isComplete()); + copy.sealForPublication(); + copy.prepareSizeEstimate(new HiveExternalMetaCache.PartitionValueCacheKey( + key.getNameMapping(), null)); + Assertions.assertTrue(copy.getSizeEstimate().isComplete()); + Assertions.assertTrue(copy.getSizeEstimate().getBytes() > 0L); + ListPartitionItem publishedItem = (ListPartitionItem) values.getIdToPartitionItem().get(1L); + Assertions.assertSame(partitionItem, publishedItem, + "cache publication must not rewrite common catalog partition objects"); + Assertions.assertSame(partitionKey, publishedItem.getItems().get(0)); + } + + @Test + public void testPartitionValuesEstimateSupportsRealLiteralGraph() throws Exception { + List types = java.util.Arrays.asList(Type.STRING, Type.INT, Type.DATEV2, Type.DECIMALV2); + HiveExternalMetaCache.PartitionValueCacheKey key = new HiveExternalMetaCache.PartitionValueCacheKey( + NameMapping.createForTest("db", "tbl"), types); + PartitionKey partitionKey = PartitionKey.createListPartitionKeyWithTypes( + java.util.Arrays.asList( + new PartitionValue("tail-value"), + new PartitionValue("42"), + new PartitionValue("2026-08-12"), + new PartitionValue("123456789.0123")), + types, true); + ListPartitionItem partitionItem = new ListPartitionItem(Collections.singletonList(partitionKey)); + HashMap items = new HashMap<>(); + items.put(1L, partitionItem); + HashBiMap names = HashBiMap.create(); + names.put("s=tail-value/i=42/d=2026-08-12/n=123456789.0123", 1L); + HashMap> partitionValues = new HashMap<>(); + partitionValues.put(1L, java.util.Arrays.asList( + "tail-value", "42", "2026-08-12", "123456789.0123")); + HiveExternalMetaCache.HivePartitionValues values = new HiveExternalMetaCache.HivePartitionValues( + items, names, partitionValues); + + values.sealForPublication(); + values.prepareSizeEstimate(key); + + Assertions.assertTrue(values.getSizeEstimate().isComplete(), + values.getSizeEstimate().getIncompleteReason()); + long estimatedBytes = values.getSizeEstimate().getBytes(); + PartitionKey publishedKey = ((ListPartitionItem) values.getIdToPartitionItem().get(1L)).getItems().get(0); + StringLiteral publishedString = (StringLiteral) publishedKey.getKeys().get(0); + // Exercise normal read-only lazy paths after publication. Their bounded memoized state is + // covered by estimator headroom without changing or cloning common expression classes. + publishedString.getExprName(); + values.getSortedPartitionRanges().orElseThrow(AssertionError::new).sortedPartitions + .forEach(partition -> partition.range.toString()); + values.prepareSizeEstimate(key); + Assertions.assertEquals(estimatedBytes, values.getSizeEstimate().getBytes()); + Assertions.assertSame(partitionKey, publishedKey, + "cache publication must not rewrite common catalog partition objects"); + } + private void putCache( MetaCacheEntry fileCache, MetaCacheEntry partitionCache, @@ -178,4 +279,22 @@ private long entrySize(MetaCacheEntry entry) { entry.forEach((k, v) -> count.incrementAndGet()); return count.get(); } + + private long partitionValueWeight( + HiveExternalMetaCache.PartitionValueCacheKey key, int partitionCount) { + Map items = sizeOnlyMap(partitionCount); + HiveExternalMetaCache.HivePartitionValues values = + new HiveExternalMetaCache.HivePartitionValues( + items, null, null, partitionCount * 16L, 1); + MetaCacheSizeEstimate estimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, values); + Assertions.assertTrue(estimate.isComplete(), estimate.getIncompleteReason()); + return estimate.getBytes(); + } + + @SuppressWarnings("unchecked") + private Map sizeOnlyMap(int size) { + Map map = Mockito.mock(Map.class); + Mockito.when(map.size()).thenReturn(size); + return map; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java index af3be4475f71ac..9992238170c2c4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java @@ -225,6 +225,8 @@ protected void runBeforeAll() throws Exception { } return invocation.callRealMethod(); }); + icebergUtilsMock.when(() -> IcebergUtils.getWritableIcebergTable( + ArgumentMatchers.any(ExternalTable.class))).thenReturn(mockedIcebergTable); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 00dde9f4cc0a74..bc8a7cc7a74ff0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -17,28 +17,768 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; +import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryStats; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.hadoop.conf.Configuration; +import org.apache.iceberg.BaseTable; +import org.apache.iceberg.DataFiles; +import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.FileMetadata; +import org.apache.iceberg.GenericBlobMetadata; +import org.apache.iceberg.GenericStatisticsFile; +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.ManifestContent; import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.Metrics; +import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SnapshotParser; +import org.apache.iceberg.SnapshotRef; +import org.apache.iceberg.SortOrder; +import org.apache.iceberg.StaticTableOperations; import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableOperations; +import org.apache.iceberg.encryption.EncryptedKey; +import org.apache.iceberg.hadoop.HadoopTables; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.types.Types; import org.junit.Assert; +import org.junit.Rule; import org.junit.Test; +import org.junit.rules.TemporaryFolder; +import org.mockito.Mockito; import java.lang.reflect.Proxy; import java.nio.ByteBuffer; import java.util.Collections; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.stream.Collectors; +import java.util.stream.IntStream; public class IcebergExternalMetaCacheTest { + @Rule + public TemporaryFolder temporaryFolder = new TemporaryFolder(); + + @Test + public void testSnapshotAndManifestWeightsScaleLinearlyToOneHundredThousandItems() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergSnapshotEntryKey snapshotKey = IcebergSnapshotEntryKey.tryCreate( + mapping, tableWithMetadataLocation("/metadata/linear-v1.json")).get(); + long snapshotBase = snapshotWeight(snapshotKey, 0); + long snapshotOneThousand = snapshotWeight(snapshotKey, 1_000); + assertLinearScale(snapshotBase, snapshotOneThousand, + snapshotWeight(snapshotKey, 10_000), snapshotWeight(snapshotKey, 100_000)); + + IcebergManifestEntryKey manifestKey = new IcebergManifestEntryKey( + "/manifest/linear.avro", ManifestContent.DATA); + long manifestBase = manifestWeight(manifestKey, 0); + long manifestOneThousand = manifestWeight(manifestKey, 1_000); + assertLinearScale(manifestBase, manifestOneThousand, + manifestWeight(manifestKey, 10_000), manifestWeight(manifestKey, 100_000)); + } + + @Test + public void testWeightedEntriesAreRegistered() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + Map properties = com.google.common.collect.Maps.newHashMap(); + properties.put("meta.cache.iceberg.table.max-weight", "4MB"); + properties.put("meta.cache.iceberg.snapshot.max-weight", "8MB"); + properties.put("meta.cache.iceberg.manifest.enable", "true"); + properties.put("meta.cache.iceberg.manifest.max-weight", "16MB"); + cache.initCatalog(1L, properties); + + Map stats = cache.stats(1L); + Assert.assertTrue(stats.get(IcebergExternalMetaCache.ENTRY_TABLE).isWeightBounded()); + Assert.assertTrue(stats.get(IcebergExternalMetaCache.ENTRY_SNAPSHOT).isWeightBounded()); + Assert.assertTrue(stats.get(IcebergExternalMetaCache.ENTRY_MANIFEST).isWeightBounded()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testSnapshotInheritsLegacyTableCountSettingsButNotWeight() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + Map properties = com.google.common.collect.Maps.newHashMap(); + properties.put("meta.cache.iceberg.table.enable", "false"); + properties.put("meta.cache.iceberg.table.ttl-second", "17"); + properties.put("meta.cache.iceberg.table.capacity", "23"); + cache.initCatalog(1L, properties); + + MetaCacheEntryStats snapshot = cache.stats(1L).get(IcebergExternalMetaCache.ENTRY_SNAPSHOT); + Assert.assertFalse(snapshot.isConfigEnabled()); + Assert.assertEquals(17L, snapshot.getTtlSecond()); + Assert.assertEquals(23L, snapshot.getCapacity()); + Assert.assertFalse(snapshot.isWeightBounded()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testSnapshotKeyIncludesMetadataGeneration() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + Table first = tableWithMetadataLocation("/metadata/v1.json"); + Table second = tableWithMetadataLocation("/metadata/v2.json"); + Table recreated = tableWithMetadataLocation("/metadata/v1.json"); + + IcebergSnapshotEntryKey firstKey = IcebergSnapshotEntryKey.tryCreate(mapping, first).get(); + IcebergSnapshotEntryKey sameKey = IcebergSnapshotEntryKey.tryCreate(mapping, first).get(); + IcebergSnapshotEntryKey secondKey = IcebergSnapshotEntryKey.tryCreate(mapping, second).get(); + IcebergSnapshotEntryKey recreatedKey = IcebergSnapshotEntryKey.tryCreate(mapping, recreated).get(); + + Assert.assertEquals(firstKey, sameKey); + Assert.assertNotEquals(firstKey, secondKey); + Assert.assertNotEquals("drop/recreate may reuse HadoopCatalog's v1 path", firstKey, recreatedKey); + Assert.assertEquals("/metadata/v1.json", firstKey.getMetadataFileLocation()); + Assert.assertNotEquals(firstKey.getTableUuid(), recreatedKey.getTableUuid()); + Assert.assertFalse(IcebergSnapshotEntryKey.tryCreate(mapping, + newInterfaceProxy(Table.class)).isPresent()); + } + + @Test + public void testTableSnapshotAndManifestEstimatesArePrecomputed() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + Table table = tableWithMetadataLocation("/metadata/v1.json"); + IcebergTableCacheValue tableValue = new IcebergTableCacheValue(table); + tableValue.prepareForCachePublication(mapping); + Assert.assertTrue(tableValue.getSizeEstimate().isComplete()); + Assert.assertTrue(tableValue.getSizeEstimate().getBytes() > 0L); + + IcebergSnapshotEntryKey snapshotKey = IcebergSnapshotEntryKey.tryCreate(mapping, table).get(); + IcebergSnapshotCacheValue snapshotValue = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), Optional.empty(), table); + snapshotValue.prepareForCachePublication(snapshotKey); + Assert.assertTrue(snapshotValue.getSizeEstimate().getIncompleteReason(), + snapshotValue.getSizeEstimate().isComplete()); + Assert.assertTrue(snapshotValue.getSizeEstimate().getBytes() > 0L); + + IcebergManifestEntryKey manifestKey = new IcebergManifestEntryKey("/manifest/a.avro", ManifestContent.DATA); + ManifestCacheValue manifestValue = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/a.parquet").withFileSizeInBytes(10L).withRecordCount(1L).build())); + MetaCacheSizeEstimate manifestEstimate = + IcebergCacheSizeEstimator.estimateManifestEntry(manifestKey, manifestValue); + Assert.assertTrue(manifestEstimate.getIncompleteReason(), manifestEstimate.isComplete()); + Assert.assertTrue(manifestEstimate.getBytes() > 0L); + + Table unsupportedTable = newInterfaceProxy(Table.class); + MetaCacheSizeEstimate unsupported = IcebergCacheSizeEstimator.estimateTableEntry( + mapping, new IcebergTableCacheValue(unsupportedTable)); + Assert.assertFalse(unsupported.isComplete()); + Assert.assertTrue(unsupported.getIncompleteReason().startsWith("unsupported_iceberg_table:")); + } + + @Test + public void testTableEstimateAccountsForNestedSchemaAndPropertyPayload() { + String largePayload = repeatedCharacter('x', 64 * 1024); + Table smallTable = tableWithNestedSchemaAndProperty("x", "x"); + Table largeTable = tableWithNestedSchemaAndProperty(largePayload, largePayload); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergTableCacheValue smallValue = new IcebergTableCacheValue(smallTable); + IcebergTableCacheValue largeValue = new IcebergTableCacheValue(largeTable); + + smallValue.prepareForCachePublication(mapping); + largeValue.prepareForCachePublication(mapping); + + long expectedPayloadDelta = (largePayload.length() - 1L) * 4L; + Assert.assertTrue(largeValue.getSizeEstimate().getBytes() + - smallValue.getSizeEstimate().getBytes() >= expectedPayloadDelta); + } + + @Test + public void testTablePayloadCountsHistoricalSchemaSpecAndSortFields() { + List fields = IntStream.range(0, 100) + .mapToObj(index -> Types.NestedField.optional( + index + 1, "field_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + Schema largeSchema = new Schema(0, fields); + Schema smallSchema = new Schema(1, fields.get(0)); + TableMetadata schemaHistory = TableMetadata.newTableMetadata( + largeSchema, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-history", Collections.emptyMap()); + schemaHistory = TableMetadata.buildFrom(schemaHistory) + .addSchema(smallSchema) + .setCurrentSchema(smallSchema.schemaId()) + .discardChanges() + .build(); + TableMetadata smallSchemaOnly = TableMetadata.newTableMetadata( + smallSchema, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-history", Collections.emptyMap()); + + PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(largeSchema).withSpecId(0); + SortOrder.Builder sortBuilder = SortOrder.builderFor(largeSchema).withOrderId(1); + for (Types.NestedField field : fields) { + specBuilder.identity(field.name()); + sortBuilder.asc(field.name()); + } + TableMetadata fieldHistory = TableMetadata.newTableMetadata( + largeSchema, specBuilder.build(), sortBuilder.build(), + "file:/warehouse/field-history", Collections.emptyMap()); + fieldHistory = TableMetadata.buildFrom(fieldHistory) + .setDefaultPartitionSpec( + PartitionSpec.builderFor(largeSchema).withSpecId(1).build()) + .setDefaultSortOrder(SortOrder.unsorted()) + .discardChanges() + .build(); + TableMetadata emptyFields = TableMetadata.newTableMetadata( + largeSchema, PartitionSpec.unpartitioned(), SortOrder.unsorted(), + "file:/warehouse/field-history", Collections.emptyMap()); + + long schemaDelta = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(schemaHistory)) + - IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(smallSchemaOnly)); + long specAndSortDelta = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(fieldHistory)) + - IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(emptyFields)); + + Assert.assertTrue(schemaDelta >= 99L * 512L); + Assert.assertTrue(specAndSortDelta >= 100L * (384L + 256L)); + } + + @Test + public void testTablePayloadExcludesQueryLocalHistoricalMetadata() { + String largePayload = repeatedCharacter('x', 64 * 1024); + long smallBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(metadataWithMaterializedPayload("x", 32))); + long largeBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(metadataWithMaterializedPayload(largePayload, 64 * 1024))); + + Assert.assertEquals(smallBytes, largeBytes); + } + + @Test + public void testTableEstimateExcludesQueryLocalBranchHistory() { + TableMetadata oneCommit = metadataWithSnapshotSequence(1L); + TableMetadata tenThousandCommits = metadataWithSnapshotSequence(10_000L); + IcebergTableCacheValue smallValue = new IcebergTableCacheValue(tableWithMetadata(oneCommit)); + IcebergTableCacheValue largeValue = new IcebergTableCacheValue( + tableWithMetadata(tenThousandCommits)); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + + smallValue.prepareForCachePublication(mapping); + largeValue.prepareForCachePublication(mapping); + + Assert.assertTrue(smallValue.getSizeEstimate().getIncompleteReason(), + smallValue.getSizeEstimate().isComplete()); + Assert.assertTrue(largeValue.getSizeEstimate().getIncompleteReason(), + largeValue.getSizeEstimate().isComplete()); + Assert.assertEquals(smallValue.getSizeEstimate().getBytes(), + largeValue.getSizeEstimate().getBytes()); + Mockito.verify(oneCommit, Mockito.never()).snapshots(); + Mockito.verify(tenThousandCommits, Mockito.never()).snapshots(); + Mockito.verify(oneCommit, Mockito.never()).lastSequenceNumber(); + Mockito.verify(tenThousandCommits, Mockito.never()).lastSequenceNumber(); + Mockito.verify(oneCommit, Mockito.never()).snapshotLog(); + Mockito.verify(tenThousandCommits, Mockito.never()).snapshotLog(); + Mockito.verify(oneCommit, Mockito.never()).refs(); + Mockito.verify(tenThousandCommits, Mockito.never()).refs(); + } + + @Test + public void testWeightedTablePreparationRunsInsideCatalogAuthenticator() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + AtomicBoolean authenticated = new AtomicBoolean(); + AtomicBoolean firstPreparation = new AtomicBoolean(true); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + IcebergMetadataOps metadataOps = Mockito.mock(IcebergMetadataOps.class); + Table table = tableWithMetadataLocation("/metadata/authenticated-v1.json"); + Mockito.when(catalog.getMetadataOps()).thenReturn(metadataOps); + Mockito.when(metadataOps.loadTable("remote_db", "remote_tbl")).thenReturn(table); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + Assert.assertTrue(authenticated.compareAndSet(false, true)); + try { + return task.call(); + } finally { + authenticated.set(false); + } + } + }); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor) { + @Override + protected CatalogIf getCatalog(long catalogId) { + return catalog; + } + + @Override + MetaCacheSizeEstimate prepareTableForCachePublication( + NameMapping nameMapping, IcebergTableCacheValue value) { + if (firstPreparation.compareAndSet(true, false)) { + Assert.assertTrue("publication preparation must retain Kerberos scope", + authenticated.get()); + } + return super.prepareTableForCachePublication(nameMapping, value); + } + }; + try { + cache.initCatalog(1L, Collections.singletonMap( + "meta.cache.iceberg.table.max-weight", "4MB")); + NameMapping mapping = new NameMapping( + 1L, "db", "tbl", "remote_db", "remote_tbl"); + + IcebergTableCacheValue value = cache.entry(1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).get(mapping); + IcebergTableCacheValue cached = cache.entry(1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).get(mapping); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), + value.getSizeEstimate().isComplete()); + Assert.assertSame(value, cached); + Mockito.verify(metadataOps, Mockito.times(1)).loadTable("remote_db", "remote_tbl"); + Assert.assertFalse(authenticated.get()); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testExactMetadataReadsRunInsideCatalogAuthenticator() throws Exception { + String tableLocation = temporaryFolder.newFolder("authenticated-metadata").toURI().toString(); + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + Table liveTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), tableLocation); + AtomicBoolean authenticated = new AtomicBoolean(); + AtomicInteger metadataReads = new AtomicInteger(); + ExecutionAuthenticator authenticator = new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + Assert.assertTrue(authenticated.compareAndSet(false, true)); + try { + return task.call(); + } finally { + authenticated.set(false); + } + } + }; + FileIO trackingFileIO = Mockito.mock(FileIO.class); + Mockito.when(trackingFileIO.newInputFile(Mockito.anyString())).thenAnswer(invocation -> { + Assert.assertTrue("metadata FileIO must retain catalog authentication", authenticated.get()); + metadataReads.incrementAndGet(); + return liveTable.io().newInputFile((String) invocation.getArgument(0)); + }); + TableMetadata metadata = ((HasTableOperations) liveTable).operations().current(); + Table trackedTable = new BaseTable( + new StaticTableOperations(metadata, trackingFileIO), liveTable.name()); + IcebergTableCacheValue countValue = new IcebergTableCacheValue(trackedTable, authenticator); + countValue.getWritableIcebergTable(liveTable); + Assert.assertEquals("count-based writes must not add metadata FileIO", 0, metadataReads.get()); + IcebergTableCacheValue value = new IcebergTableCacheValue(trackedTable, authenticator); + value.prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + + Table statementTable = value.newQueryScopedTable(); + IcebergSnapshotCacheValue.loadQueryMetadataForStatement(statementTable); + IcebergSnapshotCacheValue statementValue = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), + Optional.empty(), statementTable); + Assert.assertSame(statementTable, statementValue.getIcebergTable().get()); + com.google.common.collect.Lists.newArrayList( + statementValue.getIcebergTable().get().snapshots()); + Assert.assertEquals("statement handoff must reuse parsed metadata", 1, metadataReads.get()); + value.getWritableIcebergTable(liveTable); + + Assert.assertEquals(2, metadataReads.get()); + Assert.assertFalse(authenticated.get()); + } + + @Test + public void testCountModeTimeTravelDoesNotEnableQueryIsolation() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + cache.initCatalog(1L, Collections.emptyMap()); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergTableCacheValue value = new IcebergTableCacheValue( + tableWithMetadataLocation("/metadata/count-v1.json")); + cache.entry(1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).put(mapping, value); + ExternalTable table = Mockito.mock(IcebergExternalTable.class); + Mockito.when(table.getOrBuildNameMapping()).thenReturn(mapping); + + Table queryTable = cache.getQueryScopedIcebergTable(table); + + Assert.assertSame(value.getRetainedIcebergTable(), queryTable); + Assert.assertFalse(value.isQueryIsolationPrepared()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testTimeTravelGenerationBundleDoesNotMixReplacedTableValue() throws Exception { + String firstLocation = temporaryFolder.newFolder("bundle-first").toURI().toString(); + String secondLocation = temporaryFolder.newFolder("bundle-second").toURI().toString(); + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + Table firstTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), firstLocation); + firstTable.newAppend().appendFile(DataFiles.builder(firstTable.spec()) + .withPath(firstLocation + "/data/a.parquet") + .withFileSizeInBytes(10L).withRecordCount(1L).build()).commit(); + Table secondTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), secondLocation); + secondTable.newAppend().appendFile(DataFiles.builder(secondTable.spec()) + .withPath(secondLocation + "/data/b.parquet") + .withFileSizeInBytes(20L).withRecordCount(2L).build()).commit(); + long firstSnapshotId = firstTable.currentSnapshot().snapshotId(); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + cache.initCatalog(1L, Collections.singletonMap( + "meta.cache.iceberg.table.max-weight", "4MB")); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + MetaCacheEntry entry = cache.entry( + 1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class); + entry.put(mapping, new IcebergTableCacheValue(firstTable)); + ExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + + Table queryTable = cache.getQueryScopedIcebergTable(dorisTable); + entry.put(mapping, new IcebergTableCacheValue(secondTable)); + + Assert.assertEquals(firstSnapshotId, + queryTable.currentSnapshot().snapshotId()); + Assert.assertEquals(firstTable.location(), + queryTable.location()); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testMissingPinnedMetadataRefreshesBeforeStatementFence() throws Exception { + String staleLocation = temporaryFolder.newFolder("stale-metadata").toURI().toString(); + String freshLocation = temporaryFolder.newFolder("fresh-metadata").toURI().toString(); + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + Table staleTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), staleLocation); + Table freshTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), freshLocation); + String staleMetadataLocation = ((HasTableOperations) staleTable) + .operations().current().metadataFileLocation(); + IcebergTableCacheValue staleValue = new IcebergTableCacheValue(staleTable); + staleValue.prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + staleTable.io().deleteFile(staleMetadataLocation); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + IcebergMetadataOps metadataOps = Mockito.mock(IcebergMetadataOps.class); + Mockito.when(catalog.getMetadataOps()).thenReturn(metadataOps); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { }); + Mockito.when(metadataOps.loadTable("db", "tbl")).thenReturn(freshTable); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor) { + @Override + protected CatalogIf getCatalog(long catalogId) { + return catalog; + } + }; + try { + cache.initCatalog(1L, Collections.singletonMap( + "meta.cache.iceberg.table.max-weight", "4MB")); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + cache.entry(1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).put(mapping, staleValue); + ExternalTable table = Mockito.mock(IcebergExternalTable.class); + Mockito.when(table.getOrBuildNameMapping()).thenReturn(mapping); + + Table queryTable = cache.getQueryScopedIcebergTable(table); + + Assert.assertEquals(freshTable.schema().asStruct(), queryTable.schema().asStruct()); + Mockito.verify(metadataOps, Mockito.times(1)).loadTable("db", "tbl"); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testWeightedTablePublicationRetainsNonGrowingGeneration() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), + "file:/warehouse/db/tbl", Collections.emptyMap()); + Snapshot currentSnapshot = SnapshotParser.fromJson("{\"snapshot-id\":7,\"timestamp-ms\":2," + + "\"summary\":{\"operation\":\"append\"}," + + "\"manifests\":[\"/manifest/current-a.avro\"," + + "\"/manifest/current-b.avro\"],\"schema-id\":0}"); + metadata = TableMetadata.buildFrom(metadata) + .setBranchSnapshot(currentSnapshot, SnapshotRef.MAIN_BRANCH) + .discardChanges().withMetadataLocation("/metadata/v1.json").build(); + FileIO fileIO = Mockito.mock(FileIO.class); + Mockito.when(fileIO.newInputFile(Mockito.anyString())).thenAnswer(invocation -> { + InputFile inputFile = Mockito.mock(InputFile.class); + Mockito.when(inputFile.location()).thenReturn(invocation.getArgument(0)); + return inputFile; + }); + Table liveTable = new BaseTable(new StaticTableOperations(metadata, fileIO), "db.tbl"); + IcebergTableCacheValue value = new IcebergTableCacheValue(liveTable); + + value.prepareForCachePublication(mapping); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), value.getSizeEstimate().isComplete()); + Table retained = value.getRetainedIcebergTable(); + Assert.assertTrue(IcebergSnapshotCacheValue.isFrozenGeneration(retained)); + Assert.assertThrows(UnsupportedOperationException.class, + () -> retained.snapshot(7L).dataManifests(retained.io())); + Table firstUse = value.getIcebergTable(); + Table secondUse = value.getIcebergTable(); + Assert.assertNotSame(retained, firstUse); + Assert.assertNotSame(firstUse, secondUse); + Assert.assertNotSame(retained.currentSnapshot(), firstUse.currentSnapshot()); + Assert.assertNotSame(firstUse.currentSnapshot(), secondUse.currentSnapshot()); + Assert.assertEquals(2, firstUse.snapshot(7L).dataManifests(firstUse.io()).size()); + Assert.assertEquals(2, secondUse.snapshot(7L).dataManifests(secondUse.io()).size()); + Assert.assertTrue(IcebergSnapshotCacheValue.isFrozenGeneration(firstUse)); + + IcebergSnapshotEntryKey snapshotKey = + IcebergSnapshotEntryKey.tryCreate(mapping, retained).get(); + IcebergSnapshotCacheValue snapshotValue = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(7L, 0L), + Optional.empty(), retained, value.getRetainedCurrentSnapshotJson()); + snapshotValue.prepareForCachePublication(snapshotKey); + Assert.assertTrue(snapshotValue.getSizeEstimate().getIncompleteReason(), + snapshotValue.getSizeEstimate().isComplete()); + Table snapshotQuery = snapshotValue.getIcebergTable().get(); + Assert.assertEquals(2, + snapshotQuery.currentSnapshot().dataManifests(snapshotQuery.io()).size()); + } + + @Test + public void testWeightedV2ManifestListMaterializesOnlyInQueryView() throws Exception { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + String tableLocation = temporaryFolder.newFolder("v2-table").toURI().toString(); + Table liveTable = new HadoopTables(new Configuration()).create( + schema, PartitionSpec.unpartitioned(), tableLocation); + liveTable.newAppend().appendFile( + DataFiles.builder(liveTable.spec()) + .withPath(tableLocation + "/data/a.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build()).commit(); + Assert.assertNotNull(liveTable.currentSnapshot().manifestListLocation()); + IcebergTableCacheValue value = new IcebergTableCacheValue(liveTable); + + value.prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), + value.getSizeEstimate().isComplete()); + Table retained = value.getRetainedIcebergTable(); + Assert.assertThrows(UnsupportedOperationException.class, + () -> retained.currentSnapshot().dataManifests(retained.io())); + Table firstQuery = value.getIcebergTable(); + Table secondQuery = value.getIcebergTable(); + List firstManifests = + firstQuery.currentSnapshot().dataManifests(firstQuery.io()); + List secondManifests = + secondQuery.currentSnapshot().dataManifests(secondQuery.io()); + Assert.assertEquals(1, firstManifests.size()); + Assert.assertEquals(1, secondManifests.size()); + Assert.assertNotSame(firstQuery.currentSnapshot(), secondQuery.currentSnapshot()); + Assert.assertNotSame(firstManifests, secondManifests); + Assert.assertThrows(UnsupportedOperationException.class, + () -> retained.currentSnapshot().dataManifests(retained.io())); + } + + @Test + public void testTablePublicationDoesNotReadHistoricalManifestLists() { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), + "file:/warehouse/db/tbl", Collections.emptyMap()); + Snapshot historical = SnapshotParser.fromJson("{\"snapshot-id\":6,\"timestamp-ms\":1," + + "\"summary\":{\"operation\":\"append\"}," + + "\"manifest-list\":\"/manifest-list/history.avro\",\"schema-id\":0}"); + Snapshot current = SnapshotParser.fromJson("{\"snapshot-id\":7,\"timestamp-ms\":2," + + "\"summary\":{\"operation\":\"append\"},\"manifests\":[],\"schema-id\":0}"); + metadata = TableMetadata.buildFrom(metadata) + .addSnapshot(historical) + .setBranchSnapshot(current, SnapshotRef.MAIN_BRANCH) + .discardChanges().withMetadataLocation("/metadata/v2.json").build(); + FileIO fileIO = Mockito.mock(FileIO.class); + IcebergTableCacheValue value = new IcebergTableCacheValue( + new BaseTable(new StaticTableOperations(metadata, fileIO), "db.tbl")); + + value.prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), value.getSizeEstimate().isComplete()); + Mockito.verify(fileIO, Mockito.never()).newInputFile("/manifest-list/history.avro"); + } + + @Test + public void testManifestEstimateScalesWithFileCount() { + ManifestCacheValue oneFile = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/one.parquet").withFileSizeInBytes(10L).withRecordCount(1L).build())); + ManifestCacheValue twoFiles = ManifestCacheValue.forDataFiles(java.util.Arrays.asList( + oneFile.getDataFiles().get(0), oneFile.getDataFiles().get(0))); + IcebergManifestEntryKey key = new IcebergManifestEntryKey("/manifest/data.avro", ManifestContent.DATA); + + long oneFileBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, oneFile).getBytes(); + long twoFileBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, twoFiles).getBytes(); + + Assert.assertTrue(twoFileBytes > oneFileBytes); + } + + @Test + public void testManifestEstimateAccountsForSkewedFilePaths() { + String largePath = "/data/" + repeatedCharacter('x', 64 * 1024) + ".parquet"; + ManifestCacheValue smallValue = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/x.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build())); + ManifestCacheValue largeValue = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath(largePath) + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build())); + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/path-skew.avro", ManifestContent.DATA); + + long smallBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, smallValue).getBytes(); + long largeBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, largeValue).getBytes(); + + Assert.assertTrue(largeBytes - smallBytes >= (largePath.length() - "/data/x.parquet".length()) * 2L); + } + + @Test + public void testManifestEstimateAccountsForSkewedBufferPayload() { + Metrics smallMetrics = new Metrics(1L, Collections.emptyMap(), Collections.emptyMap(), + Collections.emptyMap(), Collections.emptyMap(), + Collections.singletonMap(1, ByteBuffer.allocateDirect(32)), Collections.emptyMap()); + Metrics largeMetrics = new Metrics(1L, Collections.emptyMap(), Collections.emptyMap(), + Collections.emptyMap(), Collections.emptyMap(), + Collections.singletonMap(1, ByteBuffer.allocateDirect(64 * 1024)), Collections.emptyMap()); + ManifestCacheValue smallValue = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/encrypted.parquet") + .withFileSizeInBytes(10L) + .withMetrics(smallMetrics) + .build())); + ManifestCacheValue largeValue = ManifestCacheValue.forDataFiles(Collections.singletonList( + DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/encrypted.parquet") + .withFileSizeInBytes(10L) + .withMetrics(largeMetrics) + .build())); + + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/encrypted.avro", ManifestContent.DATA); + MetaCacheSizeEstimate smallEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, smallValue); + MetaCacheSizeEstimate largeEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, largeValue); + + Assert.assertTrue(smallEstimate.getIncompleteReason(), smallEstimate.isComplete()); + Assert.assertTrue(largeEstimate.getIncompleteReason(), largeEstimate.isComplete()); + Assert.assertEquals(1L, smallValue.getDataFileMetricEntryCount()); + Assert.assertEquals(1L, largeValue.getDataFileMetricEntryCount()); + Assert.assertTrue(largeEstimate.getBytes() - smallEstimate.getBytes() >= 64 * 1024 - 32); + } + + @Test + public void testManifestEstimateAccountsForDeleteFileAuxiliaryPayload() { + String largeReference = "/data/" + repeatedCharacter('x', 64 * 1024) + ".parquet"; + List largeOffsets = IntStream.range(0, 4096) + .mapToObj(index -> (long) index).collect(Collectors.toList()); + DeleteFile smallPositionDelete = FileMetadata.deleteFileBuilder(PartitionSpec.unpartitioned()) + .ofPositionDeletes() + .withPath("/delete/position.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .withReferencedDataFile("/data/x.parquet") + .withSplitOffsets(Collections.singletonList(0L)) + .build(); + DeleteFile largePositionDelete = FileMetadata.deleteFileBuilder(PartitionSpec.unpartitioned()) + .ofPositionDeletes() + .withPath("/delete/position.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .withReferencedDataFile(largeReference) + .withSplitOffsets(largeOffsets) + .build(); + int[] largeEqualityIds = IntStream.range(0, 4096).toArray(); + DeleteFile smallEqualityDelete = FileMetadata.deleteFileBuilder(PartitionSpec.unpartitioned()) + .ofEqualityDeletes(1) + .withPath("/delete/equality.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build(); + DeleteFile largeEqualityDelete = FileMetadata.deleteFileBuilder(PartitionSpec.unpartitioned()) + .ofEqualityDeletes(largeEqualityIds) + .withPath("/delete/equality.parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build(); + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/delete.avro", ManifestContent.DELETES); + + long smallPositionBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, + ManifestCacheValue.forDeleteFiles(Collections.singletonList(smallPositionDelete))).getBytes(); + long largePositionBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, + ManifestCacheValue.forDeleteFiles(Collections.singletonList(largePositionDelete))).getBytes(); + long smallEqualityBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, + ManifestCacheValue.forDeleteFiles(Collections.singletonList(smallEqualityDelete))).getBytes(); + long largeEqualityBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, + ManifestCacheValue.forDeleteFiles(Collections.singletonList(largeEqualityDelete))).getBytes(); + + Assert.assertTrue(largePositionBytes > smallPositionBytes); + Assert.assertTrue(largeEqualityBytes > smallEqualityBytes); + } + + @Test + public void testSnapshotPublicationDoesNotMaterializeManifestLists() { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), + "file:/warehouse/db/tbl", Collections.emptyMap()); + Snapshot snapshot = SnapshotParser.fromJson("{\"snapshot-id\":7,\"timestamp-ms\":1," + + "\"summary\":{\"operation\":\"append\"}," + + "\"manifests\":[\"/manifest/a.avro\",\"/manifest/b.avro\"],\"schema-id\":0}"); + metadata = TableMetadata.buildFrom(metadata).setBranchSnapshot(snapshot, SnapshotRef.MAIN_BRANCH) + .discardChanges().withMetadataLocation("/metadata/v1.json").build(); + FileIO fileIO = Mockito.mock(FileIO.class); + Table table = new BaseTable(new StaticTableOperations(metadata, fileIO), "db.tbl"); + IcebergSnapshotCacheValue value = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(7L, 0L), Optional.empty(), table); + IcebergSnapshotEntryKey key = IcebergSnapshotEntryKey.tryCreate( + NameMapping.createForTest(1L, "db", "tbl"), table).get(); + + value.prepareForCachePublication(key); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), + value.getSizeEstimate().isComplete()); + Table queryTable = value.getIcebergTable().get(); + Assert.assertNotSame(table.currentSnapshot(), queryTable.currentSnapshot()); + Assert.assertTrue(IcebergSnapshotCacheValue.isFrozenGeneration(queryTable)); + Mockito.verifyNoInteractions(fileIO); + } @Test public void testInvalidateTableKeepsManifestCache() { @@ -52,10 +792,16 @@ public void testInvalidateTableKeepsManifestCache() { MetaCacheEntry tableEntry = cache.entry(catalogId, IcebergExternalMetaCache.ENTRY_TABLE, NameMapping.class, IcebergTableCacheValue.class); - tableEntry.put(t1, new IcebergTableCacheValue(newInterfaceProxy(Table.class), - () -> new IcebergSnapshotCacheValue(IcebergPartitionInfo.empty(), new IcebergSnapshot(1L, 1L)))); - tableEntry.put(t2, new IcebergTableCacheValue(newInterfaceProxy(Table.class), - () -> new IcebergSnapshotCacheValue(IcebergPartitionInfo.empty(), new IcebergSnapshot(2L, 2L)))); + tableEntry.put(t1, new IcebergTableCacheValue(newInterfaceProxy(Table.class))); + tableEntry.put(t2, new IcebergTableCacheValue(newInterfaceProxy(Table.class))); + + Table snapshotTable = tableWithMetadataLocation("/metadata/invalidate-v1.json"); + IcebergSnapshotEntryKey snapshotKey = IcebergSnapshotEntryKey.tryCreate(t1, snapshotTable).get(); + MetaCacheEntry snapshotEntry = cache.entry(catalogId, + IcebergExternalMetaCache.ENTRY_SNAPSHOT, + IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class); + snapshotEntry.put(snapshotKey, + new IcebergSnapshotCacheValue(IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L))); MetaCacheEntry viewEntry = cache.entry(catalogId, IcebergExternalMetaCache.ENTRY_VIEW, NameMapping.class, org.apache.iceberg.view.View.class); @@ -77,6 +823,7 @@ public void testInvalidateTableKeepsManifestCache() { Assert.assertNull(tableEntry.getIfPresent(t1)); Assert.assertNotNull(tableEntry.getIfPresent(t2)); + Assert.assertNull(snapshotEntry.getIfPresent(snapshotKey)); Assert.assertNull(viewEntry.getIfPresent(t1)); Assert.assertNotNull(viewEntry.getIfPresent(t2)); Assert.assertNotNull(manifestEntry.getIfPresent(m1)); @@ -98,10 +845,8 @@ public void testInvalidateDbAndStats() { MetaCacheEntry tableEntry = cache.entry(catalogId, IcebergExternalMetaCache.ENTRY_TABLE, NameMapping.class, IcebergTableCacheValue.class); - tableEntry.put(db1Table, new IcebergTableCacheValue(newInterfaceProxy(Table.class), - () -> new IcebergSnapshotCacheValue(IcebergPartitionInfo.empty(), new IcebergSnapshot(1L, 1L)))); - tableEntry.put(db2Table, new IcebergTableCacheValue(newInterfaceProxy(Table.class), - () -> new IcebergSnapshotCacheValue(IcebergPartitionInfo.empty(), new IcebergSnapshot(2L, 2L)))); + tableEntry.put(db1Table, new IcebergTableCacheValue(newInterfaceProxy(Table.class))); + tableEntry.put(db2Table, new IcebergTableCacheValue(newInterfaceProxy(Table.class))); MetaCacheEntry schemaEntry = cache.entry(catalogId, IcebergExternalMetaCache.ENTRY_SCHEMA, IcebergSchemaCacheKey.class, SchemaCacheValue.class); @@ -228,6 +973,139 @@ private Map manifestCacheEnabledProperties() { return properties; } + private long snapshotWeight(IcebergSnapshotEntryKey key, int partitionCount) { + IcebergPartitionInfo partitionInfo = Mockito.mock(IcebergPartitionInfo.class); + Map partitionItems = sizeOnlyMap(partitionCount); + Map partitions = sizeOnlyMap(partitionCount); + Map> aliases = sizeOnlyMap(partitionCount); + Mockito.when(partitionInfo.getNameToPartitionItem()).thenReturn(partitionItems); + Mockito.when(partitionInfo.getNameToIcebergPartition()).thenReturn(partitions); + Mockito.when(partitionInfo.getNameToIcebergPartitionNames()).thenReturn(aliases); + IcebergSnapshotCacheValue value = new IcebergSnapshotCacheValue( + partitionInfo, new IcebergSnapshot(key.getSnapshotId(), key.getSchemaId())); + MetaCacheSizeEstimate estimate = IcebergCacheSizeEstimator.estimateSnapshotEntry(key, value); + Assert.assertTrue(estimate.getIncompleteReason(), estimate.isComplete()); + return estimate.getBytes(); + } + + private long manifestWeight(IcebergManifestEntryKey key, int fileCount) { + ManifestCacheValue value = Mockito.mock(ManifestCacheValue.class); + List dataFiles = sizeOnlyList(fileCount); + Mockito.when(value.getDataFiles()).thenReturn(dataFiles); + Mockito.when(value.getDeleteFiles()).thenReturn(Collections.emptyList()); + MetaCacheSizeEstimate estimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, value); + Assert.assertTrue(estimate.getIncompleteReason(), estimate.isComplete()); + return estimate.getBytes(); + } + + private void assertLinearScale(long base, long oneThousand, long tenThousand, long oneHundredThousand) { + long oneThousandPayload = oneThousand - base; + Assert.assertTrue(oneThousandPayload > 0L); + Assert.assertEquals(oneThousandPayload * 10L, tenThousand - base); + Assert.assertEquals(oneThousandPayload * 100L, oneHundredThousand - base); + } + + @SuppressWarnings("unchecked") + private Map sizeOnlyMap(int size) { + Map map = Mockito.mock(Map.class); + Mockito.when(map.size()).thenReturn(size); + return map; + } + + @SuppressWarnings("unchecked") + private List sizeOnlyList(int size) { + List list = Mockito.mock(List.class); + Mockito.when(list.size()).thenReturn(size); + return list; + } + + private static String repeatedCharacter(char character, int count) { + char[] characters = new char[count]; + java.util.Arrays.fill(characters, character); + return new String(characters); + } + + private Table tableWithMetadataLocation(String metadataLocation) { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), + "file:/warehouse/db/tbl", Collections.emptyMap()); + metadata = TableMetadata.buildFrom(metadata).discardChanges() + .withMetadataLocation(metadataLocation).build(); + return new BaseTable(new StaticTableOperations(metadata, null), "db.tbl"); + } + + private Table tableWithMetadata(TableMetadata metadata) { + TableOperations operations = Mockito.mock(TableOperations.class); + Mockito.when(operations.current()).thenReturn(metadata); + return new BaseTable(operations, "db.tbl"); + } + + private TableMetadata metadataWithMaterializedPayload(String payload, int bufferBytes) { + TableMetadata metadata = Mockito.mock(TableMetadata.class); + Mockito.when(metadata.schemas()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.specs()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.sortOrders()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.properties()).thenReturn(Collections.emptyMap()); + Mockito.when(metadata.uuid()).thenReturn("stable-uuid"); + Mockito.when(metadata.refs()).thenReturn(Collections.singletonMap( + payload, Mockito.mock(SnapshotRef.class))); + TableMetadata.MetadataLogEntry metadataLogEntry = + Mockito.mock(TableMetadata.MetadataLogEntry.class); + Mockito.when(metadataLogEntry.file()).thenReturn(payload); + Mockito.when(metadata.previousFiles()).thenReturn( + Collections.singletonList(metadataLogEntry)); + GenericBlobMetadata blob = new GenericBlobMetadata( + payload, 1L, 1L, Collections.singletonList(1), + Collections.singletonMap(payload, payload)); + Mockito.when(metadata.statisticsFiles()).thenReturn(Collections.singletonList( + new GenericStatisticsFile(1L, payload, 1L, 1L, + Collections.singletonList(blob)))); + org.apache.iceberg.PartitionStatisticsFile partitionStatistics = + Mockito.mock(org.apache.iceberg.PartitionStatisticsFile.class); + Mockito.when(partitionStatistics.path()).thenReturn(payload); + Mockito.when(metadata.partitionStatisticsFiles()).thenReturn( + Collections.singletonList(partitionStatistics)); + EncryptedKey encryptedKey = Mockito.mock(EncryptedKey.class); + Mockito.when(encryptedKey.keyId()).thenReturn(payload); + Mockito.when(encryptedKey.encryptedById()).thenReturn(payload); + Mockito.when(encryptedKey.encryptedKeyMetadata()).thenReturn( + ByteBuffer.allocateDirect(bufferBytes)); + Mockito.when(encryptedKey.properties()).thenReturn( + Collections.singletonMap(payload, payload)); + Mockito.when(metadata.encryptionKeys()).thenReturn(Collections.singletonList(encryptedKey)); + return metadata; + } + + private TableMetadata metadataWithSnapshotSequence(long lastSequenceNumber) { + TableMetadata metadata = Mockito.mock(TableMetadata.class); + Mockito.when(metadata.schemas()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.specs()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.sortOrders()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.properties()).thenReturn(Collections.emptyMap()); + Mockito.when(metadata.snapshotLog()).thenReturn(Collections.singletonList( + Mockito.mock(org.apache.iceberg.HistoryEntry.class))); + Mockito.when(metadata.refs()).thenReturn(Collections.singletonMap( + "branch-tip", Mockito.mock(SnapshotRef.class))); + Mockito.when(metadata.previousFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.statisticsFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.partitionStatisticsFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.encryptionKeys()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.lastSequenceNumber()).thenReturn(lastSequenceNumber); + Mockito.when(metadata.metadataFileLocation()).thenReturn("/metadata/sequence.json"); + return metadata; + } + + private Table tableWithNestedSchemaAndProperty(String nestedFieldName, String propertyValue) { + Schema schema = new Schema(Types.NestedField.optional(1, "payload", + Types.StructType.of(Types.NestedField.optional( + 2, nestedFieldName, Types.StringType.get())))); + TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), + "file:/warehouse/db/tbl", Collections.singletonMap("payload", propertyValue)); + metadata = TableMetadata.buildFrom(metadata).discardChanges() + .withMetadataLocation("/metadata/nested.json").build(); + return new BaseTable(new StaticTableOperations(metadata, null), "db.tbl"); + } + private IcebergManifestEntryKey mockManifestKey(String path) { return IcebergManifestEntryKey.of(new TestingManifestFile(path, ManifestContent.DATA)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java index 720f66fd5f9e3f..de3ab8c9746397 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java @@ -92,9 +92,9 @@ public void setUp() throws IOException { Mockito.doReturn(db).when(catalog).getDbNullable(Mockito.any()); Mockito.doReturn(dorisTable).when(db).getTableNullable(Mockito.any()); - // mock IcebergUtils.getIcebergTable to return our test icebergTable + // Mock writable access used by branch and tag mutations. mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class); - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(Mockito.any())) + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.any())) .thenReturn(icebergTable); // mock Env.getCurrentEnv().getEditLog().logBranchOrTag(info) to do nothing diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index a4b882c4497132..cc019c73ed12a7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -138,7 +138,8 @@ public void testTopLevelVariantModifyOnlyUpdatesMetadataOnOrcTable() throws Thro try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)) + .thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("payload"), column, ColumnPosition.FIRST, 1L); } @@ -165,7 +166,8 @@ public void testTopLevelVariantModifyRejectsTypeConversions() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)) + .thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("variant_col"), new Column("variant_col", Type.STRING, true), null, 1L), @@ -294,7 +296,7 @@ public void testRejectUnsupportedIcebergTargetTypesBeforeUpdateSchema() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.addColumn(dorisTable, ColumnPath.fromDotName("info.new_field"), new Column("new_field", Type.LARGEINT, true), null, 1L), @@ -328,7 +330,7 @@ public void testComplexModifyPreservesRequiredNestedFields() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.child"), new Column("child", new StructType(new StructField("value", Type.BIGINT)), true), null, 1L); @@ -363,7 +365,7 @@ public void testComplexModifyPersistsDecodedStructMemberComment() throws Throwab try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.payload"), column, null, 1L); } @@ -391,7 +393,7 @@ public void testPrimitiveModifyPreservesOmittedCommentAndClearsExplicitEmptyComm try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), new Column("metric", Type.BIGINT, true), null, 1L); @@ -433,7 +435,7 @@ public void testFullStructModifyPreservesOmittedChildComments() throws Throwable try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.payload"), new Column("payload", payloadType, true), null, 1L); @@ -463,7 +465,7 @@ public void testPrimitiveModifyPreservesRequiredNestedField() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), new Column("metric", Type.BIGINT, true), null, 1L); @@ -489,7 +491,7 @@ public void testTopLevelModifyPreservesRequiredMixedCaseFields() throws Throwabl try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("id"), new Column("id", Type.BIGINT, true), null, 1L); @@ -515,7 +517,7 @@ public void testTopLevelModifyDoesNotResolveQuotedComponentAsNestedPath() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("a.b"), new Column("a.b", Type.BIGINT, true), null, 1L), @@ -537,7 +539,7 @@ public void testTopLevelModifyPreservesDottedTopLevelName() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("a.b"), new Column("a.b", Type.BIGINT, true), null, 1L); @@ -568,7 +570,7 @@ public void testPrimitiveModifyPreservesActualTypeWhenMappingDisabled() throws T try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("top_uuid"), topUuid, ColumnPosition.FIRST, 1L); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.uuid_value"), nestedUuid, @@ -603,7 +605,7 @@ public void testPrimitiveModifyPreservesActualTypeWhenMappingEnabled() throws Th try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("top_uuid"), new Column("top_uuid", ScalarType.createVarbinaryType(16), true), null, 1L); @@ -633,7 +635,7 @@ public void testComplexModifyIgnoresUnchangedMappedChildren() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("outer.payload"), new Column("payload", mappedPayloadDorisType(Type.BIGINT, 8, @@ -661,7 +663,7 @@ public void testComplexModifyRejectsChangedUnsupportedMappedChildrenBeforeUpdate try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("outer.payload"), new Column("payload", mappedPayloadDorisType(Type.LARGEINT, 8, @@ -692,7 +694,7 @@ public void testLegacyModifyColumnTreatsNullabilityAsExplicit() throws Throwable try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); // Iceberg schema columns are represented as keys in Doris, so the legacy API must not // interpret isKey as an explicit KEY clause. @@ -723,7 +725,7 @@ public void testLegacyComplexModifyDoesNotInferRecursiveNullableChanges() throws try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, column, null, 1L); } @@ -757,7 +759,7 @@ public void testExplicitNullableModifyMakesRequiredFieldsOptional() throws Throw try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("info"), topLevelColumn, null, 1L); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), nestedColumn, null, 1L); @@ -810,7 +812,7 @@ public void execute(Runnable task) { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(staleTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(staleTable); try { conflictOps.modifyColumn(dorisTable, ColumnPath.of("info"), @@ -861,7 +863,7 @@ public void testRenamePreservesNestedIdentifierFieldPaths() throws Exception { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.renameColumn(dorisTable, ColumnPath.fromDotName("root.child.id"), "renamed_id", 1L); icebergTable.refresh(); @@ -911,7 +913,7 @@ public void testRenameDoesNotRewriteDottedIdentifierSibling() throws Exception { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.renameColumn(dorisTable, "a", "renamed", 1L); icebergTable.refresh(); @@ -940,7 +942,7 @@ public void testNestedColumnOperationsRejectDefaultMetadata() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.addColumn(dorisTable, ColumnPath.fromDotName("s.new_col"), nestedAddDefaultColumn, null, 1L), @@ -971,7 +973,7 @@ public void testTopLevelColumnOperationsRejectUnsupportedDefaultMetadata() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, defaultColumn, null, 1L), "Modifying default values is not supported for Iceberg columns: id"); @@ -1002,7 +1004,7 @@ public void testUnsupportedPrimitiveModifyFailsBeforeUpdateSchema() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn( dorisTable, ColumnPath.of("info"), new Column("info", Type.INT, true), null, 1L), @@ -1035,7 +1037,7 @@ public void testRejectKeyAndGeneratedMetadataBeforeUpdateSchema() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.addColumn(dorisTable, keyColumn, null, 1L), "KEY is not supported for Iceberg ADD/MODIFY COLUMN"); @@ -1082,7 +1084,7 @@ public void testModifyComplexColumnRejectsCaseInsensitiveStructFieldAdditions() try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn( dorisTable, new Column("info", infoType, true), null, 1L), @@ -1158,7 +1160,7 @@ public void testTopLevelCaseInsensitiveCollisionsAndCaseOnlyRename() throws Thro try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.addColumn( dorisTable, new Column("id", Type.STRING, true), null, 1L), @@ -1194,7 +1196,7 @@ public void testReorderColumnsUsesCanonicalIcebergNames() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.reorderColumns(dorisTable, Arrays.asList("label", "id"), 1L); } @@ -1216,7 +1218,7 @@ public void testModifyColumnSupportsDirectArrayElementAndMapValue() throws Throw try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("arr.element"), new Column("element", Type.BIGINT, true), null, 1L); @@ -1239,7 +1241,7 @@ public void testModifyColumnRejectsPositionForDirectArrayElementAndMapValue() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("arr.element"), new Column("element", Type.BIGINT, true), ColumnPosition.FIRST, 1L), @@ -1270,7 +1272,7 @@ public void testModifyColumnCommentUsesCanonicalNestedPaths() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); ops.modifyColumnComment(dorisTable, ColumnPath.fromDotName("info.metric"), "struct comment", 1L); @@ -1296,7 +1298,7 @@ public void testRejectsCommentsOnDirectArrayElementAndMapValue() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumnComment( dorisTable, ColumnPath.fromDotName("arr.element"), "array element comment", 1L), @@ -1336,7 +1338,7 @@ public void testRejectsTopLevelRowLineageMutationsForV3Tables() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); assertUserException(() -> ops.addColumn(dorisTable, new Column("_row_id", Type.BIGINT, true), null, 1L), @@ -1392,8 +1394,10 @@ public void testAllowsV3NestedAndV2TopLevelRowLineageNames() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(v3DorisTable)).thenReturn(v3IcebergTable); - mockedIcebergUtils.when(() -> IcebergUtils.getIcebergTable(v2DorisTable)).thenReturn(v2IcebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(v3DorisTable)) + .thenReturn(v3IcebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(v2DorisTable)) + .thenReturn(v2IcebergTable); ops.addColumn(v3DorisTable, ColumnPath.fromDotName("s._last_updated_sequence_number"), new Column("_last_updated_sequence_number", Type.BIGINT, true), null, 1L); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java index 74c8c3f6954a97..84745f815f828c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java @@ -22,11 +22,28 @@ import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.Collections; import java.util.Map; import java.util.Set; public class IcebergPartitionInfoTest { + @Test + public void testRetainedPayloadCounterTracksSkewedPartitionValues() { + String largeValue = repeatedCharacter('x', 64 * 1024); + IcebergPartition small = new IcebergPartition("p=x", 0, 0, 0, 0, 1, 101, + Collections.singletonList("x"), Collections.singletonList("identity")); + IcebergPartition large = new IcebergPartition("p=" + largeValue, 0, 0, 0, 0, 1, 101, + Collections.singletonList(largeValue), Collections.singletonList("identity")); + + Assertions.assertTrue(large.getRetainedPayloadBytes() - small.getRetainedPayloadBytes() + >= (largeValue.length() - 1L) * 4L); + IcebergPartitionInfo info = new IcebergPartitionInfo( + Collections.emptyMap(), Collections.singletonMap(large.getPartitionName(), large), + Collections.emptyMap()); + Assertions.assertEquals(large.getRetainedPayloadBytes(), info.getRetainedPayloadBytes()); + } + @Test public void testGetLatestSnapshotId() { IcebergPartition p1 = new IcebergPartition("p1", 0, 0, 0, 0, 1, 101, null, null); @@ -50,4 +67,10 @@ public void testGetLatestSnapshotId() { Assertions.assertEquals(102, snapshot2); Assertions.assertEquals(103, snapshot3); } + + private static String repeatedCharacter(char character, int count) { + char[] characters = new char[count]; + java.util.Arrays.fill(characters, character); + return new String(characters); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java index e2d923f3438863..2df4ad7229dbe7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java @@ -202,7 +202,7 @@ public void testPartitionedTable() throws UserException { Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); // Allow parsePartitionValueFromString to call the real implementation mockedStatic.when(() -> IcebergUtils.parsePartitionValueFromString( @@ -318,7 +318,7 @@ public void testUnPartitionedTable() throws UserException { Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithoutPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); IcebergTransaction txn = getTxn(); @@ -429,7 +429,7 @@ public void testUnPartitionedTableOverwriteWithData() throws UserException { Mockito.when(icebergExternalTable.getDbName()).thenReturn(dbName); Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithoutPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); IcebergTransaction txn = getTxn(); @@ -455,7 +455,7 @@ public void testUnpartitionedTableOverwriteWithoutData() throws UserException { Mockito.when(icebergExternalTable.getDbName()).thenReturn(dbName); Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithoutPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); IcebergTransaction txn = getTxn(); @@ -500,7 +500,7 @@ public void testStaticPartitionOverwriteWithoutDataDeletesMatchingPartition() th Mockito.when(icebergExternalTable.getName()).thenReturn(tbWithPartition); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); mockedStatic.when(() -> IcebergUtils.parsePartitionValueFromString( ArgumentMatchers.any(), ArgumentMatchers.any())) @@ -517,7 +517,7 @@ public void testStaticPartitionOverwriteWithoutDataDeletesMatchingPartition() th checkPushDownByPartition(table, Expressions.equal("str1", "partition-b"), 1); try (MockedStatic mockedStatic = Mockito.mockStatic(IcebergUtils.class)) { - mockedStatic.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedStatic.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(table); mockedStatic.when(() -> IcebergUtils.parsePartitionValueFromString( ArgumentMatchers.any(), ArgumentMatchers.any())) @@ -590,7 +590,7 @@ public void testFinishDeleteRewritesAllSharedPuffinDeleteFilesForV3() throws Use try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class); MockedStatic mockedWriterHelper = Mockito.mockStatic(IcebergWriterHelper.class)) { - mockedUtils.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(icebergTable); mockedUtils.when(() -> IcebergUtils.getFileFormat(icebergTable)).thenReturn(FileFormat.PARQUET); mockedUtils.when(() -> IcebergUtils.getFormatVersion(icebergTable)).thenReturn(3); @@ -651,7 +651,7 @@ private void verifyFinishDeleteRewriteBehavior(int formatVersion, boolean expect try (MockedStatic mockedUtils = Mockito.mockStatic(IcebergUtils.class); MockedStatic mockedWriterHelper = Mockito.mockStatic(IcebergWriterHelper.class)) { - mockedUtils.when(() -> IcebergUtils.getIcebergTable(ArgumentMatchers.any(ExternalTable.class))) + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(ArgumentMatchers.any(ExternalTable.class))) .thenReturn(icebergTable); mockedUtils.when(() -> IcebergUtils.getFileFormat(icebergTable)).thenReturn(FileFormat.PARQUET); mockedUtils.when(() -> IcebergUtils.getFormatVersion(icebergTable)).thenReturn(formatVersion); @@ -724,7 +724,7 @@ public void testRetainedGenerationCommitsThroughWritableOperations() throws User try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(liveTable); + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(liveTable); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, retainedTable, Optional.empty()); @@ -736,6 +736,26 @@ public void testRetainedGenerationCommitsThroughWritableOperations() throws User TableIdentifier.of(dbName, tbWithoutPartition)).currentSnapshot()); } + @Test + public void testWeightedTableSupportsSchemaAndPartitionSpecCommits() { + Table liveTable = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); + IcebergTableCacheValue cacheValue = new IcebergTableCacheValue(liveTable); + cacheValue.prepareForCachePublication(NameMapping.createForTest(dbName, tbWithoutPartition)); + + Table writableTable = cacheValue.getWritableIcebergTable(liveTable); + writableTable.updateSchema() + .addColumn("new_col", Types.StringType.get()) + .commit(); + writableTable.updateSpec() + .addField("int1") + .commit(); + + Table refreshed = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); + Assert.assertNotNull(refreshed.schema().findField("new_col")); + Assert.assertEquals(1, refreshed.spec().fields().size()); + Assert.assertEquals("int1", refreshed.spec().fields().get(0).name()); + } + @Test public void testRetainedGenerationRejectsConcurrentMetadataAdvance() throws UserException { Table liveTable = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); @@ -754,7 +774,7 @@ public void testRetainedGenerationRejectsConcurrentMetadataAdvance() throws User try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(liveTable); + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(liveTable); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, retainedTable, Optional.empty()); @@ -783,7 +803,7 @@ public void testRetainedGenerationRetriesAfterConcurrentDataCommit() throws User try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getIcebergTable(dorisTable)).thenReturn(liveTable); + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(liveTable); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, retainedTable, Optional.empty()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java index 6407d540d1ef03..704c82df183e3b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java @@ -28,8 +28,13 @@ import org.junit.Assert; import org.junit.Test; +import java.util.Map; +import java.util.OptionalLong; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.Future; +import java.util.concurrent.TimeUnit; public class AbstractExternalMetaCacheTest { @@ -97,6 +102,34 @@ public void testEntryFailsFastAfterCatalogRemoved() { } } + @Test + public void testCapturedCatalogGroupReturnsClosedEntryDuringConcurrentRemoval() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService workers = Executors.newSingleThreadExecutor(); + CountDownLatch groupCaptured = new CountDownLatch(1); + CountDownLatch releaseEntryLookup = new CountDownLatch(1); + LookupRaceExternalMetaCache cache = + new LookupRaceExternalMetaCache(refreshExecutor, groupCaptured, releaseEntryLookup); + try { + cache.initCatalog(1L, Maps.newHashMap()); + Future> lookup = workers.submit( + () -> cache.entry(1L, "value", String.class, Integer.class)); + Assert.assertTrue(groupCaptured.await(3L, TimeUnit.SECONDS)); + + cache.invalidateCatalog(1L); + releaseEntryLookup.countDown(); + + MetaCacheEntry capturedClosedEntry = lookup.get(3L, TimeUnit.SECONDS); + Assert.assertEquals(Integer.valueOf(1), capturedClosedEntry.get("k")); + Assert.assertNull(capturedClosedEntry.peekIfPresent("k")); + } finally { + releaseEntryLookup.countDown(); + cache.close(); + workers.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + @Test public void testEntryLevelInvalidationUsesRegisteredMatcher() { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); @@ -121,6 +154,125 @@ public void testEntryLevelInvalidationUsesRegisteredMatcher() { } } + @Test + public void testGlobalWeightAutomaticallyActivatesEntriesWithEstimator() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + WeightedExternalMetaCache cache = new WeightedExternalMetaCache( + refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.of(600L))); + try { + cache.initCatalog(1L, Maps.newHashMap()); + MetaCacheEntry entry = cache.entry(1L, "value", String.class, Integer.class); + + Assert.assertTrue(entry.isWeightBounded()); + Assert.assertEquals(600L, entry.stats().getMaxWeight()); + entry.put("first", 60); + entry.put("second", 60); + Assert.assertNull(entry.getIfPresent("first")); + Assert.assertEquals(Integer.valueOf(60), entry.getIfPresent("second")); + Assert.assertEquals(60L + MetaCacheEntry.FIXED_ENTRY_ACCOUNTING_OVERHEAD_BYTES, + entry.stats().getGlobalEstimatedWeight()); + } finally { + cache.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testEntryWeightWithoutEstimatorFailsCatalogInitialization() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + try { + TestExternalMetaCache cache = new TestExternalMetaCache(refreshExecutor); + Map properties = Maps.newHashMap(); + properties.put("meta.cache.test_engine.schema.max-weight", "1KB"); + + IllegalArgumentException exception = Assert.assertThrows( + IllegalArgumentException.class, () -> cache.initCatalog(1L, properties)); + Assert.assertTrue(exception.getMessage().contains("does not support max-weight")); + Assert.assertFalse(cache.isCatalogInitialized(1L)); + } finally { + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testCommonValidationRejectsEntryWeightAboveCatalogWeight() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + WeightedExternalMetaCache cache = new WeightedExternalMetaCache( + refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.of(4L * 1024L))); + try { + Map properties = Maps.newHashMap(); + properties.put(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, "1KB"); + properties.put("meta.cache.weighted_test.value.max-weight", "2KB"); + + IllegalArgumentException exception = Assert.assertThrows( + IllegalArgumentException.class, () -> cache.initCatalog(1L, properties)); + Assert.assertTrue(exception.getMessage().contains("entry max weight")); + Assert.assertFalse(cache.isCatalogInitialized(1L)); + } finally { + cache.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRuntimeInitClampsCatalogAcceptedOnLargerFe() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + WeightedExternalMetaCache cache = new WeightedExternalMetaCache( + refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.of(1024L))); + try { + Map properties = Maps.newHashMap(); + properties.put(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, "4KB"); + properties.put("meta.cache.weighted_test.value.max-weight", "2KB"); + + Assert.assertThrows(IllegalArgumentException.class, + () -> cache.validateCatalogProperties(properties)); + + cache.initCatalog(1L, properties); + + MetaCacheEntryStats stats = cache.stats(1L).get("value"); + Assert.assertEquals(1024L, stats.getMaxWeight()); + Assert.assertEquals(1024L, stats.getCatalogMaxWeight()); + } finally { + cache.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testConcurrentCatalogRemoveAndInitDoesNotDuplicateBudgetScope() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService workers = Executors.newFixedThreadPool(2); + WeightedExternalMetaCache cache = new WeightedExternalMetaCache( + refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.of(100L))); + CountDownLatch start = new CountDownLatch(1); + try { + Future first = workers.submit(() -> repeatedlyRebuildCatalog(cache, start)); + Future second = workers.submit(() -> repeatedlyRebuildCatalog(cache, start)); + start.countDown(); + first.get(10L, TimeUnit.SECONDS); + second.get(10L, TimeUnit.SECONDS); + cache.initCatalog(1L, Maps.newHashMap()); + Assert.assertTrue(cache.isCatalogInitialized(1L)); + } finally { + cache.close(); + workers.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + private static void repeatedlyRebuildCatalog(WeightedExternalMetaCache cache, CountDownLatch start) { + try { + Assert.assertTrue(start.await(3L, TimeUnit.SECONDS)); + for (int i = 0; i < 100; i++) { + cache.initCatalog(1L, Maps.newHashMap()); + cache.invalidateCatalog(1L); + } + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + private static final class TestExternalMetaCache extends AbstractExternalMetaCache { private TestExternalMetaCache(ExecutorService refreshExecutor) { super("test_engine", refreshExecutor); @@ -135,4 +287,44 @@ private TestExternalMetaCache(ExecutorService refreshExecutor) { MetaCacheEntryInvalidation.forNameMapping(SchemaCacheKey::getNameMapping))); } } + + private static final class WeightedExternalMetaCache extends AbstractExternalMetaCache { + private WeightedExternalMetaCache( + ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { + super("weighted_test", refreshExecutor, budgetManager); + registerEntry(MetaCacheEntryDef.of( + "value", + String.class, + Integer.class, + key -> 1, + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L)) + .withSizeEstimator((key, value) -> MetaCacheSizeEstimate.complete(value.longValue()))); + } + } + + private static final class LookupRaceExternalMetaCache extends AbstractExternalMetaCache { + private final CountDownLatch groupCaptured; + private final CountDownLatch releaseEntryLookup; + + private LookupRaceExternalMetaCache(ExecutorService refreshExecutor, + CountDownLatch groupCaptured, CountDownLatch releaseEntryLookup) { + super("lookup_race", refreshExecutor); + this.groupCaptured = groupCaptured; + this.releaseEntryLookup = releaseEntryLookup; + registerEntry(MetaCacheEntryDef.of( + "value", String.class, Integer.class, key -> 1, + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L))); + } + + @Override + void beforeCatalogEntryLookupForTest(long catalogId, String entryName) { + groupCaptured.countDown(); + try { + Assert.assertTrue(releaseEntryLookup.await(3L, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CacheSpecTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CacheSpecTest.java index 05acbb539a26d6..6ea171abd6d717 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CacheSpecTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/CacheSpecTest.java @@ -19,6 +19,7 @@ import org.apache.doris.common.DdlException; +import com.google.common.collect.ImmutableSet; import com.google.common.collect.Maps; import org.junit.Assert; import org.junit.Test; @@ -68,6 +69,7 @@ public void testFromPropertiesWithPropertySpecBuilder() { public void testFromPropertiesWithEngineEntryKeys() { Map properties = Maps.newHashMap(); properties.put("meta.cache.hive.schema.ttl-second", "0"); + properties.put("meta.cache.hive.schema.max-weight", "2KB"); CacheSpec defaultSpec = CacheSpec.fromProperties( Maps.newHashMap(), @@ -79,6 +81,8 @@ public void testFromPropertiesWithEngineEntryKeys() { Assert.assertTrue(spec.isEnable()); Assert.assertEquals(0, spec.getTtlSecond()); Assert.assertEquals(100, spec.getCapacity()); + Assert.assertTrue(spec.isWeightBounded()); + Assert.assertEquals(2048L, spec.getMaxWeight().getAsLong()); } @Test @@ -108,6 +112,7 @@ public void testOfSemantics() { Assert.assertTrue(enabled.isEnable()); Assert.assertEquals(60, enabled.getTtlSecond()); Assert.assertEquals(100, enabled.getCapacity()); + Assert.assertFalse(enabled.isWeightBounded()); CacheSpec zeroTtl = CacheSpec.of(true, 0, 100); Assert.assertTrue(zeroTtl.isEnable()); @@ -147,6 +152,10 @@ public void testIsCacheEnabled() { Assert.assertFalse(CacheSpec.isCacheEnabled(false, CacheSpec.CACHE_NO_TTL, 1)); Assert.assertFalse(CacheSpec.isCacheEnabled(true, 0, 1)); Assert.assertFalse(CacheSpec.isCacheEnabled(true, CacheSpec.CACHE_NO_TTL, 0)); + Assert.assertFalse(CacheSpec.ofWeight( + true, CacheSpec.CACHE_NO_TTL, 0L, 1L).isCacheEnabled()); + Assert.assertFalse(CacheSpec.ofWeight( + true, CacheSpec.CACHE_NO_TTL, 1L, 0L).isCacheEnabled()); } @Test @@ -166,4 +175,51 @@ public void testToExpireAfterAccess() { Assert.assertTrue(negativeOther.isPresent()); Assert.assertEquals(0, negativeOther.getAsLong()); } + + @Test + public void testParseWeight() { + Assert.assertEquals(1L, CacheSpec.parseWeight("1", "weight", false, 0L)); + Assert.assertEquals(1024L, CacheSpec.parseWeight("1KB", "weight", false, 0L)); + Assert.assertEquals(2L * 1024L * 1024L, + CacheSpec.parseWeight("2 mb", "weight", false, 0L)); + Assert.assertEquals(250L, CacheSpec.parseWeight("25%", "weight", true, 1000L)); + Assert.assertEquals(0L, CacheSpec.parseWeight("0", "weight", false, 0L)); + + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.parseWeight("1.5GB", "weight", false, 0L)); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.parseWeight("101%", "weight", true, 1000L)); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.parseWeight("1PB000", "weight", false, 0L)); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.parseWeight("999999999999999999PB", "weight", false, 0L)); + } + + @Test + public void testStrictEnginePropertyAllowlist() { + Map properties = Maps.newHashMap(); + properties.put("meta.cache.hive.partition_values.enable", "true"); + properties.put("meta.cache.hive.partition_values.ttl-second", "-1"); + properties.put("meta.cache.hive.partition_values.capacity", "10"); + properties.put("meta.cache.hive.partition_values.max-weight", "2MB"); + CacheSpec.validateEngineProperties(properties, "hive", + ImmutableSet.of("partition_values", "schema"), ImmutableSet.of("partition_values")); + + properties.put("meta.cache.hive.partiton_values.capacity", "10"); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.validateEngineProperties(properties, "hive", + ImmutableSet.of("partition_values", "schema"), ImmutableSet.of("partition_values"))); + properties.remove("meta.cache.hive.partiton_values.capacity"); + + properties.put("meta.cache.hive.partition_values.enabel", "true"); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.validateEngineProperties(properties, "hive", + ImmutableSet.of("partition_values", "schema"), ImmutableSet.of("partition_values"))); + properties.remove("meta.cache.hive.partition_values.enabel"); + + properties.put("meta.cache.hive.schema.max-weight", "1MB"); + Assert.assertThrows(IllegalArgumentException.class, + () -> CacheSpec.validateEngineProperties(properties, "hive", + ImmutableSet.of("partition_values", "schema"), ImmutableSet.of("partition_values"))); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java new file mode 100644 index 00000000000000..4d35dd15fd63b4 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java @@ -0,0 +1,191 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.apache.doris.common.Config; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.AdmissionReservation; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.EntryBudget; + +import org.junit.Assert; +import org.junit.Test; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; + +public class ExternalMetaCacheBudgetManagerTest { + + @Test + public void testGlobalCatalogAndEntryLimits() { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget first = manager.createEntryBudget( + 1L, "hive", "file", OptionalLong.of(80L), OptionalLong.of(60L)); + EntryBudget second = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.of(80L), OptionalLong.of(50L)); + + AdmissionReservation firstReservation = first.tryReserve(60L).get(); + Assert.assertFalse(second.tryReserve(30L).isPresent()); + AdmissionReservation secondReservation = second.tryReserve(20L).get(); + Assert.assertEquals(80L, manager.getGlobalUsedWeight()); + Assert.assertFalse(secondReservation.tryResize(30L)); + + firstReservation.release(); + Assert.assertTrue(secondReservation.tryResize(30L)); + Assert.assertEquals(30L, manager.getGlobalUsedWeight()); + + secondReservation.release(); + first.close(); + second.close(); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } + + @Test + public void testConcurrentReservationNeverExceedsGlobalLimit() throws Exception { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget budget = manager.createEntryBudget( + 1L, "iceberg", "manifest", OptionalLong.empty(), OptionalLong.empty()); + ExecutorService executor = Executors.newFixedThreadPool(8); + CountDownLatch start = new CountDownLatch(1); + List reservations = Collections.synchronizedList(new ArrayList<>()); + try { + for (int i = 0; i < 200; i++) { + executor.submit(() -> { + await(start); + Optional reservation = budget.tryReserve(1L); + reservation.ifPresent(reservations::add); + }); + } + start.countDown(); + executor.shutdown(); + Assert.assertTrue(executor.awaitTermination(10L, TimeUnit.SECONDS)); + Assert.assertEquals(100, reservations.size()); + Assert.assertEquals(100L, manager.getGlobalUsedWeight()); + } finally { + executor.shutdownNow(); + reservations.forEach(AdmissionReservation::release); + budget.close(); + } + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } + + @Test + public void testRejectChildLargerThanParent() { + ExternalMetaCacheBudgetManager manager = manager(100L); + Assert.assertThrows(IllegalArgumentException.class, () -> manager.createEntryBudget( + 1L, "hive", "file", OptionalLong.of(80L), OptionalLong.of(90L))); + + Map properties = new HashMap<>(); + properties.put(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, "120"); + Assert.assertEquals(120L, manager.parseCatalogMaxWeight(properties).getAsLong()); + Assert.assertThrows(IllegalArgumentException.class, () -> manager.validateCatalogMaxWeight(properties)); + } + + @Test + public void testRuntimeBudgetClampsReplayedCatalogToLocalGlobalLimit() { + ExternalMetaCacheBudgetManager observerManager = manager(100L); + + EntryBudget budget = observerManager.createEntryBudget( + 1L, "iceberg", "table", OptionalLong.of(400L), OptionalLong.of(300L)); + + Assert.assertEquals(100L, budget.getEffectiveMaxWeight()); + Assert.assertEquals(100L, budget.getCatalogMaxWeight()); + AdmissionReservation reservation = budget.tryReserve(100L).get(); + Assert.assertFalse(budget.tryReserve(1L).isPresent()); + reservation.release(); + budget.close(); + } + + @Test + public void testGlobalConfigSupportsPercentageAndDisabledZero() { + String original = Config.external_meta_cache_max_weight; + try { + Config.external_meta_cache_max_weight = "25%"; + ExternalMetaCacheBudgetManager percentageManager = ExternalMetaCacheBudgetManager.fromConfig(); + Assert.assertEquals(Runtime.getRuntime().maxMemory() / 4L, + percentageManager.getGlobalMaxWeight().getAsLong()); + + Config.external_meta_cache_max_weight = "0"; + Assert.assertFalse(ExternalMetaCacheBudgetManager.fromConfig().getGlobalMaxWeight().isPresent()); + + Config.external_meta_cache_max_weight = "0%"; + Assert.assertThrows(IllegalArgumentException.class, ExternalMetaCacheBudgetManager::fromConfig); + } finally { + Config.external_meta_cache_max_weight = original; + } + } + + @Test + public void testReservationReleaseIsIdempotent() { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget budget = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + AdmissionReservation reservation = budget.tryReserve(40L).get(); + + reservation.release(); + reservation.release(); + + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + budget.close(); + } + + @Test + public void testClosedBudgetRejectsStaleHandleAndReservationResize() { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget staleBudget = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + AdmissionReservation zeroByteReservation = staleBudget.tryReserve(0L).get(); + + staleBudget.close(); + staleBudget.close(); + + Assert.assertFalse(staleBudget.tryReserve(1L).isPresent()); + Assert.assertFalse(zeroByteReservation.tryResize(1L)); + Assert.assertEquals(0L, staleBudget.getRejectedCount()); + Assert.assertEquals(0L, manager.getGlobalRejectedCount()); + zeroByteReservation.release(); + + EntryBudget replacement = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + AdmissionReservation replacementReservation = replacement.tryReserve(100L).get(); + Assert.assertEquals(100L, manager.getGlobalUsedWeight()); + replacementReservation.release(); + replacement.close(); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } + + private static ExternalMetaCacheBudgetManager manager(long maxWeight) { + return new ExternalMetaCacheBudgetManager(OptionalLong.of(maxWeight)); + } + + private static void await(CountDownLatch latch) { + try { + Assert.assertTrue(latch.await(3L, TimeUnit.SECONDS)); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + throw new RuntimeException(e); + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java index ef1090dd5300c6..fa013d58f4a653 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java @@ -20,17 +20,27 @@ import org.apache.doris.common.Config; import com.github.benmanes.caffeine.cache.LoadingCache; +import com.github.benmanes.caffeine.cache.RemovalCause; import com.google.common.collect.Maps; import org.junit.Assert; import org.junit.Test; +import java.lang.ref.Reference; +import java.lang.ref.SoftReference; +import java.lang.ref.WeakReference; import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.util.Arrays; import java.util.Map; +import java.util.OptionalLong; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.Future; +import java.util.concurrent.LinkedBlockingQueue; +import java.util.concurrent.ThreadPoolExecutor; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; public class MetaCacheEntryTest { @@ -311,6 +321,876 @@ void beforeManualCachePutForTest(String key, Integer loaded) { } } + @Test + public void testExplicitPutWinsAgainstInFlightManualLoad() throws Exception { + boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; + Config.enable_external_meta_cache_manual_miss_load = true; + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newSingleThreadExecutor(); + CountDownLatch beforePutStarted = new CountDownLatch(1); + CountDownLatch releaseBeforePut = new CountDownLatch(1); + try { + MetaCacheEntry entry = new MetaCacheEntry( + "test", key -> 1, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false) { + @Override + void beforeManualCachePutForTest(String key, Integer loaded) { + beforePutStarted.countDown(); + awaitLatch(releaseBeforePut); + } + }; + + Future load = queryExecutor.submit(() -> entry.get("k")); + Assert.assertTrue(beforePutStarted.await(3L, TimeUnit.SECONDS)); + entry.put("k", 2); + releaseBeforePut.countDown(); + + Assert.assertEquals(Integer.valueOf(1), load.get(3L, TimeUnit.SECONDS)); + Assert.assertEquals(Integer.valueOf(2), entry.peekIfPresent("k")); + } finally { + releaseBeforePut.countDown(); + Config.enable_external_meta_cache_manual_miss_load = originalManualMissLoad; + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testWeightedReplacementDoesNotQueueOldValuesOnRefreshExecutor() throws Exception { + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + ThreadPoolExecutor refreshExecutor = new ThreadPoolExecutor( + 1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>()); + ExternalMetaCacheBudgetManager budgetManager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(1_000L)); + ExternalMetaCacheBudgetManager.EntryBudget entryBudget = budgetManager.createEntryBudget( + 1L, "test", "value", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "value", key -> new byte[1], CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), entryBudget); + try { + refreshExecutor.execute(() -> { + workerBlocked.countDown(); + awaitLatch(releaseWorker); + }); + Assert.assertTrue(workerBlocked.await(3L, TimeUnit.SECONDS)); + + entry.put("k", new byte[100]); + for (int i = 0; i < 100; i++) { + entry.put("k", new byte[100]); + } + + Assert.assertTrue("removal callbacks must not retain replaced values in the executor queue", + refreshExecutor.getQueue().isEmpty()); + Assert.assertEquals(accountedWeight(100L), entry.stats().getEstimatedWeight()); + } finally { + releaseWorker.countDown(); + entry.close(); + refreshExecutor.shutdown(); + Assert.assertTrue(refreshExecutor.awaitTermination(3L, TimeUnit.SECONDS)); + } + } + + @Test + public void testQueuedWeightedRefreshDoesNotCaptureCurrentValue() throws Exception { + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + ThreadPoolExecutor refreshExecutor = new ThreadPoolExecutor( + 1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>()); + AtomicInteger loaderCalls = new AtomicInteger(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "refresh-capture", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "refresh-capture", key -> { + loaderCalls.incrementAndGet(); + return new byte[2]; + }, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + refreshExecutor.execute(() -> { + workerBlocked.countDown(); + awaitLatch(releaseWorker); + }); + Assert.assertTrue(workerBlocked.await(3L, TimeUnit.SECONDS)); + + byte[] currentValue = new byte[1]; + entry.put("k", currentValue); + entry.triggerRefreshForTest("k"); + + Runnable queuedRefresh = refreshExecutor.getQueue().peek(); + Assert.assertNotNull(queuedRefresh); + for (Field field : queuedRefresh.getClass().getDeclaredFields()) { + field.setAccessible(true); + Assert.assertNotSame("queued refresh must not directly retain the cached value", + currentValue, field.get(queuedRefresh)); + } + + entry.invalidateAll(); + releaseWorker.countDown(); + refreshExecutor.shutdown(); + Assert.assertTrue(refreshExecutor.awaitTermination(3L, TimeUnit.SECONDS)); + Assert.assertEquals(0, loaderCalls.get()); + } finally { + releaseWorker.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testIdentityConditionalFenceSuppressesOlderWeightedRefresh() throws Exception { + CountDownLatch workerBlocked = new CountDownLatch(1); + CountDownLatch releaseWorker = new CountDownLatch(1); + ThreadPoolExecutor refreshExecutor = new ThreadPoolExecutor( + 1, 1, 0L, TimeUnit.MILLISECONDS, new LinkedBlockingQueue<>()); + AtomicInteger loaderCalls = new AtomicInteger(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "refresh-fence", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "refresh-fence", key -> { + loaderCalls.incrementAndGet(); + return new byte[2]; + }, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + refreshExecutor.execute(() -> { + workerBlocked.countDown(); + awaitLatch(releaseWorker); + }); + Assert.assertTrue(workerBlocked.await(3L, TimeUnit.SECONDS)); + + byte[] currentValue = new byte[1]; + entry.put("k", currentValue); + entry.triggerRefreshForTest("k"); + Assert.assertFalse(refreshExecutor.getQueue().isEmpty()); + + Assert.assertTrue(entry.fenceInFlightLoadIfSame("k", currentValue)); + Assert.assertSame(currentValue, entry.peekIfPresent("k")); + releaseWorker.countDown(); + refreshExecutor.shutdown(); + Assert.assertTrue(refreshExecutor.awaitTermination(3L, TimeUnit.SECONDS)); + + Assert.assertEquals("the older refresh must be rejected before it calls the loader", + 0, loaderCalls.get()); + Assert.assertSame(currentValue, entry.peekIfPresent("k")); + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + } finally { + releaseWorker.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testIdentityConditionalFenceSuppressesOlderCountRefresh() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + CountDownLatch loaderEntered = new CountDownLatch(1); + CountDownLatch releaseLoader = new CountDownLatch(1); + CountDownLatch loaderFinished = new CountDownLatch(1); + MetaCacheEntry entry = new MetaCacheEntry<>( + "count-refresh-fence", key -> { + loaderEntered.countDown(); + awaitLatch(releaseLoader); + loaderFinished.countDown(); + return "stale"; + }, + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length()), null); + try { + String currentValue = new String("current"); + entry.put("k", currentValue); + entry.triggerRefreshForTest("k"); + Assert.assertTrue(loaderEntered.await(3L, TimeUnit.SECONDS)); + + Assert.assertTrue(entry.fenceInFlightLoadIfSame("k", currentValue)); + Assert.assertSame(currentValue, entry.peekIfPresent("k")); + releaseLoader.countDown(); + Assert.assertTrue(loaderFinished.await(3L, TimeUnit.SECONDS)); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + Assert.assertSame("the event fence must suppress refresh write-back", + currentValue, entry.peekIfPresent("k")); + } finally { + releaseLoader.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testConcurrentWeightedRefreshesForDifferentKeysDoNotFenceEachOther() throws Exception { + ExecutorService refreshExecutor = Executors.newFixedThreadPool(2); + CountDownLatch loadersEntered = new CountDownLatch(2); + CountDownLatch releaseLoaders = new CountDownLatch(1); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(4_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "multi-key-weighted-refresh", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "multi-key-weighted-refresh", key -> { + loadersEntered.countDown(); + awaitLatch(releaseLoaders); + return new byte["a".equals(key) ? 2 : 3]; + }, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 4_000L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + entry.put("a", new byte[1]); + entry.put("b", new byte[1]); + entry.triggerRefreshForTest("a"); + entry.triggerRefreshForTest("b"); + Assert.assertTrue(loadersEntered.await(3L, TimeUnit.SECONDS)); + + releaseLoaders.countDown(); + awaitValueLength(entry, "a", 2); + awaitValueLength(entry, "b", 3); + Assert.assertEquals(accountedWeight(2L) + accountedWeight(3L), manager.getGlobalUsedWeight()); + } finally { + releaseLoaders.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testConcurrentCountRefreshesForDifferentKeysDoNotFenceEachOther() throws Exception { + ExecutorService refreshExecutor = Executors.newFixedThreadPool(2); + CountDownLatch loadersEntered = new CountDownLatch(2); + CountDownLatch releaseLoaders = new CountDownLatch(1); + MetaCacheEntry entry = new MetaCacheEntry<>( + "multi-key-count-refresh", key -> { + loadersEntered.countDown(); + awaitLatch(releaseLoaders); + return key + "-refreshed"; + }, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length()), null); + try { + entry.put("a", "a-current"); + entry.put("b", "b-current"); + entry.triggerRefreshForTest("a"); + entry.triggerRefreshForTest("b"); + Assert.assertTrue(loadersEntered.await(3L, TimeUnit.SECONDS)); + + releaseLoaders.countDown(); + awaitValue(entry, "a", "a-refreshed"); + awaitValue(entry, "b", "b-refreshed"); + } finally { + releaseLoaders.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testInvalidatingOneKeyDoesNotSuppressAnotherKeysConcurrentMissAdmission() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + CountDownLatch loadersEntered = new CountDownLatch(2); + CountDownLatch releaseLoaders = new CountDownLatch(1); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(4_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "multi-key-miss", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "multi-key-miss", key -> { + loadersEntered.countDown(); + awaitLatch(releaseLoaders); + return new byte[1]; + }, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 4_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + Future first = queryExecutor.submit(() -> entry.get("a")); + Future second = queryExecutor.submit(() -> entry.get("b")); + Assert.assertTrue(loadersEntered.await(3L, TimeUnit.SECONDS)); + + entry.invalidateKey("a"); + releaseLoaders.countDown(); + first.get(3L, TimeUnit.SECONDS); + second.get(3L, TimeUnit.SECONDS); + + Assert.assertNull(entry.peekIfPresent("a")); + Assert.assertNotNull(entry.peekIfPresent("b")); + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + } finally { + releaseLoaders.countDown(); + entry.close(); + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRemovalCleanupDoesNotDeadlockWithInvalidateAll() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "deadlock", OptionalLong.empty(), OptionalLong.empty()); + CountDownLatch removalListenerEntered = new CountDownLatch(1); + CountDownLatch invalidateHasAdmissionLock = new CountDownLatch(1); + CountDownLatch releaseRemovalListener = new CountDownLatch(1); + AtomicBoolean armRemovalHook = new AtomicBoolean(false); + MetaCacheEntry entry = new MetaCacheEntry( + "deadlock", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget) { + @Override + void beforeRemovalReleaseForTest(String key) { + if (armRemovalHook.compareAndSet(true, false)) { + removalListenerEntered.countDown(); + awaitLatch(releaseRemovalListener); + } + } + + @Override + void beforeWeightedInvalidateAllForTest() { + invalidateHasAdmissionLock.countDown(); + } + }; + try { + entry.put("k", new byte[1]); + LoadingCache loadingCache = extractLoadingCache(entry); + armRemovalHook.set(true); + + Future eviction = queryExecutor.submit( + () -> loadingCache.policy().eviction().get().setMaximum(1L)); + Assert.assertTrue(removalListenerEntered.await(3L, TimeUnit.SECONDS)); + Future invalidate = queryExecutor.submit(entry::invalidateAll); + Assert.assertTrue(invalidateHasAdmissionLock.await(3L, TimeUnit.SECONDS)); + + releaseRemovalListener.countDown(); + eviction.get(3L, TimeUnit.SECONDS); + invalidate.get(3L, TimeUnit.SECONDS); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + releaseRemovalListener.countDown(); + entry.close(); + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testDelayedRemovalDoesNotReleaseSameIdentityReinsert() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "same-identity-aba", OptionalLong.empty(), OptionalLong.empty()); + CountDownLatch oldRemovalBeforeOwnerSnapshot = new CountDownLatch(1); + CountDownLatch replacementOwnerPublished = new CountDownLatch(1); + CountDownLatch releaseOldRemoval = new CountDownLatch(1); + CountDownLatch releaseReplacementPut = new CountDownLatch(1); + CountDownLatch cleanupBeforeAdmissionLock = new CountDownLatch(1); + CountDownLatch oldRemovalCleanupFinished = new CountDownLatch(1); + AtomicBoolean armRemovalHook = new AtomicBoolean(false); + AtomicBoolean armPutHook = new AtomicBoolean(false); + MetaCacheEntry entry = new MetaCacheEntry( + "same-identity-aba", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget) { + @Override + void beforeRemovalOwnerSnapshotForTest(String key) { + if (armRemovalHook.compareAndSet(true, false)) { + oldRemovalBeforeOwnerSnapshot.countDown(); + awaitLatch(releaseOldRemoval); + } + } + + @Override + void beforeWeightedCachePutForTest(String key, byte[] value) { + if (armPutHook.compareAndSet(true, false)) { + replacementOwnerPublished.countDown(); + awaitLatch(releaseReplacementPut); + } + } + + @Override + void beforeRemovalCleanupLockForTest(String key) { + cleanupBeforeAdmissionLock.countDown(); + } + + @Override + void afterRemovalCleanupForTest(String key) { + oldRemovalCleanupFinished.countDown(); + } + }; + try { + byte[] sameValue = new byte[1]; + entry.put("k", sameValue); + LoadingCache loadingCache = extractLoadingCache(entry); + armRemovalHook.set(true); + + Future oldRemoval = queryExecutor.submit(() -> loadingCache.invalidate("k")); + Assert.assertTrue(oldRemovalBeforeOwnerSnapshot.await(3L, TimeUnit.SECONDS)); + armPutHook.set(true); + Future reinsert = queryExecutor.submit(() -> entry.put("k", sameValue)); + Assert.assertTrue(replacementOwnerPublished.await(3L, TimeUnit.SECONDS)); + + releaseOldRemoval.countDown(); + oldRemoval.get(3L, TimeUnit.SECONDS); + Assert.assertTrue(cleanupBeforeAdmissionLock.await(3L, TimeUnit.SECONDS)); + Assert.assertFalse("cleanup must wait for the publishing admission critical section", + reinsert.isDone()); + releaseReplacementPut.countDown(); + reinsert.get(3L, TimeUnit.SECONDS); + Assert.assertTrue(oldRemovalCleanupFinished.await(3L, TimeUnit.SECONDS)); + Assert.assertSame(sameValue, entry.peekIfPresent("k")); + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + } finally { + releaseOldRemoval.countDown(); + releaseReplacementPut.countDown(); + entry.close(); + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testDelayedCountRemovalDoesNotDropSameIdentityRefreshOwner() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + CountDownLatch oldRemovalBeforeOwnerSnapshot = new CountDownLatch(1); + CountDownLatch replacementOwnerPublished = new CountDownLatch(1); + CountDownLatch releaseOldRemoval = new CountDownLatch(1); + CountDownLatch releaseReplacementPut = new CountDownLatch(1); + CountDownLatch cleanupBeforeAdmissionLock = new CountDownLatch(1); + CountDownLatch oldRemovalCleanupFinished = new CountDownLatch(1); + AtomicBoolean armRemovalHook = new AtomicBoolean(false); + AtomicBoolean armPutHook = new AtomicBoolean(false); + AtomicInteger loaderCalls = new AtomicInteger(); + byte[] sameValue = new byte[1]; + MetaCacheEntry entry = new MetaCacheEntry( + "count-same-identity-aba", key -> { + loaderCalls.incrementAndGet(); + return sameValue; + }, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), null) { + @Override + void beforeRemovalOwnerSnapshotForTest(String key) { + if (armRemovalHook.compareAndSet(true, false)) { + oldRemovalBeforeOwnerSnapshot.countDown(); + awaitLatch(releaseOldRemoval); + } + } + + @Override + void beforeNonWeightedCachePutForTest(String key, byte[] value) { + if (armPutHook.compareAndSet(true, false)) { + replacementOwnerPublished.countDown(); + awaitLatch(releaseReplacementPut); + } + } + + @Override + void beforeRemovalCleanupLockForTest(String key) { + cleanupBeforeAdmissionLock.countDown(); + } + + @Override + void afterRemovalCleanupForTest(String key) { + oldRemovalCleanupFinished.countDown(); + } + }; + try { + entry.put("k", sameValue); + LoadingCache loadingCache = extractLoadingCache(entry); + armRemovalHook.set(true); + + Future oldRemoval = queryExecutor.submit(() -> loadingCache.invalidate("k")); + Assert.assertTrue(oldRemovalBeforeOwnerSnapshot.await(3L, TimeUnit.SECONDS)); + armPutHook.set(true); + Future reinsert = queryExecutor.submit(() -> entry.put("k", sameValue)); + Assert.assertTrue(replacementOwnerPublished.await(3L, TimeUnit.SECONDS)); + + releaseOldRemoval.countDown(); + oldRemoval.get(3L, TimeUnit.SECONDS); + Assert.assertTrue(cleanupBeforeAdmissionLock.await(3L, TimeUnit.SECONDS)); + Assert.assertFalse("cleanup must wait for the publishing admission critical section", + reinsert.isDone()); + releaseReplacementPut.countDown(); + reinsert.get(3L, TimeUnit.SECONDS); + Assert.assertTrue(oldRemovalCleanupFinished.await(3L, TimeUnit.SECONDS)); + Assert.assertSame(sameValue, entry.peekIfPresent("k")); + + entry.triggerRefreshForTest("k"); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + Assert.assertEquals("the replacement refresh owner must remain usable", 1, loaderCalls.get()); + Assert.assertSame(sameValue, entry.peekIfPresent("k")); + } finally { + releaseOldRemoval.countDown(); + releaseReplacementPut.countDown(); + entry.close(); + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testExpiredSameIdentityCallbackKeepsCurrentCountRefreshOwner() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + AtomicInteger loaderCalls = new AtomicInteger(); + byte[] sameValue = new byte[1]; + MetaCacheEntry entry = new MetaCacheEntry<>( + "count-expired-same-identity", key -> { + loaderCalls.incrementAndGet(); + return sameValue; + }, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), null); + try { + entry.put("k", sameValue); + entry.put("k", sameValue); + entry.notifyRemovalUnderAdmissionLockForTest("k", sameValue, RemovalCause.EXPIRED); + + entry.triggerRefreshForTest("k"); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + Assert.assertEquals("EXPIRED callback for the old mapping must retain the new refresh owner", + 1, loaderCalls.get()); + Assert.assertSame(sameValue, entry.peekIfPresent("k")); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testExpiredSameIdentityCallbackKeepsCurrentWeightedReservation() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted-expired-same-identity", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted-expired-same-identity", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + byte[] sameValue = new byte[1]; + entry.put("k", sameValue); + entry.put("k", sameValue); + entry.notifyRemovalUnderAdmissionLockForTest("k", sameValue, RemovalCause.EXPIRED); + + Assert.assertSame(sameValue, entry.peekIfPresent("k")); + Assert.assertEquals("EXPIRED callback for the old mapping must retain the new reservation", + accountedWeight(1L), manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testWeightedCacheUsesSoftValuesAndReleasesCollectedReservation() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "soft-value", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "soft-value", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + byte[] value = new byte[1]; + entry.put("k", value); + LoadingCache loadingCache = extractLoadingCache(entry); + Reference valueReference = extractValueReference(loadingCache); + + Assert.assertTrue("weighted values must be held through Caffeine SoftReference", + valueReference instanceof SoftReference); + Map owners = (Map) readField(entry, "reservations"); + Object owner = owners.get("k"); + Assert.assertNotNull(owner); + for (Field field : owner.getClass().getDeclaredFields()) { + field.setAccessible(true); + Assert.assertNotSame("reservation ownership must not strongly retain V", value, + field.get(owner)); + } + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + + valueReference.clear(); + Assert.assertTrue(valueReference.enqueue()); + loadingCache.cleanUp(); + + awaitGlobalWeight(manager, 0L); + Assert.assertNull(entry.peekIfPresent("k")); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testStrongQueryReferenceSurvivesSoftValueCollectionChecks() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "query-reference", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "query-reference", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + byte[] queryReference = entry.get("k"); + WeakReference observed = new WeakReference<>(queryReference); + + for (int i = 0; i < 3; i++) { + System.gc(); + extractLoadingCache(entry).cleanUp(); + } + + Assert.assertSame(queryReference, observed.get()); + Assert.assertSame(queryReference, entry.peekIfPresent("k")); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testDelayedCollectedCallbackCannotReleaseReplacementGeneration() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "collected-aba", OptionalLong.empty(), OptionalLong.empty()); + CountDownLatch collectedBeforeOwnerSnapshot = new CountDownLatch(1); + CountDownLatch replacementOwnerPublished = new CountDownLatch(1); + CountDownLatch releaseCollectedCallback = new CountDownLatch(1); + CountDownLatch releaseReplacementPut = new CountDownLatch(1); + CountDownLatch collectedCleanupFinished = new CountDownLatch(1); + AtomicBoolean armRemovalHook = new AtomicBoolean(false); + AtomicBoolean armPutHook = new AtomicBoolean(false); + MetaCacheEntry entry = new MetaCacheEntry( + "collected-aba", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget) { + @Override + void beforeRemovalOwnerSnapshotForTest(String key) { + if (armRemovalHook.compareAndSet(true, false)) { + collectedBeforeOwnerSnapshot.countDown(); + awaitLatch(releaseCollectedCallback); + } + } + + @Override + void beforeWeightedCachePutForTest(String key, byte[] value) { + if (armPutHook.compareAndSet(true, false)) { + replacementOwnerPublished.countDown(); + awaitLatch(releaseReplacementPut); + } + } + + @Override + void afterRemovalCleanupForTest(String key) { + collectedCleanupFinished.countDown(); + } + }; + try { + entry.put("k", new byte[1]); + LoadingCache loadingCache = extractLoadingCache(entry); + Reference oldValueReference = extractValueReference(loadingCache); + armRemovalHook.set(true); + + Future collection = queryExecutor.submit(() -> { + oldValueReference.clear(); + oldValueReference.enqueue(); + loadingCache.cleanUp(); + }); + Assert.assertTrue(collectedBeforeOwnerSnapshot.await(3L, TimeUnit.SECONDS)); + + byte[] replacement = new byte[1]; + armPutHook.set(true); + Future replacementPut = queryExecutor.submit(() -> entry.put("k", replacement)); + Assert.assertTrue(replacementOwnerPublished.await(3L, TimeUnit.SECONDS)); + + releaseCollectedCallback.countDown(); + collection.get(3L, TimeUnit.SECONDS); + releaseReplacementPut.countDown(); + replacementPut.get(3L, TimeUnit.SECONDS); + Assert.assertTrue(collectedCleanupFinished.await(3L, TimeUnit.SECONDS)); + + Assert.assertSame(replacement, entry.peekIfPresent("k")); + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + } finally { + releaseCollectedCallback.countDown(); + releaseReplacementPut.countDown(); + entry.close(); + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testOwnershipRecordsHaveNoGenericValueReference() { + for (Class nested : MetaCacheEntry.class.getDeclaredClasses()) { + if (nested.getSimpleName().equals("ReservationRecord") + || nested.getSimpleName().equals("RefreshRecord")) { + Assert.assertFalse(nested.getSimpleName() + " must not retain V", + Arrays.stream(nested.getDeclaredFields()) + .anyMatch(field -> field.getType() == Object.class)); + } + } + } + + @Test + public void testRemovalCleanupRetriesAfterTransientFailure() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "removal-retry", OptionalLong.empty(), OptionalLong.empty()); + CountDownLatch firstAttempt = new CountDownLatch(1); + CountDownLatch cleanupFinished = new CountDownLatch(1); + AtomicInteger attempts = new AtomicInteger(); + MetaCacheEntry entry = new MetaCacheEntry( + "removal-retry", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2_000L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget) { + @Override + void beforeRemovalCleanupLockForTest(String key) { + if (attempts.incrementAndGet() == 1) { + firstAttempt.countDown(); + throw new IllegalStateException("transient cleanup failure"); + } + } + + @Override + void afterRemovalCleanupForTest(String key) { + cleanupFinished.countDown(); + } + }; + try { + entry.put("k", new byte[1]); + extractLoadingCache(entry).invalidate("k"); + + Assert.assertTrue(firstAttempt.await(3L, TimeUnit.SECONDS)); + Assert.assertTrue(cleanupFinished.await(3L, TimeUnit.SECONDS)); + Assert.assertTrue("cleanup should be retried", attempts.get() >= 2); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testBulkInvalidateDoesNotEnqueueOneCleanupPerEntry() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + long maxWeight = 1_000L * accountedWeight(1L); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(maxWeight)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "bulk-invalidate", OptionalLong.empty(), OptionalLong.empty()); + AtomicInteger queuedCleanupCount = new AtomicInteger(); + MetaCacheEntry entry = new MetaCacheEntry( + "bulk-invalidate", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 2_000L, maxWeight), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget) { + @Override + void beforeRemovalReleaseForTest(String key) { + queuedCleanupCount.incrementAndGet(); + } + }; + try { + for (int i = 0; i < 1_000; i++) { + entry.put("k-" + i, new byte[1]); + } + + entry.invalidateAll(); + + Assert.assertEquals(0, queuedCleanupCount.get()); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testNonWeightedInvalidateLinearizesWithFinalManualPut() throws Exception { + boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; + Config.enable_external_meta_cache_manual_miss_load = true; + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + CountDownLatch insideFinalPut = new CountDownLatch(1); + CountDownLatch releaseFinalPut = new CountDownLatch(1); + try { + MetaCacheEntry entry = new MetaCacheEntry( + "test", key -> 1, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false) { + @Override + void beforeNonWeightedManualCachePutForTest(String key, Integer loaded) { + insideFinalPut.countDown(); + awaitLatch(releaseFinalPut); + } + }; + + Future load = queryExecutor.submit(() -> entry.get("k")); + Assert.assertTrue(insideFinalPut.await(3L, TimeUnit.SECONDS)); + Future invalidate = queryExecutor.submit(() -> entry.invalidateKey("k")); + Assert.assertFalse("invalidation must wait for the final put linearization point", + invalidate.isDone()); + releaseFinalPut.countDown(); + + Assert.assertEquals(Integer.valueOf(1), load.get(3L, TimeUnit.SECONDS)); + invalidate.get(3L, TimeUnit.SECONDS); + Assert.assertNull(entry.peekIfPresent("k")); + } finally { + releaseFinalPut.countDown(); + Config.enable_external_meta_cache_manual_miss_load = originalManualMissLoad; + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testNonWeightedInvalidateAllLinearizesWithFinalManualPut() throws Exception { + boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; + Config.enable_external_meta_cache_manual_miss_load = true; + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newFixedThreadPool(2); + CountDownLatch insideFinalPut = new CountDownLatch(1); + CountDownLatch releaseFinalPut = new CountDownLatch(1); + try { + MetaCacheEntry entry = new MetaCacheEntry( + "test", key -> 1, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false) { + @Override + void beforeNonWeightedManualCachePutForTest(String key, Integer loaded) { + insideFinalPut.countDown(); + awaitLatch(releaseFinalPut); + } + }; + + Future load = queryExecutor.submit(() -> entry.get("k")); + Assert.assertTrue(insideFinalPut.await(3L, TimeUnit.SECONDS)); + Future invalidate = queryExecutor.submit(entry::invalidateAll); + Assert.assertFalse("invalidation must wait for the final put linearization point", + invalidate.isDone()); + releaseFinalPut.countDown(); + + Assert.assertEquals(Integer.valueOf(1), load.get(3L, TimeUnit.SECONDS)); + invalidate.get(3L, TimeUnit.SECONDS); + Assert.assertNull(entry.peekIfPresent("k")); + } finally { + releaseFinalPut.countDown(); + Config.enable_external_meta_cache_manual_miss_load = originalManualMissLoad; + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + @Test public void testManualMissLoadAllowsNullWithoutCaching() { boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; @@ -373,6 +1253,357 @@ public void testManualMissLoadDoesNotCacheWhenEntryDisabled() { } } + @Test + public void testClosedEntryCanNotBeRepopulated() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + try { + AtomicInteger loadCounter = new AtomicInteger(); + MetaCacheEntry entry = new MetaCacheEntry<>( + "test", + key -> loadCounter.incrementAndGet(), + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, + false); + Assert.assertEquals(Integer.valueOf(1), entry.get("k")); + + entry.close(); + + Assert.assertNull(entry.getIfPresent("k")); + Assert.assertEquals(Integer.valueOf(2), entry.get("k")); + Assert.assertNull(entry.getIfPresent("k")); + } finally { + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testManualMissLoadDoesNotWriteBackAcrossClose() throws Exception { + boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; + Config.enable_external_meta_cache_manual_miss_load = true; + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExecutorService queryExecutor = Executors.newSingleThreadExecutor(); + CountDownLatch beforePut = new CountDownLatch(1); + CountDownLatch releasePut = new CountDownLatch(1); + try { + MetaCacheEntry entry = new MetaCacheEntry( + "test", key -> 1, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false) { + @Override + void beforeManualCachePutForTest(String key, Integer loaded) { + beforePut.countDown(); + awaitLatch(releasePut); + } + }; + Future load = queryExecutor.submit(() -> entry.get("k")); + Assert.assertTrue(beforePut.await(3L, TimeUnit.SECONDS)); + entry.close(); + releasePut.countDown(); + + Assert.assertEquals(Integer.valueOf(1), load.get(3L, TimeUnit.SECONDS)); + Assert.assertNull(entry.peekIfPresent("k")); + } finally { + releasePut.countDown(); + Config.enable_external_meta_cache_manual_miss_load = originalManualMissLoad; + queryExecutor.shutdownNow(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testCompareAndReplaceUsesIdentityAndPeekDoesNotPolluteStats() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + try { + MetaCacheEntry entry = new MetaCacheEntry<>( + "test", key -> "loaded", CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false); + String current = new String("same"); + entry.put("k", current); + + Assert.assertSame(current, entry.peekIfPresent("k")); + Assert.assertEquals(0L, entry.stats().getRequestCount()); + Assert.assertEquals(MetaCacheEntry.ReplaceResult.NOT_CURRENT, + entry.tryReplace("k", new String("same"), "wrong")); + Assert.assertEquals(MetaCacheEntry.ReplaceResult.REPLACED, + entry.tryReplace("k", current, "new")); + Assert.assertEquals("new", entry.peekIfPresent("k")); + Assert.assertEquals(0L, entry.stats().getRequestCount()); + } finally { + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testWeightedAdmissionAndReplacementAccounting() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(1_000L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", + key -> 30, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_000L), + refreshExecutor, + false, + false, + (key, value) -> MetaCacheSizeEstimate.complete(value.longValue()), + budget); + try { + Assert.assertEquals(Integer.valueOf(30), entry.get("k")); + Assert.assertEquals(accountedWeight(30L), manager.getGlobalUsedWeight()); + + entry.put("k", 40); + Assert.assertEquals(Integer.valueOf(40), entry.getIfPresent("k")); + Assert.assertEquals(accountedWeight(40L), manager.getGlobalUsedWeight()); + + entry.put("k", 10); + Assert.assertEquals(Integer.valueOf(10), entry.getIfPresent("k")); + Assert.assertEquals(accountedWeight(10L), manager.getGlobalUsedWeight()); + + entry.invalidateKey("k"); + Assert.assertNull(entry.getIfPresent("k")); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testCaffeineWeigherOnlyReadsPreparedReservationWeight() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(1_000L)); + AtomicInteger estimateCalls = new AtomicInteger(); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", key -> 1, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_000L), + refreshExecutor, false, false, + (key, value) -> { + estimateCalls.incrementAndGet(); + return MetaCacheSizeEstimate.complete(value.longValue()); + }, budget); + try { + Integer first = Integer.valueOf(20); + entry.put("k", first); + Assert.assertEquals(1, estimateCalls.get()); + Assert.assertEquals(MetaCacheEntry.ReplaceResult.REPLACED, + entry.tryReplace("k", first, Integer.valueOf(30))); + Assert.assertEquals(2, estimateCalls.get()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testIncompleteEstimateReturnsValueWithoutCaching() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(1_080L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", + key -> 30, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_080L), + refreshExecutor, + false, + false, + (key, value) -> MetaCacheSizeEstimate.incomplete("unclassified_field"), + budget); + try { + Assert.assertEquals(Integer.valueOf(30), entry.get("k")); + Assert.assertNull(entry.getIfPresent("k")); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + Assert.assertEquals(1L, entry.stats().getWeightAdmissionRejectedCount()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testNegativeEstimateFailsImmediately() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(100L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(50L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", + key -> 30, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 50L), + refreshExecutor, + false, + false, + (key, value) -> MetaCacheSizeEstimate.complete(-1L), + budget); + try { + Assert.assertThrows(IllegalArgumentException.class, () -> entry.put("k", 30)); + Assert.assertNull(entry.getIfPresent("k")); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testZeroEstimateIsRejectedWithoutCaching() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(100L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(50L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", key -> 1, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 50L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(0L), budget); + try { + entry.put("k", 1); + + Assert.assertNull(entry.peekIfPresent("k")); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + Assert.assertEquals(1L, entry.stats().getWeightAdmissionRejectedCount()); + Assert.assertEquals("invalid_estimate", entry.stats().getLastWeightRejectReason()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testWeightedEntryEvictsItsOwnColdestValueBeforeAdmission() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(1_080L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", + String::length, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_080L), + refreshExecutor, + false, + false, + (key, value) -> MetaCacheSizeEstimate.complete(value.longValue()), + budget); + try { + entry.put("first", 30); + entry.put("second", 30); + Assert.assertNull(entry.getIfPresent("first")); + Assert.assertEquals(Integer.valueOf(30), entry.getIfPresent("second")); + Assert.assertEquals(accountedWeight(30L), manager.getGlobalUsedWeight()); + Assert.assertEquals(1L, entry.stats().getEvictionCount()); + Assert.assertEquals(accountedWeight(30L), entry.stats().getEvictionWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testWeightedAdmissionCanReclaimMoreThanOneThousandSmallValues() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + long maxWeight = 1_500L * accountedWeight(1L); + ExternalMetaCacheBudgetManager manager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(maxWeight)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "many-small-values", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "many-small-values", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10_000L, maxWeight), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + for (int i = 0; i < 1_500; i++) { + entry.put("small-" + i, new byte[1]); + } + + byte[] large = new byte[600_000]; + entry.put("large", large); + + Assert.assertSame(large, entry.peekIfPresent("large")); + Assert.assertTrue(entry.stats().getEvictionCount() > 1_024L); + Assert.assertTrue(entry.stats().getEstimatedWeight() <= maxWeight); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testOversizedValueIsRejectedWithoutEvictingUsefulValues() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(2_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(1_100L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", key -> 1, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_100L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.longValue()), budget); + try { + entry.put("first", 20); + entry.put("second", 20); + entry.put("oversized", 600); + + Assert.assertEquals(Integer.valueOf(20), entry.peekIfPresent("first")); + Assert.assertEquals(Integer.valueOf(20), entry.peekIfPresent("second")); + Assert.assertNull(entry.peekIfPresent("oversized")); + Assert.assertEquals(2L * accountedWeight(20L), manager.getGlobalUsedWeight()); + Assert.assertEquals(0L, entry.stats().getEvictionCount()); + Assert.assertEquals(1L, entry.stats().getWeightAdmissionRejectedCount()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRejectedAtomicReplacementKeepsExpectedValueUntilConditionalInvalidation() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(1_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(600L)); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", key -> 1, CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 600L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.longValue()), budget); + try { + Integer current = Integer.valueOf(30); + entry.put("k", current); + Assert.assertEquals(MetaCacheEntry.ReplaceResult.REJECTED, + entry.tryReplace("k", current, Integer.valueOf(100))); + Assert.assertSame(current, entry.peekIfPresent("k")); + Assert.assertTrue(entry.invalidateKeyIfSame("k", current)); + Assert.assertNull(entry.peekIfPresent("k")); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testDisabledWeightedEntrySkipsEstimator() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(100L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "weighted", OptionalLong.empty(), OptionalLong.of(50L)); + AtomicInteger estimateCalls = new AtomicInteger(); + MetaCacheEntry entry = new MetaCacheEntry<>( + "weighted", key -> 30, CacheSpec.ofWeight(false, CacheSpec.CACHE_NO_TTL, 10L, 50L), + refreshExecutor, false, false, + (key, value) -> { + estimateCalls.incrementAndGet(); + return MetaCacheSizeEstimate.complete(value.longValue()); + }, budget); + try { + Assert.assertEquals(Integer.valueOf(30), entry.get("k")); + Assert.assertEquals(0, estimateCalls.get()); + Assert.assertNull(entry.peekIfPresent("k")); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + // Keep the loader blocking helper in one place so concurrent tests stay readable. private void awaitLatch(CountDownLatch latch) { try { @@ -383,12 +1614,87 @@ private void awaitLatch(CountDownLatch latch) { } } + private void awaitValueLength(MetaCacheEntry entry, String key, int expectedLength) + throws InterruptedException { + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); + while (System.nanoTime() < deadlineNanos) { + byte[] value = entry.peekIfPresent(key); + if (value != null && value.length == expectedLength) { + return; + } + Thread.sleep(10L); + } + Assert.assertEquals(expectedLength, entry.peekIfPresent(key).length); + } + + private void awaitValue(MetaCacheEntry entry, String key, String expected) + throws InterruptedException { + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); + while (System.nanoTime() < deadlineNanos) { + if (expected.equals(entry.peekIfPresent(key))) { + return; + } + Thread.sleep(10L); + } + Assert.assertEquals(expected, entry.peekIfPresent(key)); + } + + private void awaitGlobalWeight(ExternalMetaCacheBudgetManager manager, long expected) + throws InterruptedException { + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); + while (manager.getGlobalUsedWeight() != expected && System.nanoTime() < deadlineNanos) { + Thread.sleep(10L); + } + Assert.assertEquals(expected, manager.getGlobalUsedWeight()); + } + + private Reference extractValueReference(LoadingCache loadingCache) throws Exception { + Object boundedLocalCache = readField(loadingCache, "cache"); + Map nodes = (Map) readField(boundedLocalCache, "data"); + Assert.assertEquals(1, nodes.size()); + Object node = nodes.values().iterator().next(); + Method valueReferenceMethod = findMethod(node.getClass(), "getValueReference"); + Object valueReference = valueReferenceMethod.invoke(node); + Assert.assertTrue(valueReference instanceof Reference); + return (Reference) valueReference; + } + + private Object readField(Object target, String name) throws Exception { + for (Class type = target.getClass(); type != null; type = type.getSuperclass()) { + try { + Field field = type.getDeclaredField(name); + field.setAccessible(true); + return field.get(target); + } catch (NoSuchFieldException ignored) { + // Continue through Caffeine's generated cache hierarchy. + } + } + throw new NoSuchFieldException(name); + } + + private Method findMethod(Class type, String name) throws Exception { + for (Class current = type; current != null; current = current.getSuperclass()) { + try { + Method method = current.getDeclaredMethod(name); + method.setAccessible(true); + return method; + } catch (NoSuchMethodException ignored) { + // Continue through Caffeine's generated node hierarchy. + } + } + throw new NoSuchMethodException(name); + } + @SuppressWarnings("unchecked") - private LoadingCache extractLoadingCache(MetaCacheEntry entry) throws Exception { + private LoadingCache extractLoadingCache(MetaCacheEntry entry) throws Exception { Field dataField = MetaCacheEntry.class.getDeclaredField("loadingData"); dataField.setAccessible(true); Object raw = dataField.get(entry); Assert.assertTrue(raw instanceof LoadingCache); - return (LoadingCache) raw; + return (LoadingCache) raw; + } + + private static long accountedWeight(long estimatedPayloadBytes) { + return estimatedPayloadBytes + MetaCacheEntry.FIXED_ENTRY_ACCOUNTING_OVERHEAD_BYTES; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 228bb112ff0016..066905dbfaa6ae 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -24,9 +24,11 @@ import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.ExternalCatalog; +import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.metacache.MetaCacheEntryStats; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; import org.apache.doris.datasource.metacache.paimon.PaimonLatestSnapshotProjectionLoader; import org.apache.doris.datasource.metacache.paimon.PaimonPartitionInfoLoader; @@ -38,6 +40,8 @@ import org.apache.paimon.data.BinaryRow; import org.apache.paimon.fs.Path; import org.apache.paimon.fs.local.LocalFileIO; +import org.apache.paimon.privilege.PrivilegeChecker; +import org.apache.paimon.privilege.PrivilegedFileStoreTable; import org.apache.paimon.schema.Schema; import org.apache.paimon.schema.SchemaManager; import org.apache.paimon.schema.TableSchema; @@ -50,6 +54,7 @@ import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.IntType; +import org.apache.paimon.types.RowType; import org.junit.Assert; import org.junit.Assume; import org.junit.Rule; @@ -58,6 +63,7 @@ import org.mockito.MockedStatic; import org.mockito.Mockito; +import java.lang.reflect.Field; import java.util.Collections; import java.util.HashMap; import java.util.Map; @@ -69,6 +75,240 @@ public class PaimonExternalMetaCacheTest { @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); + @Test + public void testSnapshotWeightScalesLinearlyToOneHundredThousandPartitions() throws Exception { + FileStoreTable table = newPartitionedTable("linear_snapshot_estimate", Collections.emptyMap()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey( + mapping, 1L, table.schema().id(), 1L); + long base = snapshotWeight(key, table, 0); + long oneThousand = snapshotWeight(key, table, 1_000); + long tenThousand = snapshotWeight(key, table, 10_000); + long oneHundredThousand = snapshotWeight(key, table, 100_000); + + long oneThousandPayload = oneThousand - base; + Assert.assertTrue(oneThousandPayload > 0L); + Assert.assertEquals(oneThousandPayload * 10L, tenThousand - base); + Assert.assertEquals(oneThousandPayload * 100L, oneHundredThousand - base); + } + + @Test + public void testSnapshotWeightAccountsForSkewedTableOptions() throws Exception { + FileStoreTable smallTable = newPartitionedTable( + "option_small", Collections.singletonMap("payload", "x")); + String largePayload = repeatedCharacter('x', 64 * 1024); + FileStoreTable largeTable = newPartitionedTable( + "option_large", Collections.singletonMap("payload", largePayload)); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey smallKey = new PaimonSnapshotEntryKey( + mapping, 1L, smallTable.schema().id(), 1L); + PaimonSnapshotEntryKey largeKey = new PaimonSnapshotEntryKey( + mapping, 1L, largeTable.schema().id(), 1L); + + long smallBytes = snapshotWeight(smallKey, smallTable, 0); + long largeBytes = snapshotWeight(largeKey, largeTable, 0); + + Assert.assertTrue(largeBytes - smallBytes >= (largePayload.length() - 1L) * 2L); + } + + @Test + public void testSnapshotWeightAccountsForNestedSchemaPayload() throws Exception { + String largeFieldName = repeatedCharacter('x', 64 * 1024); + FileStoreTable smallTable = newPartitionedTableWithNestedField("nested_small", "x"); + FileStoreTable largeTable = newPartitionedTableWithNestedField( + "nested_large", largeFieldName); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey smallKey = new PaimonSnapshotEntryKey( + mapping, 1L, smallTable.schema().id(), 1L); + PaimonSnapshotEntryKey largeKey = new PaimonSnapshotEntryKey( + mapping, 1L, largeTable.schema().id(), 1L); + + long smallBytes = snapshotWeight(smallKey, smallTable, 0); + long largeBytes = snapshotWeight(largeKey, largeTable, 0); + + Assert.assertTrue(largeBytes - smallBytes >= (largeFieldName.length() - 1L) * 2L); + } + + @Test + public void testSnapshotWeightAccountsForTableComment() throws Exception { + String largeComment = repeatedCharacter('x', 64 * 1024); + FileStoreTable smallTable = newPartitionedTable( + "comment_small", Collections.emptyMap(), "x"); + FileStoreTable largeTable = newPartitionedTable( + "comment_large", Collections.emptyMap(), largeComment); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + + long smallBytes = snapshotWeight(new PaimonSnapshotEntryKey( + mapping, 1L, smallTable.schema().id(), 1L), smallTable, 0); + long largeBytes = snapshotWeight(new PaimonSnapshotEntryKey( + mapping, 1L, largeTable.schema().id(), 1L), largeTable, 0); + + Assert.assertTrue(largeBytes - smallBytes >= (largeComment.length() - 1L) * 2L); + } + + @Test + public void testSnapshotWeightEntryAndPrecomputedEstimate() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + cache.initCatalog(1L, Collections.singletonMap( + "meta.cache.paimon.snapshot.max-weight", "8MB")); + Assert.assertTrue(cache.stats(1L).get(PaimonExternalMetaCache.ENTRY_SNAPSHOT).isWeightBounded()); + + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + FileStoreTable table = newPartitionedTable("snapshot_estimate", Collections.emptyMap()); + Object lazyStoreBefore = readField(table, table.getClass(), "lazyStore"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey(mapping, 1L, table.schema().id(), 1L); + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, table.schema().id(), table)); + value.prepareForCachePublication(key); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), + value.getSizeEstimate().isComplete()); + Assert.assertTrue(value.getSizeEstimate().getBytes() > 0L); + Assert.assertSame("cache admission must not materialize FileStoreTable.store()", + lazyStoreBefore, readField(table, table.getClass(), "lazyStore")); + + PaimonSnapshotCacheValue unsupportedValue = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, 1L, Mockito.mock(Table.class))); + unsupportedValue.prepareForCachePublication(new PaimonSnapshotEntryKey(mapping, 1L, 1L, 1L)); + Assert.assertFalse(unsupportedValue.getSizeEstimate().isComplete()); + Assert.assertTrue(unsupportedValue.getSizeEstimate().getIncompleteReason() + .startsWith("unsupported_paimon_table:")); + } finally { + executor.shutdownNow(); + } + } + + @Test + public void testSnapshotKeySeparatesReloadedTableGenerations() { + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshot fence = new PaimonSnapshot(7L, 3L, null); + PaimonSnapshotCacheValue fenceValue = new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, fence); + PaimonTableCacheValue first = new PaimonTableCacheValue(null, fenceValue); + PaimonTableCacheValue reloaded = new PaimonTableCacheValue(null, fenceValue); + + PaimonSnapshotEntryKey firstKey = PaimonSnapshotEntryKey.of( + mapping, fence, first.getGeneration()); + PaimonSnapshotEntryKey reloadedKey = PaimonSnapshotEntryKey.of( + mapping, fence, reloaded.getGeneration()); + + Assert.assertNotEquals(firstKey, reloadedKey); + Assert.assertNotEquals(firstKey.getTableGeneration(), reloadedKey.getTableGeneration()); + } + + @Test + public void testSnapshotHitReusesFenceCapturedByTableGeneration() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try { + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.emptyMap()); + NameMapping mapping = new NameMapping(catalogId, "db", "tbl", "remote_db", "remote_tbl"); + FileStoreTable table = Mockito.mock(FileStoreTable.class); + PaimonSnapshot fence = new PaimonSnapshot(7L, 3L, table); + PaimonSnapshotCacheValue snapshotValue = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, fence); + PaimonTableCacheValue tableValue = new PaimonTableCacheValue(table, snapshotValue); + PaimonSnapshotEntryKey key = PaimonSnapshotEntryKey.of( + mapping, fence, tableValue.getGeneration()); + cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class).put(mapping, tableValue); + cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_SNAPSHOT, + PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class).put(key, snapshotValue); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + + Assert.assertSame(snapshotValue, cache.getSnapshotCache(dorisTable)); + Assert.assertSame(snapshotValue, cache.getSnapshotCache(dorisTable)); + Assert.assertSame(snapshotValue, cache.loadLatestSnapshotFence(dorisTable)); + Assert.assertSame(snapshotValue, cache.loadLatestSnapshotFence(dorisTable)); + + Mockito.verifyNoInteractions(table); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testSnapshotEstimateSupportsPrivilegedTableWrapper() throws Exception { + FileStoreTable table = newPartitionedTable("privileged_estimate", Collections.emptyMap()); + FileStoreTable privileged = PrivilegedFileStoreTable.wrap( + table, Mockito.mock(PrivilegeChecker.class), Identifier.create("db", "tbl")); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey( + mapping, 1L, table.schema().id(), 1L); + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, + new PaimonSnapshot(1L, table.schema().id(), privileged)); + + value.prepareForCachePublication(key); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), + value.getSizeEstimate().isComplete()); + } + + @Test + public void testSnapshotEstimateDoesNotMaterializeNestedRowTypeIndexes() throws Exception { + RowType nested = DataTypes.ROW( + DataTypes.FIELD(10, "nested_id", DataTypes.INT()), + DataTypes.FIELD(11, "nested_name", DataTypes.STRING())); + TableSchema schema = new TableSchema( + 0, + java.util.Arrays.asList( + new DataField(0, "id", new IntType()), + new DataField(1, "payload", nested)), + 11, + Collections.emptyList(), + Collections.emptyList(), + Collections.emptyMap(), + null); + FileStoreTable table = new AppendOnlyFileStoreTable( + LocalFileIO.create(), + new Path(temporaryFolder.newFolder("nested_row_estimate").toURI()), + schema, + CatalogEnvironment.empty()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey(mapping, 1L, schema.id(), 1L); + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, schema.id(), table)); + + Map stateBefore = new HashMap<>(); + for (String fieldName : java.util.Arrays.asList( + "laziedNameToField", "laziedNameToIndex", "laziedFieldIdToField", "laziedFieldIdToIndex")) { + stateBefore.put(fieldName, readField(nested, fieldName)); + } + value.prepareForCachePublication(key); + + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), value.getSizeEstimate().isComplete()); + for (Map.Entry entry : stateBefore.entrySet()) { + Assert.assertSame(entry.getKey() + " must not be changed by cache admission", + entry.getValue(), readField(nested, entry.getKey())); + } + } + + @Test + public void testSnapshotInheritsLegacyTableCountSettingsButNotWeight() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + try { + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + Map properties = new HashMap<>(); + properties.put("meta.cache.paimon.table.enable", "false"); + properties.put("meta.cache.paimon.table.ttl-second", "17"); + properties.put("meta.cache.paimon.table.capacity", "23"); + cache.initCatalog(1L, properties); + + MetaCacheEntryStats snapshot = cache.stats(1L).get(PaimonExternalMetaCache.ENTRY_SNAPSHOT); + Assert.assertFalse(snapshot.isConfigEnabled()); + Assert.assertEquals(17L, snapshot.getTtlSecond()); + Assert.assertEquals(23L, snapshot.getCapacity()); + Assert.assertFalse(snapshot.isWeightBounded()); + } finally { + executor.shutdownNow(); + } + } + @Test public void testLatestSnapshotUsesLatestSchemaForPinnedRead() { PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( @@ -320,6 +560,11 @@ public void testPartitionProjectionIgnoresReaderOnlyPhysicalOptions() throws Exc } private FileStoreTable newPartitionedTable(String name, Map options) throws Exception { + return newPartitionedTable(name, options, null); + } + + private FileStoreTable newPartitionedTable( + String name, Map options, String comment) throws Exception { TableSchema schema = new TableSchema( 0, java.util.Arrays.asList( @@ -329,6 +574,27 @@ private FileStoreTable newPartitionedTable(String name, Map opti Collections.singletonList("part"), Collections.emptyList(), options, + comment); + return new AppendOnlyFileStoreTable( + LocalFileIO.create(), + new Path(temporaryFolder.newFolder(name).toURI()), + schema, + CatalogEnvironment.empty()); + } + + private FileStoreTable newPartitionedTableWithNestedField( + String name, String nestedFieldName) throws Exception { + RowType nestedType = new RowType(Collections.singletonList( + new DataField(2, nestedFieldName, DataTypes.STRING()))); + TableSchema schema = new TableSchema( + 0, + java.util.Arrays.asList( + new DataField(0, "payload", nestedType), + new DataField(1, "part", new IntType())), + 1, + Collections.singletonList("part"), + Collections.emptyList(), + Collections.emptyMap(), null); return new AppendOnlyFileStoreTable( LocalFileIO.create(), @@ -337,6 +603,42 @@ private FileStoreTable newPartitionedTable(String name, Map opti CatalogEnvironment.empty()); } + private long snapshotWeight(PaimonSnapshotEntryKey key, FileStoreTable table, int partitionCount) { + PaimonPartitionInfo partitionInfo = Mockito.mock(PaimonPartitionInfo.class); + Map partitionItems = sizeOnlyMap(partitionCount); + Map partitions = sizeOnlyMap(partitionCount); + Mockito.when(partitionInfo.getNameToPartitionItem()).thenReturn(partitionItems); + Mockito.when(partitionInfo.getNameToPartition()).thenReturn(partitions); + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + partitionInfo, new PaimonSnapshot(1L, table.schema().id(), table)); + MetaCacheSizeEstimate estimate = value.prepareForCachePublication(key); + Assert.assertTrue(estimate.getIncompleteReason(), estimate.isComplete()); + return estimate.getBytes(); + } + + @SuppressWarnings("unchecked") + private Map sizeOnlyMap(int size) { + Map map = Mockito.mock(Map.class); + Mockito.when(map.size()).thenReturn(size); + return map; + } + + private Object readField(RowType rowType, String fieldName) throws Exception { + return readField(rowType, RowType.class, fieldName); + } + + private static String repeatedCharacter(char character, int count) { + char[] characters = new char[count]; + java.util.Arrays.fill(characters, character); + return new String(characters); + } + + private Object readField(Object target, Class owner, String fieldName) throws Exception { + Field field = owner.getDeclaredField(fieldName); + field.setAccessible(true); + return field.get(target); + } + @Test public void testInvalidateTablePrecise() { ExecutorService executor = Executors.newSingleThreadExecutor(); @@ -350,15 +652,24 @@ public void testInvalidateTablePrecise() { org.apache.doris.datasource.metacache.MetaCacheEntry tableEntry = cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_TABLE, NameMapping.class, PaimonTableCacheValue.class); - tableEntry.put(t1, new PaimonTableCacheValue(null, - () -> new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, 1L, null)))); - tableEntry.put(t2, new PaimonTableCacheValue(null, - () -> new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, new PaimonSnapshot(2L, 2L, null)))); + PaimonSnapshotCacheValue fence = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(-1L, 0L, null)); + tableEntry.put(t1, new PaimonTableCacheValue(null, fence)); + tableEntry.put(t2, new PaimonTableCacheValue(null, fence)); + + PaimonSnapshotEntryKey snapshotKey = new PaimonSnapshotEntryKey(t1, 1L, 2L, 1L); + org.apache.doris.datasource.metacache.MetaCacheEntry< + PaimonSnapshotEntryKey, PaimonSnapshotCacheValue> snapshotEntry = cache.entry(catalogId, + PaimonExternalMetaCache.ENTRY_SNAPSHOT, + PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class); + snapshotEntry.put(snapshotKey, new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, 2L, null))); cache.invalidateTable(catalogId, "db1", "tbl1"); Assert.assertNull(tableEntry.getIfPresent(t1)); Assert.assertNotNull(tableEntry.getIfPresent(t2)); + Assert.assertNull(snapshotEntry.getIfPresent(snapshotKey)); } finally { executor.shutdownNow(); } @@ -377,10 +688,10 @@ public void testInvalidateDbAndStats() { org.apache.doris.datasource.metacache.MetaCacheEntry tableEntry = cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_TABLE, NameMapping.class, PaimonTableCacheValue.class); - tableEntry.put(db1Table, new PaimonTableCacheValue(null, - () -> new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, 1L, null)))); - tableEntry.put(db2Table, new PaimonTableCacheValue(null, - () -> new PaimonSnapshotCacheValue(PaimonPartitionInfo.EMPTY, new PaimonSnapshot(2L, 2L, null)))); + PaimonSnapshotCacheValue fence = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(-1L, 0L, null)); + tableEntry.put(db1Table, new PaimonTableCacheValue(null, fence)); + tableEntry.put(db2Table, new PaimonTableCacheValue(null, fence)); org.apache.doris.datasource.metacache.MetaCacheEntry schemaEntry = cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_SCHEMA, diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java index 09bcee53985f5d..17fa76bb5593bc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java @@ -36,6 +36,7 @@ import org.apache.paimon.data.BinaryString; import org.apache.paimon.data.Timestamp; import org.apache.paimon.manifest.PartitionEntry; +import org.apache.paimon.partition.Partition; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.Table; import org.apache.paimon.table.source.ReadBuilder; @@ -87,6 +88,19 @@ private static PartitionEntry partitionEntry(BinaryRow partition, long sequence) return new PartitionEntry(partition, sequence, sequence, sequence, sequence, 1); } + @Test + public void testCompatibilityConstructorDerivesRetainedPartitionPayload() { + String largeValue = repeatedCharacter('x', 64 * 1024); + Partition partition = new Partition( + Collections.singletonMap("part", largeValue), + 1L, 1L, 1L, 1L, 1, false); + + PaimonPartitionInfo info = new PaimonPartitionInfo( + Collections.emptyMap(), Collections.singletonMap("part=" + largeValue, partition)); + + Assert.assertTrue(info.getRetainedPayloadBytes() >= largeValue.length() * 4L); + } + @Test public void testSchemaForVarcharAndChar() { DataField c1 = new DataField(1, "c1", new VarCharType(32)); @@ -247,6 +261,7 @@ public void testGeneratePartitionInfoWithSpecialCharacters() { Assert.assertEquals(1, partitionInfo.getNameToPartitionItem().size()); String partitionName = "source=dataset%2Fteam-a%2Fsegment-01" + "/part_str=%2Fymd%3D20260701%2Fhour%3D%5B0-9%5D%5B0-9%5D%2F%2A.jsonl/pass=s1"; + Assert.assertTrue(partitionInfo.getRetainedPayloadBytes() > partitionName.length() * 2L); Assert.assertTrue(partitionInfo.getNameToPartition().containsKey(partitionName)); PartitionItem partitionItem = partitionInfo.getNameToPartitionItem().values().iterator().next(); List actualValues = ((ListPartitionItem) partitionItem).getItems().get(0) @@ -257,6 +272,21 @@ public void testGeneratePartitionInfoWithSpecialCharacters() { "s1"), actualValues); } + @Test + public void testRetainedPayloadCounterTracksSkewedPartitionValues() { + List partitionColumns = Collections.singletonList(new Column("part", Type.STRING)); + Table table = mockPartitionTable(Collections.emptyMap(), + DataTypes.FIELD(0, "part", DataTypes.STRING())); + PaimonPartitionInfo small = PaimonUtil.generatePartitionInfo(table, partitionColumns, + Collections.singletonList(partitionEntry(stringPartitionRow("x"), 1L))); + String largeValue = repeatedCharacter('x', 64 * 1024); + PaimonPartitionInfo large = PaimonUtil.generatePartitionInfo(table, partitionColumns, + Collections.singletonList(partitionEntry(stringPartitionRow(largeValue), 1L))); + + Assert.assertTrue(large.getRetainedPayloadBytes() - small.getRetainedPayloadBytes() + >= (largeValue.length() - 1L) * 4L); + } + @Test public void testGeneratePartitionInfoUsesPartitionColumnOrder() { List partitionColumns = Arrays.asList( @@ -532,4 +562,10 @@ public void testAuditLogHistorySchemaWithoutSequenceNumber() { Assert.assertEquals("id", fields.get(1).getFieldPtr().getName()); Assert.assertEquals("name", fields.get(2).getFieldPtr().getName()); } + + private static String repeatedCharacter(char character, int count) { + char[] characters = new char[count]; + Arrays.fill(characters, character); + return new String(characters); + } } diff --git a/fe/pom.xml b/fe/pom.xml index 3783d2660e6b9b..6821b6c3fce2ab 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -438,6 +438,12 @@ under the License. + + benchmark + + fe-benchmark + + From 41e63e0efbf7cf666379078e77d7db52e4cf1699 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Mon, 17 Aug 2026 14:01:43 +0800 Subject: [PATCH 02/45] [fix](fe) Harden external metadata cache memory governance Address review findings around upgrade compatibility, generation retirement, live Iceberg writes, Paimon snapshot freshness, authenticated metadata access, refresh failure handling, and fair catalog/global budget reclaim. Make optional size accounting fail closed without failing queries, add JOL calibration and long-tail coverage, and keep retained-size traversal off default count-based cache paths. Tests: 170 targeted FE tests; external metadata cache size benchmark. --- .../run-external-meta-cache-size-benchmark.sh | 5 +- fe/fe-core/pom.xml | 5 + .../apache/doris/datasource/CatalogMgr.java | 10 +- .../datasource/ExternalMetaCacheMgr.java | 36 ++- .../hive/HiveExternalMetaCache.java | 6 +- .../iceberg/IcebergCacheSizeEstimator.java | 51 ++- .../iceberg/IcebergExternalMetaCache.java | 141 ++++++--- .../iceberg/IcebergSchemaCacheKey.java | 16 +- .../iceberg/IcebergSnapshotCacheValue.java | 282 ++++------------- .../iceberg/IcebergSnapshotEntryKey.java | 8 + .../iceberg/IcebergSysExternalTable.java | 49 +-- .../iceberg/IcebergTableCacheValue.java | 66 +++- .../datasource/iceberg/IcebergUtils.java | 27 +- .../iceberg/cache/ManifestCacheValue.java | 49 ++- .../metacache/AbstractExternalMetaCache.java | 40 ++- .../doris/datasource/metacache/CacheSpec.java | 30 +- .../metacache/CatalogEntryGroup.java | 13 +- .../ExternalMetaCacheBudgetManager.java | 181 ++++++++++- .../datasource/metacache/MetaCacheEntry.java | 121 ++++++-- .../metacache/MetaCacheEntryDef.java | 27 +- .../MetaCacheEntryReplacementListener.java | 26 ++ .../metacache/MetaCacheSizeEstimator.java | 9 +- .../PaimonLatestSnapshotProjectionLoader.java | 24 +- .../metacache/paimon/PaimonTableLoader.java | 11 + .../paimon/PaimonExternalMetaCache.java | 92 +++++- .../paimon/PaimonExternalTable.java | 8 + .../paimon/PaimonSchemaCacheKey.java | 14 +- .../paimon/PaimonSnapshotCacheValue.java | 20 +- .../paimon/PaimonTableCacheValue.java | 15 +- .../doris/datasource/paimon/PaimonUtils.java | 5 + .../ExternalMetaCacheRouteResolverTest.java | 46 ++- .../hive/HiveMetaStoreCacheTest.java | 42 +++ .../iceberg/IcebergExternalMetaCacheTest.java | 293 +++++++++++++++--- .../iceberg/IcebergSysExternalTableTest.java | 28 ++ .../AbstractExternalMetaCacheTest.java | 22 +- .../EstimatorCalibrationAssertions.java | 67 ++++ .../ExternalMetaCacheBudgetManagerTest.java | 100 ++++++ .../metacache/MetaCacheEntryTest.java | 160 ++++++++++ .../paimon/PaimonExternalMetaCacheTest.java | 244 ++++++++++++++- .../paimon/PaimonExternalTableTest.java | 2 +- fe/pom.xml | 6 + .../test_iceberg_table_meta_cache.groovy | 19 +- 42 files changed, 1929 insertions(+), 487 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryReplacementListener.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java diff --git a/fe/fe-benchmark/run-external-meta-cache-size-benchmark.sh b/fe/fe-benchmark/run-external-meta-cache-size-benchmark.sh index 12e28e0dd9a7a4..7ce2e0be8e3d4f 100755 --- a/fe/fe-benchmark/run-external-meta-cache-size-benchmark.sh +++ b/fe/fe-benchmark/run-external-meta-cache-size-benchmark.sh @@ -33,7 +33,10 @@ trap 'rm -f "${CLASSPATH_FILE}"' EXIT -Dmdep.outputFile="${CLASSPATH_FILE}" ) -REACTOR_CLASSES=$(find "${FE_DIR}" -type d -path '*/target/classes' -printf '%p:') +REACTOR_CLASSES= +while IFS= read -r -d '' CLASSES_DIR; do + REACTOR_CLASSES+="${CLASSES_DIR}:" +done < <(find "${FE_DIR}" -type d -path '*/target/classes' -print0) DEPENDENCY_CLASSES=$(tr -d '\n' < "${CLASSPATH_FILE}") BENCHMARK_FILTER=${BENCHMARK_FILTER:-'HivePartitionValuesSizeBenchmark|IcebergCacheSizeBenchmark|PaimonCacheSizeBenchmark|MetaCacheSoftValueBenchmark'} diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index a880c5ffad4db4..93ceeb49b58144 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -851,6 +851,11 @@ under the License. mockito-inline test + + org.openjdk.jol + jol-core + test + diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index b44cb735f2ac61..3e8a8f813571ea 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -648,7 +648,15 @@ public void replayAlterCatalogProps(CatalogLog log, Map oldPrope // Only legacy validators publish a tentative candidate. Detached validators // leave the live CatalogProperty untouched while concurrent initialization runs. if (oldProperties != null && tentativelyMutated) { - ((ExternalCatalog) catalog).rollBackCatalogProps(oldProperties); + Env currentEnv = Env.getCurrentEnv(); + ExternalMetaCacheMgr cacheMgr = currentEnv == null + ? null : currentEnv.getExtMetaCacheMgr(); + if (cacheMgr == null) { + ((ExternalCatalog) catalog).rollBackCatalogProps(oldProperties); + } else { + cacheMgr.rollbackCatalogProperties( + (ExternalCatalog) catalog, oldProperties); + } } if (validationException instanceof DdlException) { throw (DdlException) validationException; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index a80d0c8b71485f..e58a984eacc462 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -209,8 +209,9 @@ public void prepareCatalog(long catalogId) { logMissingCatalogSkip(catalogId, "prepareCatalog"); return; } - validateCatalogCachePropertiesForRuntime(catalogProperties); - routeCatalogEngines(catalogId, cache -> cache.initCatalog(catalogId, catalogProperties)); + Map runtimeProperties = sanitizeCatalogCachePropertiesForRuntime( + catalogId, catalogProperties); + routeCatalogEngines(catalogId, cache -> cache.initCatalog(catalogId, runtimeProperties)); } finally { lifecycleLock.unlock(); } @@ -254,7 +255,7 @@ private void prepareCatalogByEngineLocked( Map safeCatalogProperties = catalogProperties == null ? Maps.newHashMap() : Maps.newHashMap(catalogProperties); - validateCatalogCachePropertiesForRuntime(safeCatalogProperties); + safeCatalogProperties = sanitizeCatalogCachePropertiesForRuntime(catalogId, safeCatalogProperties); targetCache.initCatalog(catalogId, safeCatalogProperties); } @@ -264,9 +265,17 @@ public void validateCatalogCacheProperties(Map catalogProperties cacheRegistry.allCaches().forEach(cache -> cache.validateCatalogProperties(catalogProperties)); } - private void validateCatalogCachePropertiesForRuntime(Map catalogProperties) { - budgetManager.parseCatalogMaxWeight(catalogProperties); - validateCatalogCachePropertyNamespaces(catalogProperties); + private Map sanitizeCatalogCachePropertiesForRuntime( + long catalogId, Map catalogProperties) { + Map sanitized = Maps.newHashMap(catalogProperties); + try { + budgetManager.parseCatalogMaxWeight(sanitized); + } catch (IllegalArgumentException e) { + sanitized.remove(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY); + LOG.warn("Ignoring invalid persisted external metadata cache property '{}' for catalog {}: {}", + ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, catalogId, e.getMessage()); + } + return sanitized; } private void validateCatalogCachePropertyNamespaces(Map catalogProperties) { @@ -335,6 +344,21 @@ public void removeCatalog(long catalogId) { } } + /** Restore catalog properties and retire any group initialized from the rejected candidate atomically. */ + public void rollbackCatalogProperties(ExternalCatalog catalog, Map oldProperties) { + long catalogId = catalog.getId(); + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + catalog.rollBackCatalogProps(oldProperties); + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "rollbackCatalogProperties", + () -> cache.invalidateCatalog(catalogId))); + } finally { + lifecycleLock.unlock(); + } + } + public void removeCatalogByEngine(long catalogId, String engine) { Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); lifecycleLock.lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java index 527497237b1bf2..dfa5b00289feb9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java @@ -756,8 +756,10 @@ private void addPartitionsCache(NameMapping nameMapping, Map addedItems = new HashMap<>(); for (String partitionName : partitionNames) { if (allNames.containsKey(partitionName)) { - LOG.info("addPartitionsCache partitionName:[{}] has exist in table:[{}]", - partitionName, localTblName); + if (attempt == 0) { + LOG.info("addPartitionsCache partitionName:[{}] has exist in table:[{}]", + partitionName, localTblName); + } continue; } long partitionId = Util.genIdByName( diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java index e60f78eef90182..8d483ecfb5153f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java @@ -24,17 +24,21 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.PartitionSpec; +import org.apache.iceberg.PartitionStatisticsFile; import org.apache.iceberg.Schema; +import org.apache.iceberg.Snapshot; import org.apache.iceberg.SortOrder; +import org.apache.iceberg.StatisticsFile; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.encryption.EncryptedKey; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import java.nio.ByteBuffer; import java.util.Map; -/** Constant-time retained-weight formulas for Iceberg cache entries. */ +/** Publication-time retained-weight formulas for Iceberg cache entries. */ final class IcebergCacheSizeEstimator { private static final long KEY_BASE_BYTES = 128L; private static final long TABLE_BASE_BYTES = 16L * 1024L; @@ -47,6 +51,13 @@ final class IcebergCacheSizeEstimator { private static final long SORT_FIELD_BYTES = 256L; private static final long TABLE_PROPERTY_BYTES = 256L; private static final long CURRENT_SNAPSHOT_BYTES = 512L; + private static final long HISTORICAL_SNAPSHOT_BYTES = 1024L; + private static final long SNAPSHOT_LOG_ENTRY_BYTES = 64L; + private static final long METADATA_LOG_ENTRY_BYTES = 128L; + private static final long SNAPSHOT_REF_BYTES = 128L; + private static final long STATISTICS_FILE_BYTES = 512L; + private static final long PARTITION_STATISTICS_FILE_BYTES = 256L; + private static final long ENCRYPTED_KEY_BYTES = 256L; private static final long PARTITION_BYTES = 512L; private static final long PARTITION_ALIAS_BYTES = 256L; private static final long NAME_MAPPING_ENTRY_BYTES = 256L; @@ -110,6 +121,9 @@ static MetaCacheSizeEstimate estimateSnapshotEntry( static MetaCacheSizeEstimate estimateManifestEntry( IcebergManifestEntryKey key, ManifestCacheValue value) { + if (!value.isAccountingComplete()) { + return MetaCacheSizeEstimate.incomplete("iceberg_manifest_accounting_incomplete"); + } long bytes = MetaCacheWeightUtils.saturatedAdd( MANIFEST_ENTRY_BASE_BYTES, MetaCacheWeightUtils.estimatedStringBytes(key.getManifestPath())); @@ -189,10 +203,45 @@ static long retainedTablePayloadBytes(Table table) { bytes = addString(bytes, property.getKey()); bytes = addString(bytes, property.getValue()); } + for (Snapshot snapshot : metadata.snapshots()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, HISTORICAL_SNAPSHOT_BYTES); + bytes = addString(bytes, snapshot.operation()); + bytes = addString(bytes, snapshot.manifestListLocation()); + bytes = addStringMap(bytes, snapshot.summary(), TABLE_PROPERTY_BYTES); + } + bytes = addCount(bytes, metadata.snapshotLog().size(), SNAPSHOT_LOG_ENTRY_BYTES); + for (TableMetadata.MetadataLogEntry previousFile : metadata.previousFiles()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, METADATA_LOG_ENTRY_BYTES); + bytes = addString(bytes, previousFile.file()); + } + bytes = addCount(bytes, metadata.refs().size(), SNAPSHOT_REF_BYTES); + for (String refName : metadata.refs().keySet()) { + bytes = addString(bytes, refName); + } + for (StatisticsFile statisticsFile : metadata.statisticsFiles()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, STATISTICS_FILE_BYTES); + bytes = addString(bytes, statisticsFile.path()); + bytes = addCount(bytes, statisticsFile.blobMetadata().size(), TABLE_PROPERTY_BYTES); + } + for (PartitionStatisticsFile statisticsFile : metadata.partitionStatisticsFiles()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, PARTITION_STATISTICS_FILE_BYTES); + bytes = addString(bytes, statisticsFile.path()); + } + for (EncryptedKey encryptedKey : metadata.encryptionKeys()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ENCRYPTED_KEY_BYTES); + bytes = addString(bytes, encryptedKey.keyId()); + bytes = addString(bytes, encryptedKey.encryptedById()); + bytes = addBufferPayload(bytes, encryptedKey.encryptedKeyMetadata()); + bytes = addStringMap(bytes, encryptedKey.properties(), TABLE_PROPERTY_BYTES); + } bytes = addString(bytes, metadata.uuid()); return bytes; } + private static long addBufferPayload(long bytes, ByteBuffer buffer) { + return buffer == null ? bytes : MetaCacheWeightUtils.saturatedAdd(bytes, buffer.capacity()); + } + private static long addFieldPayload(long bytes, Types.NestedField field, boolean nested) { bytes = MetaCacheWeightUtils.saturatedAdd(bytes, nested ? NESTED_SCHEMA_FIELD_BYTES : SCHEMA_FIELD_BYTES); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index e6402dd234839c..e8b24299d53df2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -53,6 +53,7 @@ import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.function.Consumer; +import javax.annotation.Nullable; /** * Iceberg engine implementation of {@link AbstractExternalMetaCache}. @@ -103,7 +104,8 @@ public IcebergExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCac tableEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_TABLE, NameMapping.class, IcebergTableCacheValue.class, this::loadTableCacheValue, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping)) - .withSizeEstimator(this::prepareTableForCachePublication)); + .withSizeEstimator(this::prepareTableForCachePublication) + .withReplacementListener(this::retireTableGeneration)); snapshotEntry = registerEntry(MetaCacheEntryDef.contextualOnly(ENTRY_SNAPSHOT, IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(IcebergSnapshotEntryKey::getNameMapping)) @@ -128,27 +130,16 @@ public Table getIcebergTable(ExternalTable dorisTable) { public Table getWritableIcebergTable(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - IcebergTableCacheValue tableValue = - tableEntry.get(nameMapping.getCtlId()).get(nameMapping); CatalogIf catalog = getCatalog(nameMapping.getCtlId()); if (catalog == null) { throw new RuntimeException("Cannot find catalog " + nameMapping.getCtlId() + " when loading a writable Iceberg table"); } IcebergMetadataOps ops = resolveMetadataOps(catalog); - Table liveTable = executeAuthenticated(catalog, () -> ops.loadTable( + // DDL/actions must start from the live catalog generation. DML that was planned against a + // retained read generation wraps this live table separately in IcebergTransaction. + return executeAuthenticated(catalog, () -> ops.loadTable( nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName())); - try { - return tableValue.getWritableIcebergTable(liveTable); - } catch (IcebergSnapshotCacheValue.StaleMetadataException e) { - MetaCacheEntry entry = - tableEntry.get(nameMapping.getCtlId()); - entry.invalidateKeyIfSame(nameMapping, tableValue); - IcebergTableCacheValue refreshedValue = entry.get(nameMapping); - Table refreshedLiveTable = executeAuthenticated(catalog, () -> ops.loadTable( - nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName())); - return refreshedValue.getWritableIcebergTable(refreshedLiveTable); - } } Table getQueryScopedIcebergTable(ExternalTable dorisTable) { @@ -157,12 +148,7 @@ Table getQueryScopedIcebergTable(ExternalTable dorisTable) { tableEntry.get(nameMapping.getCtlId()); IcebergTableCacheValue tableValue = entry.get(nameMapping); - try { - return createQueryTable(nameMapping, tableValue); - } catch (IcebergSnapshotCacheValue.StaleMetadataException e) { - entry.invalidateKeyIfSame(nameMapping, tableValue); - return createQueryTable(nameMapping, entry.get(nameMapping)); - } + return createQueryTable(nameMapping, tableValue); } private Table createQueryTable( @@ -199,18 +185,26 @@ public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { snapshotEntry.get(nameMapping.getCtlId()); boolean isolateForQueries = tableValue.isQueryIsolationPrepared() || entry.isWeightBounded(); - return entry.get(key, ignored -> executeAuthenticated(nameMapping.getCtlId(), () -> { - Table projectionTable = isolateForQueries - ? tableValue.newQueryScopedTable() : tableValue.getIcebergTable(); - IcebergSnapshotCacheValue value = loadSnapshotProjection( - dorisTable, projectionTable, - tableValue.getRetainedIcebergTable(), - tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries); - if (entry.isWeightBounded()) { - value.prepareForCachePublication(key); - } - return value; - })); + IcebergSnapshotCacheValue snapshotValue = entry.get(key, + ignored -> executeAuthenticated(nameMapping.getCtlId(), () -> { + Table projectionTable = isolateForQueries + ? tableValue.newQueryScopedTable() : tableValue.getIcebergTable(); + IcebergSnapshotCacheValue value = loadSnapshotProjection( + dorisTable, projectionTable, + tableValue.getRetainedIcebergTable(), + tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries); + if (entry.isWeightBounded()) { + value.prepareForCachePublication(key); + } + return value; + })); + IcebergTableCacheValue currentTable = tableEntry.get(nameMapping.getCtlId()).peekIfPresent(nameMapping); + if (currentTable != null && !tableValue.isSamePhysicalGeneration(currentTable)) { + // A query may have captured the previous table immediately before refresh publication. + // It can use that immutable value, but must not republish an unreachable old projection. + entry.invalidateKeyIfSame(key, snapshotValue); + } + return snapshotValue; } public List getSnapshotList(ExternalTable dorisTable) { @@ -226,8 +220,31 @@ public View getIcebergView(ExternalTable dorisTable) { } public IcebergSchemaCacheValue getIcebergSchemaCacheValue(NameMapping nameMapping, long schemaId) { - IcebergSchemaCacheKey key = new IcebergSchemaCacheKey(nameMapping, schemaId); - SchemaCacheValue schemaCacheValue = schemaEntry.get(nameMapping.getCtlId()).get(key); + IcebergTableCacheValue tableValue = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + return getIcebergSchemaCacheValue(nameMapping, schemaId, tableValue.getRetainedIcebergTable()); + } + + IcebergSchemaCacheValue getIcebergSchemaCacheValue( + NameMapping nameMapping, long schemaId, Table retainedTable) { + Optional generation = IcebergSnapshotEntryKey.tryCreate(nameMapping, retainedTable); + if (!generation.isPresent()) { + return (IcebergSchemaCacheValue) loadSchemaCacheValue( + new IcebergSchemaCacheKey(nameMapping, schemaId), retainedTable); + } + IcebergSchemaCacheKey key = new IcebergSchemaCacheKey( + nameMapping, generation.get().getTableUuid(), schemaId); + MetaCacheEntry entry = schemaEntry.get(nameMapping.getCtlId()); + SchemaCacheValue schemaCacheValue = entry + .get(key, ignored -> loadSchemaCacheValue(key, retainedTable)); + IcebergTableCacheValue currentTable = tableEntry.get(nameMapping.getCtlId()).peekIfPresent(nameMapping); + if (currentTable != null) { + Optional currentGeneration = IcebergSnapshotEntryKey.tryCreate( + nameMapping, currentTable.getRetainedIcebergTable()); + if (!currentGeneration.isPresent() + || !currentGeneration.get().getTableUuid().equals(generation.get().getTableUuid())) { + entry.invalidateKeyIfSame(key, schemaCacheValue); + } + } return (IcebergSchemaCacheValue) schemaCacheValue; } @@ -244,7 +261,8 @@ public ManifestCacheValue getManifestCacheValue(ExternalTable dorisTable, cacheHitRecorder.accept(hit); } return manifestEntry.get(key, - ignored -> loadManifestCacheValue(manifest, icebergTable, key.getContent())); + ignored -> loadManifestCacheValue( + manifest, icebergTable, key.getContent(), manifestEntry.isWeightBounded())); } @Override @@ -269,8 +287,7 @@ private IcebergTableCacheValue loadTableCacheValue(NameMapping nameMapping) { IcebergMetadataOps ops = resolveMetadataOps(catalog); return executeAuthenticated(catalog, () -> { Table table = ops.loadTable(nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName()); - IcebergTableCacheValue value = new IcebergTableCacheValue( - table, ((ExternalCatalog) catalog).getExecutionAuthenticator()); + IcebergTableCacheValue value = new IcebergTableCacheValue(table); MetaCacheEntry currentEntry = tableEntry.getIfInitialized(nameMapping.getCtlId()); if (currentEntry != null && currentEntry.isWeightBounded()) { @@ -300,7 +317,7 @@ private View loadView(NameMapping nameMapping) { } private ManifestCacheValue loadManifestCacheValue(org.apache.iceberg.ManifestFile manifest, Table icebergTable, - ManifestContent content) { + ManifestContent content, boolean accountRetainedSize) { if (manifest == null || icebergTable == null) { String manifestPath = manifest == null ? "null" : manifest.path(); throw new CacheException("Manifest cache loader context is missing for %s", @@ -308,9 +325,9 @@ private ManifestCacheValue loadManifestCacheValue(org.apache.iceberg.ManifestFil } try { if (content == ManifestContent.DELETES) { - return loadDeleteFiles(manifest, icebergTable); + return loadDeleteFiles(manifest, icebergTable, accountRetainedSize); } - return loadDataFiles(manifest, icebergTable); + return loadDataFiles(manifest, icebergTable, accountRetainedSize); } catch (IOException e) { throw new CacheException("Failed to read manifest %s", e, manifest.path()); } @@ -324,6 +341,38 @@ private SchemaCacheValue loadSchemaCacheValue(IcebergSchemaCacheKey key) { key.getNameMapping().getLocalTblName(), key.getSchemaId())); } + private SchemaCacheValue loadSchemaCacheValue(IcebergSchemaCacheKey key, Table retainedTable) { + ExternalTable dorisTable = findExternalTable(key.getNameMapping(), ENGINE); + dorisTable.setUpdateTime(System.currentTimeMillis()); + boolean isView = dorisTable instanceof IcebergExternalTable + && ((IcebergExternalTable) dorisTable).isView(); + return IcebergUtils.loadSchemaCacheValue( + dorisTable, key.getSchemaId(), isView, retainedTable).orElseThrow(() -> + new CacheException("failed to load iceberg schema cache value for: %s.%s.%s, schemaId: %s", + null, key.getNameMapping().getCtlId(), key.getNameMapping().getLocalDbName(), + key.getNameMapping().getLocalTblName(), key.getSchemaId())); + } + + private void retireTableGeneration(NameMapping nameMapping, + @Nullable IcebergTableCacheValue previousValue, IcebergTableCacheValue currentValue) { + if (previousValue != null && previousValue.isSamePhysicalGeneration(currentValue)) { + return; + } + MetaCacheEntry snapshots = + snapshotEntry.getIfInitialized(nameMapping.getCtlId()); + if (snapshots != null) { + snapshots.invalidateIf(key -> key.getNameMapping().equals(nameMapping) + && !key.belongsTo(currentValue)); + } + Optional currentUuid = currentValue.getTableUuid(); + MetaCacheEntry schemas = + schemaEntry.getIfInitialized(nameMapping.getCtlId()); + if (schemas != null) { + schemas.invalidateIf(key -> key.getNameMapping().equals(nameMapping) + && !key.getTableUuid().equals(currentUuid)); + } + } + private IcebergSnapshotCacheValue loadSnapshotProjection( ExternalTable dorisTable, Table projectionTable, Table retainedTable, String retainedCurrentSnapshotJson, boolean isolateForQueries) { @@ -393,9 +442,10 @@ protected Map catalogPropertyCompatibilityMap() { return compatibility; } - private ManifestCacheValue loadDataFiles(org.apache.iceberg.ManifestFile manifest, Table table) + private ManifestCacheValue loadDataFiles( + org.apache.iceberg.ManifestFile manifest, Table table, boolean accountRetainedSize) throws IOException { - ManifestCacheValue.Builder builder = ManifestCacheValue.dataFilesBuilder(); + ManifestCacheValue.Builder builder = ManifestCacheValue.dataFilesBuilder(accountRetainedSize); try (ManifestReader reader = ManifestFiles.read(manifest, table.io())) { for (org.apache.iceberg.DataFile dataFile : reader) { builder.addDataFile(dataFile.copy()); @@ -404,9 +454,10 @@ private ManifestCacheValue loadDataFiles(org.apache.iceberg.ManifestFile manifes return builder.build(); } - private ManifestCacheValue loadDeleteFiles(org.apache.iceberg.ManifestFile manifest, Table table) + private ManifestCacheValue loadDeleteFiles( + org.apache.iceberg.ManifestFile manifest, Table table, boolean accountRetainedSize) throws IOException { - ManifestCacheValue.Builder builder = ManifestCacheValue.deleteFilesBuilder(); + ManifestCacheValue.Builder builder = ManifestCacheValue.deleteFilesBuilder(accountRetainedSize); try (ManifestReader reader = ManifestFiles.readDeleteManifest(manifest, table.io(), table.specs())) { for (org.apache.iceberg.DeleteFile deleteFile : reader) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java index 7c2d09511a2c93..9916d0afbb17e1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSchemaCacheKey.java @@ -22,14 +22,26 @@ import com.google.common.base.Objects; +import java.util.Optional; + public class IcebergSchemaCacheKey extends SchemaCacheKey { + private final String tableUuid; private final long schemaId; public IcebergSchemaCacheKey(NameMapping nameMapping, long schemaId) { + this(nameMapping, "", schemaId); + } + + public IcebergSchemaCacheKey(NameMapping nameMapping, String tableUuid, long schemaId) { super(nameMapping); + this.tableUuid = java.util.Objects.requireNonNull(tableUuid, "tableUuid can not be null"); this.schemaId = schemaId; } + public Optional getTableUuid() { + return tableUuid.isEmpty() ? Optional.empty() : Optional.of(tableUuid); + } + public long getSchemaId() { return schemaId; } @@ -46,11 +58,11 @@ public boolean equals(Object o) { return false; } IcebergSchemaCacheKey that = (IcebergSchemaCacheKey) o; - return schemaId == that.schemaId; + return schemaId == that.schemaId && tableUuid.equals(that.tableUuid); } @Override public int hashCode() { - return Objects.hashCode(super.hashCode(), schemaId); + return Objects.hashCode(super.hashCode(), tableUuid, schemaId); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index 406d4ee11ab003..7cfc5bcc2bcd31 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -17,27 +17,19 @@ package org.apache.doris.datasource.iceberg; -import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import com.google.common.collect.ImmutableList; import org.apache.iceberg.BaseTable; -import org.apache.iceberg.DataFile; -import org.apache.iceberg.DeleteFile; import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.HistoryEntry; -import org.apache.iceberg.ManifestFile; -import org.apache.iceberg.PartitionSpec; -import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; import org.apache.iceberg.SnapshotParser; import org.apache.iceberg.SnapshotRef; -import org.apache.iceberg.SortOrder; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; -import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.TableOperations; import org.apache.iceberg.encryption.EncryptionManager; import org.apache.iceberg.exceptions.CommitFailedException; @@ -138,14 +130,16 @@ public Optional
    getIcebergTable() { MetaCacheSizeEstimate prepareForCachePublication(IcebergSnapshotEntryKey key) { if (sizeEstimate == null) { - if (retainedCurrentSnapshotJson == null) { - retainedCurrentSnapshotJson = icebergTable - .map(IcebergSnapshotCacheValue::retainCurrentSnapshotJson).orElse(null); - } - retainedTablePayloadBytes = icebergTable - .map(IcebergCacheSizeEstimator::retainedTablePayloadBytes).orElse(0L); sizeEstimate = MetaCacheSizeEstimator.estimateSafely("iceberg_snapshot_preparation_failed", - () -> IcebergCacheSizeEstimator.estimateSnapshotEntry(key, this)); + () -> { + if (retainedCurrentSnapshotJson == null) { + retainedCurrentSnapshotJson = icebergTable + .map(IcebergSnapshotCacheValue::retainCurrentSnapshotJson).orElse(null); + } + retainedTablePayloadBytes = icebergTable + .map(IcebergCacheSizeEstimator::retainedTablePayloadBytes).orElse(0L); + return IcebergCacheSizeEstimator.estimateSnapshotEntry(key, this); + }); if (sizeEstimate.isComplete()) { icebergTable = icebergTable.map( IcebergSnapshotCacheValue::retainNonGrowingGeneration); @@ -177,10 +171,6 @@ Optional
    getRetainedIcebergTable() { } static Table retainTableGeneration(Table table) { - return retainTableGeneration(table, null); - } - - static Table retainTableGeneration(Table table, ExecutionAuthenticator authenticator) { if (!(table instanceof HasTableOperations) || isFrozenGeneration(table)) { return table; } @@ -188,46 +178,21 @@ static Table retainTableGeneration(Table table, ExecutionAuthenticator authentic // Capture current() exactly once so every projection derived from the returned table sees // one metadata generation even when the shared catalog handle refreshes concurrently. TableOperations frozenOperations = new FrozenTableOperations( - operations, operations.current(), authenticator); + operations, operations.current(), false); return tableWithOperations(table, frozenOperations); } static Table retainNonGrowingGeneration(Table table) { - if (!isFrozenGeneration(table)) { + if (!isFrozenGeneration(table) || isNonGrowingGeneration(table)) { return table; } TableOperations retainedOperations = ((HasTableOperations) table).operations(); - TableMetadata source = retainedOperations.current(); - if (source.schemas().isEmpty() || source.specs().isEmpty() - || source.sortOrders().isEmpty()) { - return table; - } - TableMetadata.Builder builder = TableMetadata.buildFromEmpty(source.formatVersion()); - if (source.uuid() != null) { - builder.assignUUID(source.uuid()); - } - for (Schema schema : source.schemas()) { - builder.addSchema(schema); - } - builder.setCurrentSchema(source.currentSchemaId()); - for (PartitionSpec spec : source.specs()) { - builder.addPartitionSpec(spec); - } - builder.setDefaultPartitionSpec(source.defaultSpecId()); - for (SortOrder sortOrder : source.sortOrders()) { - builder.addSortOrder(sortOrder); - } - builder.setDefaultSortOrder(source.defaultSortOrderId()); - builder.setLocation(source.location()); - builder.setProperties(source.properties()); - if (source.currentSnapshot() != null) { - builder.setBranchSnapshot( - new NonGrowingSnapshot(source.currentSnapshot()), SnapshotRef.MAIN_BRANCH); - } - TableMetadata retainedMetadata = builder.discardChanges() - .withMetadataLocation(source.metadataFileLocation()).build(); + // Do not rebuild parsed metadata with Iceberg's write-side Builder. Builder validation and + // ID reuse rules are intentionally stricter than metadata parsing and can reject legal + // upgraded tables or renumber sparse/equivalent schema histories. The frozen metadata is + // never exposed after query isolation; each caller receives exact query-local operations. return tableWithOperations(table, new FrozenTableOperations( - retainedOperations, retainedMetadata)); + retainedOperations, retainedOperations.current(), true)); } static String retainCurrentSnapshotJson(Table table) { @@ -275,8 +240,7 @@ static TableOperations unwrapRetainedTableOperations(TableOperations operations) return current; } - static Table createWritableTable( - Table retainedTable, Table liveTable, boolean reloadRetainedMetadata) { + static Table createWritableTable(Table retainedTable, Table liveTable) { if (!isFrozenGeneration(retainedTable)) { return retainedTable; } @@ -286,45 +250,18 @@ static Table createWritableTable( "Iceberg commit table must provide writable table operations"); } TableOperations retainedOperations = ((HasTableOperations) retainedTable).operations(); - TableMetadata retainedMetadata = reloadRetainedMetadata - ? loadQueryMetadata(retainedOperations) : retainedOperations.current(); + TableMetadata retainedMetadata = retainedOperations.current(); TableOperations liveOperations = unwrapRetainedTableOperations( ((HasTableOperations) liveTable).operations()); return tableWithOperations(retainedTable, new WritableTableOperations(liveOperations, retainedMetadata)); } - static Table createWritableTable(Table retainedTable, Table liveTable) { - return createWritableTable(retainedTable, liveTable, - isNonGrowingGeneration(retainedTable)); - } - - private static boolean isNonGrowingGeneration(Table table) { + static boolean isNonGrowingGeneration(Table table) { return isFrozenGeneration(table) && ((FrozenTableOperations) ((HasTableOperations) table).operations()).nonGrowing; } - private static TableMetadata loadQueryMetadata(TableOperations retainedOperations) { - TableMetadata retainedMetadata = retainedOperations.current(); - TableOperations serviceOperations = unwrapRetainedTableOperations(retainedOperations); - String metadataLocation = retainedMetadata.metadataFileLocation(); - if (metadataLocation != null && !metadataLocation.isEmpty() && serviceOperations.io() != null) { - ExecutionAuthenticator authenticator = retainedOperations instanceof FrozenTableOperations - ? ((FrozenTableOperations) retainedOperations).authenticator : null; - try { - return authenticator == null - ? TableMetadataParser.read(serviceOperations.io(), metadataLocation) - : authenticator.execute( - () -> TableMetadataParser.read(serviceOperations.io(), metadataLocation)); - } catch (Exception e) { - throw new StaleMetadataException( - "Iceberg metadata generation is no longer readable: " + metadataLocation, e); - } - } - throw new IllegalStateException( - "Iceberg query-local metadata requires a stable metadata location and FileIO"); - } - private static Table tableWithOperations(Table table, TableOperations operations) { if (table instanceof BaseTable) { return new BaseTable(operations, table.name(), ((BaseTable) table).reporter()); @@ -377,26 +314,14 @@ private static class FrozenTableOperations implements TableOperations { private final FileIO fileIO; private final EncryptionManager encryptionManager; private final LocationProvider locationProvider; - private final ExecutionAuthenticator authenticator; private final boolean nonGrowing; - private FrozenTableOperations(TableOperations source, TableMetadata metadata) { - this(source, metadata, source instanceof FrozenTableOperations - ? ((FrozenTableOperations) source).authenticator : null, true); - } - - private FrozenTableOperations(TableOperations source, TableMetadata metadata, - ExecutionAuthenticator authenticator) { - this(source, metadata, authenticator, false); - } - private FrozenTableOperations(TableOperations source, TableMetadata metadata, - ExecutionAuthenticator authenticator, boolean nonGrowing) { + boolean nonGrowing) { this.metadata = metadata; this.fileIO = source.io(); this.encryptionManager = source.encryption(); this.locationProvider = source.locationProvider(); - this.authenticator = authenticator; this.nonGrowing = nonGrowing; } @@ -489,19 +414,39 @@ private boolean isWriterCompatible(TableMetadata refreshedMetadata) { } } + /** Query-local operations expose exact retained metadata without shared lazy table state. */ + private static final class QueryScopedTableOperations extends RetainedTableOperations { + private QueryScopedTableOperations(TableOperations retainedOperations) { + super(retainedOperations, retainedOperations.current()); + } + + @Override + public void commit(TableMetadata base, TableMetadata metadata) { + throw new UnsupportedOperationException("Query-scoped Iceberg table is read-only"); + } + } + /** A per-caller view whose Iceberg lazy snapshot state is never written into the cache value. */ private static final class QueryScopedTable extends BaseTable { - private final TableOperations retainedOperations; + private final QueryScopedTableOperations queryOperations; private final Snapshot currentSnapshot; - private TableMetadata queryMetadata; + private final Map querySnapshots = new HashMap<>(); private QueryScopedTable(TableOperations retainedOperations, String name, org.apache.iceberg.metrics.MetricsReporter reporter, String currentSnapshotJson) { - super(retainedOperations, name, reporter == null + this(new QueryScopedTableOperations(retainedOperations), name, reporter, currentSnapshotJson); + } + + private QueryScopedTable(QueryScopedTableOperations queryOperations, String name, + org.apache.iceberg.metrics.MetricsReporter reporter, String currentSnapshotJson) { + super(queryOperations, name, reporter == null ? org.apache.iceberg.metrics.LoggingMetricsReporter.instance() : reporter); - this.retainedOperations = retainedOperations; + this.queryOperations = queryOperations; this.currentSnapshot = currentSnapshotJson == null ? null : SnapshotParser.fromJson(currentSnapshotJson); + if (currentSnapshot != null) { + querySnapshots.put(currentSnapshot.snapshotId(), currentSnapshot); + } } @Override @@ -514,12 +459,16 @@ public Snapshot snapshot(long snapshotId) { if (currentSnapshot != null && currentSnapshot.snapshotId() == snapshotId) { return currentSnapshot; } - return queryMetadata().snapshot(snapshotId); + return copyForQuery(queryMetadata().snapshot(snapshotId)); } @Override public Iterable snapshots() { - return queryMetadata().snapshots(); + ImmutableList.Builder snapshots = ImmutableList.builder(); + for (Snapshot snapshot : queryMetadata().snapshots()) { + snapshots.add(copyForQuery(snapshot)); + } + return snapshots.build(); } @Override @@ -543,132 +492,15 @@ public List partitionStatisticsFiles } private synchronized TableMetadata queryMetadata() { - if (queryMetadata == null) { - queryMetadata = loadQueryMetadata(retainedOperations); - } - return queryMetadata; - } - } - - static final class StaleMetadataException extends RuntimeException { - private StaleMetadataException(String message, Throwable cause) { - super(message, cause); - } - } - - /** Scalar-only snapshot retained by the cache-owned metadata generation. */ - private static final class NonGrowingSnapshot implements Snapshot { - private final long sequenceNumber; - private final long snapshotId; - private final Long parentId; - private final long timestampMillis; - private final Integer schemaId; - private final Long firstRowId; - private final Long addedRows; - - private NonGrowingSnapshot(Snapshot snapshot) { - this.sequenceNumber = snapshot.sequenceNumber(); - this.snapshotId = snapshot.snapshotId(); - this.parentId = snapshot.parentId(); - this.timestampMillis = snapshot.timestampMillis(); - this.schemaId = snapshot.schemaId(); - this.firstRowId = snapshot.firstRowId(); - this.addedRows = snapshot.addedRows(); - } - - @Override - public long sequenceNumber() { - return sequenceNumber; - } - - @Override - public long snapshotId() { - return snapshotId; - } - - @Override - public Long parentId() { - return parentId; - } - - @Override - public long timestampMillis() { - return timestampMillis; - } - - @Override - public List allManifests(FileIO fileIO) { - throw queryScopedSnapshotRequired(); - } - - @Override - public List dataManifests(FileIO fileIO) { - throw queryScopedSnapshotRequired(); - } - - @Override - public List deleteManifests(FileIO fileIO) { - throw queryScopedSnapshotRequired(); + return queryOperations.current(); } - @Override - public String operation() { - return null; - } - - @Override - public Map summary() { - return Collections.emptyMap(); - } - - @Override - public Iterable addedDataFiles(FileIO fileIO) { - throw queryScopedSnapshotRequired(); - } - - @Override - public Iterable removedDataFiles(FileIO fileIO) { - throw queryScopedSnapshotRequired(); - } - - @Override - public Iterable addedDeleteFiles(FileIO fileIO) { - throw queryScopedSnapshotRequired(); - } - - @Override - public Iterable removedDeleteFiles(FileIO fileIO) { - throw queryScopedSnapshotRequired(); - } - - @Override - public String manifestListLocation() { - return null; - } - - @Override - public Integer schemaId() { - return schemaId; - } - - @Override - public Long firstRowId() { - return firstRowId; - } - - @Override - public Long addedRows() { - return addedRows; - } - - @Override - public String keyId() { - return null; - } - - private UnsupportedOperationException queryScopedSnapshotRequired() { - return new UnsupportedOperationException( - "Cache-owned Iceberg snapshots cannot materialize manifests or files"); + private synchronized Snapshot copyForQuery(Snapshot snapshot) { + if (snapshot == null) { + return null; + } + return querySnapshots.computeIfAbsent(snapshot.snapshotId(), ignored -> + SnapshotParser.fromJson(SnapshotParser.toJson(snapshot, false))); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotEntryKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotEntryKey.java index 782a51d5f135db..13db520e2d755d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotEntryKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotEntryKey.java @@ -91,6 +91,14 @@ public int getDefaultSpecId() { return defaultSpecId; } + boolean belongsTo(IcebergTableCacheValue tableValue) { + Optional generation = tryCreate( + nameMapping, tableValue.getRetainedIcebergTable()); + return generation.isPresent() + && tableUuid.equals(generation.get().tableUuid) + && metadataFileLocation.equals(generation.get().metadataFileLocation); + } + @Override public boolean equals(Object object) { if (this == object) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java index 28e45b47acd250..5bb537340ba6e0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java @@ -44,9 +44,6 @@ public class IcebergSysExternalTable extends ExternalTable { private final IcebergExternalTable sourceTable; private final String sysTableType; - private volatile Table sysIcebergTable; - private volatile List fullSchema; - private volatile SchemaCacheValue schemaCacheValue; public IcebergSysExternalTable(IcebergExternalTable sourceTable, String sysTableType) { super(generateSysTableId(sourceTable.getId(), sysTableType), @@ -100,24 +97,19 @@ public boolean supportsSnapshotSelection() { } public Table getSysIcebergTable() { - if (sysIcebergTable == null) { - synchronized (this) { - if (sysIcebergTable == null) { - Table baseTable = sourceTable.getIcebergTable(); - MetadataTableType tableType = MetadataTableType.from(sysTableType); - if (tableType == null) { - throw new IllegalArgumentException("Unknown iceberg system table type: " + sysTableType); - } - sysIcebergTable = MetadataTableUtils.createMetadataTableInstance(baseTable, tableType); - } - } + Table baseTable = IcebergUtils.getQueryScopedIcebergTable(sourceTable); + MetadataTableType tableType = MetadataTableType.from(sysTableType); + if (tableType == null) { + throw new IllegalArgumentException("Unknown iceberg system table type: " + sysTableType); } - return sysIcebergTable; + // Metadata tables capture their base operations. Keep them statement-local so exact + // previousFiles/history state and stale-generation retry never leak into this table object. + return MetadataTableUtils.createMetadataTableInstance(baseTable, tableType); } @Override public List getFullSchema() { - return getOrCreateSchemaCacheValue().getSchema(); + return loadSchemaCacheValue().getSchema(); } @Override @@ -156,12 +148,12 @@ public long fetchRowCount() { @Override public Optional initSchema(SchemaCacheKey key) { - return Optional.of(getOrCreateSchemaCacheValue()); + return Optional.of(loadSchemaCacheValue()); } @Override public Optional getSchemaCacheValue() { - return Optional.of(getOrCreateSchemaCacheValue()); + return Optional.of(loadSchemaCacheValue()); } @Override @@ -178,19 +170,12 @@ private static long generateSysTableId(long sourceTableId, String sysTableType) return sourceTableId ^ (sysTableType.hashCode() * 31L); } - private SchemaCacheValue getOrCreateSchemaCacheValue() { - if (schemaCacheValue == null) { - synchronized (this) { - if (schemaCacheValue == null) { - if (fullSchema == null) { - fullSchema = IcebergUtils.parseSchema(getSysIcebergTable().schema(), - getCatalog().getEnableMappingVarbinary(), - getCatalog().getEnableMappingTimestampTz()); - } - schemaCacheValue = new SchemaCacheValue(fullSchema); - } - } - } - return schemaCacheValue; + private SchemaCacheValue loadSchemaCacheValue() { + // Metadata-table schemas may change after source schema or partition-spec evolution. + // Resolve the schema from the same latest-generation path instead of permanently pairing + // this long-lived system-table object with its first observed generation. + return new SchemaCacheValue(IcebergUtils.parseSchema(getSysIcebergTable().schema(), + getCatalog().getEnableMappingVarbinary(), + getCatalog().getEnableMappingTimestampTz())); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java index 6403e6bfdbafa8..a4d4f600f66d6e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java @@ -17,15 +17,19 @@ package org.apache.doris.datasource.iceberg; -import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.Table; +import org.apache.iceberg.TableMetadata; + +import java.util.Objects; +import java.util.Optional; public class IcebergTableCacheValue { - private Table icebergTable; + private volatile Table icebergTable; private String retainedCurrentSnapshotJson; private volatile boolean queryIsolationPrepared; private long retainedTablePayloadBytes; @@ -35,30 +39,29 @@ public IcebergTableCacheValue(Table icebergTable) { this.icebergTable = IcebergSnapshotCacheValue.retainTableGeneration(icebergTable); } - IcebergTableCacheValue(Table icebergTable, ExecutionAuthenticator authenticator) { - this.icebergTable = IcebergSnapshotCacheValue.retainTableGeneration( - icebergTable, authenticator); - } - public Table getIcebergTable() { - return queryIsolationPrepared + Table retainedTable = icebergTable; + return queryIsolationPrepared || IcebergSnapshotCacheValue.isNonGrowingGeneration(retainedTable) ? IcebergSnapshotCacheValue.createQueryScopedTable( - icebergTable, retainedCurrentSnapshotJson) - : icebergTable; + retainedTable, retainedCurrentSnapshotJson) + : retainedTable; } public Table getWritableIcebergTable(Table liveTable) { - return IcebergSnapshotCacheValue.createWritableTable( - icebergTable, liveTable, queryIsolationPrepared); + Table retainedTable = icebergTable; + return IcebergSnapshotCacheValue.createWritableTable(retainedTable, liveTable); } - MetaCacheSizeEstimate prepareForCachePublication(NameMapping key) { + synchronized MetaCacheSizeEstimate prepareForCachePublication(NameMapping key) { if (sizeEstimate == null) { - retainedCurrentSnapshotJson = - IcebergSnapshotCacheValue.retainCurrentSnapshotJson(icebergTable); - retainedTablePayloadBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes(icebergTable); sizeEstimate = MetaCacheSizeEstimator.estimateSafely("iceberg_table_preparation_failed", - () -> IcebergCacheSizeEstimator.estimateTableEntry(key, this)); + () -> { + retainedCurrentSnapshotJson = + IcebergSnapshotCacheValue.retainCurrentSnapshotJson(icebergTable); + retainedTablePayloadBytes = + IcebergCacheSizeEstimator.retainedTablePayloadBytes(icebergTable); + return IcebergCacheSizeEstimator.estimateTableEntry(key, this); + }); if (sizeEstimate.isComplete()) { icebergTable = IcebergSnapshotCacheValue.retainNonGrowingGeneration(icebergTable); queryIsolationPrepared = true; @@ -78,6 +81,12 @@ Table getRetainedIcebergTable() { synchronized Table newQueryScopedTable() { if (!queryIsolationPrepared) { + // A failed optional size preparation must only reject weighted cache admission. Do not + // repeat the same unsupported metadata access on the query path and turn it into a + // table-load failure; this value is not retained by the weighted cache in that case. + if (sizeEstimate != null && !sizeEstimate.isComplete()) { + return icebergTable; + } retainedCurrentSnapshotJson = IcebergSnapshotCacheValue.retainCurrentSnapshotJson(icebergTable); icebergTable = IcebergSnapshotCacheValue.retainNonGrowingGeneration(icebergTable); @@ -103,4 +112,27 @@ long getRetainedCurrentSnapshotPayloadBytes() { return IcebergSnapshotCacheValue.retainedSnapshotJsonBytes( retainedCurrentSnapshotJson); } + + Optional getTableUuid() { + TableMetadata metadata = retainedMetadata(); + return metadata == null || metadata.uuid() == null || metadata.uuid().isEmpty() + ? Optional.empty() : Optional.of(metadata.uuid()); + } + + boolean isSamePhysicalGeneration(IcebergTableCacheValue other) { + if (other == null) { + return false; + } + TableMetadata left = retainedMetadata(); + TableMetadata right = other.retainedMetadata(); + return left != null && right != null + && Objects.equals(left.uuid(), right.uuid()) + && Objects.equals(left.metadataFileLocation(), right.metadataFileLocation()); + } + + private TableMetadata retainedMetadata() { + Table retainedTable = icebergTable; + return retainedTable instanceof HasTableOperations + ? ((HasTableOperations) retainedTable).operations().current() : null; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 1c370222d71a83..bee37c0c37cea4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -1058,6 +1058,10 @@ public static Table getIcebergTable(ExternalTable dorisTable) { return icebergExternalMetaCache(dorisTable).getIcebergTable(dorisTable); } + public static Table getQueryScopedIcebergTable(ExternalTable dorisTable) { + return icebergExternalMetaCache(dorisTable).getQueryScopedIcebergTable(dorisTable); + } + public static Table getWritableIcebergTable(ExternalTable dorisTable) { return icebergExternalMetaCache(dorisTable).getWritableIcebergTable(dorisTable); } @@ -1721,6 +1725,12 @@ public static IcebergSchemaCacheValue getSchemaCacheValue(ExternalTable dorisTab .getIcebergSchemaCacheValue(dorisTable.getOrBuildNameMapping(), schemaId); } + static IcebergSchemaCacheValue getSchemaCacheValue( + ExternalTable dorisTable, long schemaId, Table retainedTable) { + return icebergExternalMetaCache(dorisTable).getIcebergSchemaCacheValue( + dorisTable.getOrBuildNameMapping(), schemaId, retainedTable); + } + public static IcebergSnapshot getLatestIcebergSnapshot(Table table) { Snapshot snapshot = table.currentSnapshot(); long snapshotId = snapshot == null ? IcebergUtils.UNKNOWN_SNAPSHOT_ID : snapshot.snapshotId(); @@ -1758,7 +1768,8 @@ public static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, T Map nameToPartitionItem = Maps.newHashMap(); long retainedPayloadBytes = 0L; - List partitionColumns = IcebergUtils.getSchemaCacheValue(dorisTable, schemaId).getPartitionColumns(); + List partitionColumns = IcebergUtils.getSchemaCacheValue( + dorisTable, schemaId, table).getPartitionColumns(); for (IcebergPartition partition : icebergPartitions) { nameToPartition.put(partition.getPartitionName(), partition); retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( @@ -1996,7 +2007,10 @@ public int compare(Map.Entry p1, Map.Entry retainedTable = sv.getRetainedIcebergTable(); + return retainedTable.isPresent() + ? getSchemaCacheValue(dorisTable, sv.getSnapshot().getSchemaId(), retainedTable.get()) + : getSchemaCacheValue(dorisTable, sv.getSnapshot().getSchemaId()); } public static IcebergSnapshotCacheValue getLatestSnapshotCacheValue(ExternalTable dorisTable) { @@ -2065,9 +2079,16 @@ public static View getIcebergView(ExternalTable dorisTable) { public static Optional loadSchemaCacheValue( ExternalTable dorisTable, long schemaId, boolean isView) { + return loadSchemaCacheValue(dorisTable, schemaId, isView, null); + } + + public static Optional loadSchemaCacheValue( + ExternalTable dorisTable, long schemaId, boolean isView, Table retainedTable) { return isView ? loadViewSchemaCacheValue(dorisTable, schemaId) - : loadTableSchemaCacheValue(dorisTable, schemaId); + : retainedTable == null + ? loadTableSchemaCacheValue(dorisTable, schemaId) + : Optional.of(buildTableSchemaCacheValue(dorisTable, schemaId, retainedTable)); } private static Optional loadViewSchemaCacheValue(ExternalTable dorisTable, long schemaId) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java index 47f8c8cde3e7df..4d63af348324e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java @@ -41,14 +41,17 @@ public class ManifestCacheValue { private final long dataFileMetricEntryCount; private final long deleteFileMetricEntryCount; private final long retainedPayloadBytes; + private final boolean accountingComplete; private ManifestCacheValue(List dataFiles, List deleteFiles, - long dataFileMetricEntryCount, long deleteFileMetricEntryCount, long retainedPayloadBytes) { + long dataFileMetricEntryCount, long deleteFileMetricEntryCount, long retainedPayloadBytes, + boolean accountingComplete) { this.dataFiles = ImmutableList.copyOf(dataFiles); this.deleteFiles = ImmutableList.copyOf(deleteFiles); this.dataFileMetricEntryCount = dataFileMetricEntryCount; this.deleteFileMetricEntryCount = deleteFileMetricEntryCount; this.retainedPayloadBytes = retainedPayloadBytes; + this.accountingComplete = accountingComplete; } public static ManifestCacheValue forDataFiles(List dataFiles) { @@ -68,11 +71,19 @@ public static ManifestCacheValue forDeleteFiles(List deleteFiles) { } public static Builder dataFilesBuilder() { - return new Builder(true); + return dataFilesBuilder(true); + } + + public static Builder dataFilesBuilder(boolean accountRetainedSize) { + return new Builder(true, accountRetainedSize); } public static Builder deleteFilesBuilder() { - return new Builder(false); + return deleteFilesBuilder(true); + } + + public static Builder deleteFilesBuilder(boolean accountRetainedSize) { + return new Builder(false, accountRetainedSize); } public List getDataFiles() { @@ -95,16 +106,24 @@ public long getRetainedPayloadBytes() { return retainedPayloadBytes; } + public boolean isAccountingComplete() { + return accountingComplete; + } + /** Accumulates retained-size counters in the manifest reader's existing file loop. */ public static final class Builder { private final boolean dataContent; + private final boolean accountRetainedSize; private final List dataFiles = new ArrayList<>(); private final List deleteFiles = new ArrayList<>(); private long metricEntryCount; private long retainedPayloadBytes; + private boolean accountingComplete; - private Builder(boolean dataContent) { + private Builder(boolean dataContent, boolean accountRetainedSize) { this.dataContent = dataContent; + this.accountRetainedSize = accountRetainedSize; + this.accountingComplete = accountRetainedSize; } public void addDataFile(DataFile file) { @@ -112,7 +131,7 @@ public void addDataFile(DataFile file) { throw new IllegalStateException("delete manifest builder cannot accept a data file"); } dataFiles.add(file); - account(file); + accountSafely(file); } public void addDeleteFile(DeleteFile file) { @@ -120,14 +139,30 @@ public void addDeleteFile(DeleteFile file) { throw new IllegalStateException("data manifest builder cannot accept a delete file"); } deleteFiles.add(file); - account(file); + accountSafely(file); } public ManifestCacheValue build() { return new ManifestCacheValue(dataFiles, deleteFiles, dataContent ? metricEntryCount : 0L, dataContent ? 0L : metricEntryCount, - retainedPayloadBytes); + retainedPayloadBytes, accountingComplete); + } + + private void accountSafely(ContentFile file) { + if (!accountRetainedSize || !accountingComplete) { + return; + } + try { + account(file); + } catch (RuntimeException e) { + // A new or third-party ContentFile implementation must not turn optional cache + // accounting into a manifest-read failure. Keep the files for the current query + // and mark the value incomplete so weighted admission rejects it. + metricEntryCount = 0L; + retainedPayloadBytes = 0L; + accountingComplete = false; + } } private void account(ContentFile file) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java index 4cf326b9110b8d..906804e5307c21 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java @@ -27,6 +27,8 @@ import org.apache.doris.datasource.SchemaCacheValue; import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.Collection; import java.util.Collections; @@ -47,6 +49,8 @@ * to initialize a catalog explicitly before accessing entries. */ public abstract class AbstractExternalMetaCache implements ExternalMetaCache { + private static final Logger LOG = LogManager.getLogger(AbstractExternalMetaCache.class); + protected static CacheSpec defaultEntryCacheSpec() { return CacheSpec.of( true, @@ -106,6 +110,40 @@ public void initCatalog(long catalogId, Map catalogProperties) { } Map safeCatalogProperties = CacheSpec.applyCompatibilityMap( catalogProperties, catalogPropertyCompatibilityMap()); + safeCatalogProperties = CacheSpec.sanitizeEnginePropertiesForRuntime( + safeCatalogProperties, engine, metaCacheEntryDefs, + warning -> LOG.warn("{} (engine={}, catalog={})", warning, engine, catalogId)); + try { + budgetManager.parseCatalogMaxWeight(safeCatalogProperties); + } catch (IllegalArgumentException e) { + safeCatalogProperties.remove(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY); + LOG.warn("Ignoring invalid persisted external metadata cache property '{}' " + + "for engine {}, catalog {}: {}", + ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, + engine, catalogId, e.getMessage()); + } + OptionalLong runtimeCatalogMaxWeight = budgetManager.parseCatalogMaxWeight(safeCatalogProperties); + for (MetaCacheEntryDef entryDef : metaCacheEntryDefs.values()) { + if (entryDef.getSizeEstimator() == null) { + continue; + } + String maxWeightKey = CacheSpec.metaCacheKeyPrefix(engine) + + entryDef.getName() + ".max-weight"; + if (!safeCatalogProperties.containsKey(maxWeightKey)) { + continue; + } + CacheSpec cacheSpec = CacheSpec.fromProperties( + safeCatalogProperties, engine, entryDef.getName(), entryDef.getDefaultCacheSpec()); + try { + budgetManager.validateCatalogEntryHierarchy( + runtimeCatalogMaxWeight, cacheSpec.getMaxWeight()); + } catch (IllegalArgumentException e) { + safeCatalogProperties.remove(maxWeightKey); + LOG.warn("Ignoring invalid persisted external metadata cache property '{}' " + + "for engine {}, catalog {}: {}", + maxWeightKey, engine, catalogId, e.getMessage()); + } + } validateMappedCatalogProperties(safeCatalogProperties, false); catalogEntries.put(catalogId, buildCatalogEntryGroup(catalogId, safeCatalogProperties)); } @@ -368,7 +406,7 @@ private MetaCacheEntry newMetaCacheEntry( wrapSchemaValidator(entryDef.getLoader(), entryDef.getValueType()), cacheSpec, refreshExecutor, entryDef.isAutoRefresh(), entryDef.isContextualOnly(), - entryDef.getSizeEstimator(), entryBudget); + entryDef.getSizeEstimator(), entryBudget, entryDef.getReplacementListener()); } catch (RuntimeException | Error e) { if (entryBudget != null) { entryBudget.close(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java index 1cea5d20a3a591..34ccf41c718b1b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java @@ -23,12 +23,14 @@ import java.math.BigDecimal; import java.math.BigInteger; +import java.util.Collections; import java.util.HashMap; import java.util.Locale; import java.util.Map; import java.util.Objects; import java.util.OptionalLong; import java.util.Set; +import java.util.function.Consumer; import java.util.regex.Matcher; import java.util.regex.Pattern; @@ -39,7 +41,8 @@ *
      *
    • enable=false disables cache
      • *
    • ttlSecond=0 disables cache, ttlSecond=-1 means no expiration
      • - *
    • capacity=0 disables cache; capacity is count-based
      • + *
    • capacity=0 disables cache; otherwise capacity is the count limit only when max-weight is absent
      • + *
    • when max-weight is present, Caffeine uses the weight limit instead of the positive capacity
      • *
    */ public final class CacheSpec { @@ -268,6 +271,31 @@ static void validateEngineProperties(Map properties, String engi validateEngineProperties(properties, engine, entryDefs.keySet(), weightedEntries); } + /** + * Ignore invalid persisted cache options during image/replay initialization. + * New CREATE/ALTER statements still use {@link #validateEngineProperties} and fail strictly. + */ + static Map sanitizeEnginePropertiesForRuntime( + Map properties, String engine, + Map> entryDefs, Consumer warningConsumer) { + Map sanitized = new HashMap<>(properties); + String enginePrefix = metaCacheKeyPrefix(engine); + for (Map.Entry property : properties.entrySet()) { + String key = property.getKey(); + if (key == null || !key.startsWith(enginePrefix)) { + continue; + } + try { + validateEngineProperties(Collections.singletonMap(key, property.getValue()), engine, entryDefs); + } catch (IllegalArgumentException e) { + sanitized.remove(key); + warningConsumer.accept("Ignoring invalid persisted external metadata cache property '" + + key + "': " + e.getMessage()); + } + } + return sanitized; + } + public static void validateEngineProperties(Map properties, String engine, Set entryNames, Set weightedEntryNames) { if (properties == null || properties.isEmpty()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java index 09376a1a129051..d37e91a8922019 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CatalogEntryGroup.java @@ -18,6 +18,8 @@ package org.apache.doris.datasource.metacache; import com.google.common.collect.Maps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import java.util.Map; import java.util.Objects; @@ -27,6 +29,8 @@ * Catalog scoped entry container. */ public class CatalogEntryGroup { + private static final Logger LOG = LogManager.getLogger(CatalogEntryGroup.class); + private final Map> entries = new ConcurrentHashMap<>(); public MetaCacheEntry get(String entryName) { @@ -48,7 +52,14 @@ public void invalidateAll() { } public void close() { - entries.values().forEach(MetaCacheEntry::close); + entries.forEach((name, entry) -> { + try { + entry.close(); + } catch (RuntimeException e) { + LOG.error("Failed to close external metadata cache entry {}; continuing group retirement", + name, e); + } + }); // Keep the closed entries reachable from this retired group. A query may have captured the // group immediately before its catalog is removed; returning a closed entry lets that query // serve an uncached load instead of spuriously observing an uninitialized entry. The group diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java index a41e53bfb35b95..904a3fb0ad8f75 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java @@ -19,12 +19,22 @@ import org.apache.doris.common.Config; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; import java.util.OptionalLong; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.RejectedExecutionException; +import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicLong; +import java.util.function.LongUnaryOperator; +import java.util.stream.Collectors; /** * FE-wide admission accounting for managed external metadata caches. @@ -34,12 +44,20 @@ * operations while making global/catalog/entry reservation atomic. */ public final class ExternalMetaCacheBudgetManager { + private static final Logger LOG = LogManager.getLogger(ExternalMetaCacheBudgetManager.class); + private static final ExecutorService PEER_RECLAIM_EXECUTOR = Executors.newSingleThreadExecutor(runnable -> { + Thread thread = new Thread(runnable, "external-meta-cache-peer-reclaim"); + thread.setDaemon(true); + return thread; + }); + public static final String CATALOG_MAX_WEIGHT_PROPERTY = "meta.cache.max-weight"; private final Object lock = new Object(); private final OptionalLong globalMaxWeight; private final Map catalogBuckets = new HashMap<>(); private final Map entryBuckets = new HashMap<>(); + private final Map entryBudgets = new HashMap<>(); private long globalUsedWeight; private final AtomicLong globalRejectedCount = new AtomicLong(); @@ -114,8 +132,11 @@ public EntryBudget createEntryBudget(long catalogId, String engine, String entry throw new IllegalStateException("Duplicated external meta cache budget: " + scope); } Bucket entryBucket = new Bucket(effectiveMax.getAsLong()); + EntryBudget entryBudget = new EntryBudget( + this, scope, catalogBucket, entryBucket, effectiveMax.getAsLong()); entryBuckets.put(scope, entryBucket); - return new EntryBudget(this, scope, catalogBucket, entryBucket, effectiveMax.getAsLong()); + entryBudgets.put(scope, entryBudget); + return entryBudget; } } @@ -208,6 +229,11 @@ private void release(AdmissionReservation reservation) { if (!reservation.active) { return; } + if (reservation.entryBudget.closed) { + reservation.bytes = 0L; + reservation.active = false; + return; + } subtractUsed(reservation.entryBudget, reservation.bytes); reservation.bytes = 0L; reservation.active = false; @@ -220,10 +246,29 @@ private void close(EntryBudget entryBudget) { return; } if (entryBudget.entryBucket.usedWeight != 0L) { - throw new IllegalStateException("entry budget closed with active reservations: " + entryBudget.scope); + long leakedWeight = entryBudget.entryBucket.usedWeight; + LOG.error("Force-closing external metadata cache budget {} with {} bytes still reserved", + entryBudget.scope, leakedWeight); + if (leakedWeight <= globalUsedWeight + && leakedWeight <= entryBudget.catalogBucket.usedWeight) { + globalUsedWeight -= leakedWeight; + entryBudget.catalogBucket.usedWeight -= leakedWeight; + entryBudget.entryBucket.usedWeight = 0L; + } else { + LOG.error("External metadata cache accounting is inconsistent while closing {}; " + + "globalUsed={}, catalogUsed={}, entryUsed={}", + entryBudget.scope, globalUsedWeight, + entryBudget.catalogBucket.usedWeight, leakedWeight); + globalUsedWeight = Math.max(0L, globalUsedWeight - leakedWeight); + entryBudget.catalogBucket.usedWeight = Math.max( + 0L, entryBudget.catalogBucket.usedWeight - leakedWeight); + entryBudget.entryBucket.usedWeight = 0L; + } } entryBudget.closed = true; + entryBudget.reclaimer = null; entryBuckets.remove(entryBudget.scope, entryBudget.entryBucket); + entryBudgets.remove(entryBudget.scope, entryBudget); Bucket catalogBucket = entryBudget.catalogBucket; boolean catalogStillReferenced = entryBuckets.keySet().stream() .anyMatch(scope -> scope.catalogId == entryBudget.scope.catalogId); @@ -250,6 +295,123 @@ private void subtractUsed(EntryBudget entryBudget, long bytes) { entryBudget.entryBucket.usedWeight -= bytes; } + private void requestPeerReclaim(EntryBudget requester, long additionalBytes) { + if (additionalBytes <= 0L || requester.closed) { + return; + } + long reclaimBytes; + synchronized (lock) { + if (requester.closed) { + return; + } + long globalDeficit = deficit(limitOf(globalMaxWeight), globalUsedWeight, additionalBytes); + long catalogDeficit = deficit( + requester.catalogBucket.maxWeight, requester.catalogBucket.usedWeight, additionalBytes); + reclaimBytes = Math.max(globalDeficit, catalogDeficit); + } + if (reclaimBytes <= 0L) { + return; + } + // Rejected values are returned uncached; there is no queue of pending admissions to fund. + // Coalesce concurrent misses to the largest single admission instead of summing identical + // deficits and evicting an entire peer cache during a miss burst. + requester.requestedAdmissionBytes.accumulateAndGet(additionalBytes, Math::max); + schedulePeerReclaim(requester); + } + + private void schedulePeerReclaim(EntryBudget requester) { + if (!requester.reclaimScheduled.compareAndSet(false, true)) { + return; + } + try { + PEER_RECLAIM_EXECUTOR.execute(() -> drainPeerReclaim(requester)); + } catch (RejectedExecutionException e) { + requester.reclaimScheduled.set(false); + LOG.warn("Failed to schedule peer reclamation for external metadata cache budget {}", + requester.scope, e); + } + } + + private void drainPeerReclaim(EntryBudget requester) { + try { + long requestedAdmissionBytes = requester.requestedAdmissionBytes.getAndSet(0L); + if (requestedAdmissionBytes <= 0L || requester.closed) { + return; + } + List candidates; + synchronized (lock) { + candidates = entryBudgets.values().stream() + .filter(candidate -> candidate != requester && !candidate.closed) + .filter(candidate -> candidate.reclaimer != null) + .filter(candidate -> candidate.entryBucket.usedWeight > 0L) + .sorted((left, right) -> { + boolean leftSibling = left.scope.catalogId == requester.scope.catalogId; + boolean rightSibling = right.scope.catalogId == requester.scope.catalogId; + if (leftSibling != rightSibling) { + return leftSibling ? -1 : 1; + } + return Long.compare( + right.entryBucket.usedWeight, left.entryBucket.usedWeight); + }) + .collect(Collectors.toList()); + } + long remaining = currentReclaimDeficit(requester, requestedAdmissionBytes); + for (EntryBudget candidate : candidates) { + boolean sibling = candidate.scope.catalogId == requester.scope.catalogId; + if (!sibling && currentCatalogDeficit(requester, requestedAdmissionBytes) > 0L) { + // Another catalog cannot create headroom under the requester's catalog limit. + continue; + } + LongUnaryOperator reclaimer = candidate.reclaimer; + if (reclaimer == null || candidate.closed) { + continue; + } + try { + reclaimer.applyAsLong(remaining); + remaining = currentReclaimDeficit(requester, requestedAdmissionBytes); + } catch (RuntimeException e) { + LOG.warn("Failed to reclaim external metadata cache budget from peer {}", + candidate.scope, e); + } + if (remaining == 0L) { + break; + } + } + } finally { + requester.reclaimScheduled.set(false); + if (!requester.closed && requester.requestedAdmissionBytes.get() > 0L) { + schedulePeerReclaim(requester); + } + } + } + + private long currentReclaimDeficit(EntryBudget requester, long additionalBytes) { + synchronized (lock) { + if (requester.closed) { + return 0L; + } + long globalDeficit = deficit(limitOf(globalMaxWeight), globalUsedWeight, additionalBytes); + long catalogDeficit = deficit( + requester.catalogBucket.maxWeight, requester.catalogBucket.usedWeight, additionalBytes); + return Math.max(globalDeficit, catalogDeficit); + } + } + + private long currentCatalogDeficit(EntryBudget requester, long additionalBytes) { + synchronized (lock) { + return requester.closed ? 0L : deficit( + requester.catalogBucket.maxWeight, + requester.catalogBucket.usedWeight, additionalBytes); + } + } + + private static long deficit(long maxWeight, long usedWeight, long additionalBytes) { + if (maxWeight == Long.MAX_VALUE || additionalBytes <= maxWeight - Math.min(usedWeight, maxWeight)) { + return 0L; + } + return MetaCacheWeightUtils.saturatedAdd(usedWeight, additionalBytes) - maxWeight; + } + private static boolean fits(long maxWeight, long usedWeight, long delta) { return delta >= 0 && usedWeight <= maxWeight && delta <= maxWeight - usedWeight; } @@ -326,8 +488,11 @@ public static final class EntryBudget { private final Bucket entryBucket; private final long effectiveMaxWeight; private final AtomicLong rejectedCount = new AtomicLong(); - // Guarded by manager.lock. A closed handle must never re-enter accounting. - private boolean closed; + private final AtomicLong requestedAdmissionBytes = new AtomicLong(); + private final AtomicBoolean reclaimScheduled = new AtomicBoolean(); + private volatile LongUnaryOperator reclaimer; + // Mutated under manager.lock and read by asynchronous reclamation workers. + private volatile boolean closed; private EntryBudget(ExternalMetaCacheBudgetManager manager, EntryScope scope, Bucket catalogBucket, Bucket entryBucket, long effectiveMaxWeight) { @@ -342,6 +507,14 @@ public Optional tryReserve(long bytes) { return manager.tryReserve(this, bytes); } + void setReclaimer(LongUnaryOperator reclaimer) { + this.reclaimer = Objects.requireNonNull(reclaimer, "reclaimer"); + } + + void requestPeerReclaim(long additionalBytes) { + manager.requestPeerReclaim(this, additionalBytes); + } + public long getEffectiveMaxWeight() { return effectiveMaxWeight; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java index 01998d16f2d3ca..de89c7b9d8d6aa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java @@ -60,6 +60,7 @@ public class MetaCacheEntry { private static final int LOAD_LOCK_STRIPES = 128; private static final int LOCAL_EVICTION_BATCH_SIZE = 16; private static final int REMOVAL_CLEANUP_BATCH_SIZE = 256; + private static final long WEIGHT_REJECT_LOG_INTERVAL_MS = TimeUnit.MINUTES.toMillis(1L); // Direct Caffeine callbacks must not wait for admissionLock. A daemon drains one coalesced // generation map per physical entry after callbacks return; cleanup tasks never capture values. private static final ExecutorService REMOVAL_CLEANUP_EXECUTOR = Executors.newSingleThreadExecutor(runnable -> { @@ -83,9 +84,12 @@ public class MetaCacheEntry { private final MetaCacheSizeEstimator sizeEstimator; @Nullable private final EntryBudget entryBudget; + @Nullable + private final MetaCacheEntryReplacementListener replacementListener; private final boolean weightBounded; - // Estimator-backed entries use the same generation-fenced refresh protocol even before a - // max-weight is configured. This keeps event ordering stable when weight governance is toggled. + // Entries with publication-time work use the same generation-fenced refresh protocol even + // before a max-weight is configured. This keeps estimation and dependency retirement on every + // load/refresh path instead of letting Caffeine publish values behind those hooks. private final boolean generationFencedRefresh; // Keep the loading cache for refreshAfterWrite and the legacy sync-load path when the feature is disabled. private final LoadingCache loadingData; @@ -122,6 +126,7 @@ public class MetaCacheEntry { private final AtomicLong localEvictionCount = new AtomicLong(0L); private final AtomicLong localEvictionWeight = new AtomicLong(0L); private final AtomicReference lastWeightRejectReason = new AtomicReference<>(""); + private final AtomicLong lastWeightRejectLogTimeMs = new AtomicLong(0L); private final AtomicBoolean closed = new AtomicBoolean(false); public MetaCacheEntry(String name, Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor) { @@ -141,6 +146,14 @@ public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec ca public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, @Nullable MetaCacheSizeEstimator sizeEstimator, @Nullable EntryBudget entryBudget) { + this(name, loader, cacheSpec, refreshExecutor, autoRefresh, contextualOnly, + sizeEstimator, entryBudget, null); + } + + public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, + ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, + @Nullable MetaCacheSizeEstimator sizeEstimator, @Nullable EntryBudget entryBudget, + @Nullable MetaCacheEntryReplacementListener replacementListener) { this.name = name; if (contextualOnly) { if (loader != null) { @@ -158,11 +171,16 @@ public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec ca this.refreshExecutor = Objects.requireNonNull(refreshExecutor, "refreshExecutor can not be null"); this.sizeEstimator = sizeEstimator; this.entryBudget = entryBudget; + this.replacementListener = replacementListener; this.weightBounded = this.cacheSpec.isWeightBounded(); - this.generationFencedRefresh = autoRefresh && sizeEstimator != null; + this.generationFencedRefresh = autoRefresh + && (sizeEstimator != null || replacementListener != null); if (weightBounded && (sizeEstimator == null || entryBudget == null)) { throw new IllegalArgumentException("weighted cache entry requires both estimator and budget: " + name); } + if (weightBounded) { + entryBudget.setReclaimer(this::reclaimForPeer); + } this.effectiveEnabled = this.cacheSpec.isCacheEnabled(); OptionalLong expireAfterAccessSec = effectiveEnabled ? CacheSpec.toExpireAfterAccess(this.cacheSpec.getTtlSecond()) : OptionalLong.empty(); @@ -245,9 +263,9 @@ public V peekIfPresent(K key) { /** * Fence loads and refreshes that started before an event, but only while the expected value is - * still current. Estimator-backed entries retain the known-good value and advance the key's - * mutation epoch. Other count-based entries must invalidate because their legacy Caffeine-managed - * refresh path does not participate in that generation protocol. + * still current. Publication-managed entries retain the known-good value and advance the key's + * mutation epoch. Other count-based entries must invalidate because their legacy + * Caffeine-managed refresh path does not participate in that generation protocol. */ public boolean fenceInFlightLoadIfSame(K key, V expectedCurrent) { Objects.requireNonNull(expectedCurrent, "expectedCurrent can not be null"); @@ -328,6 +346,9 @@ public ReplaceResult tryReplace(K key, V expectedCurrent, V newValue) { && data.asMap().get(key) == newValue) { record.published = true; } + if (result.get() == ReplaceResult.REPLACED && data.asMap().get(key) == newValue) { + notifyReplacement(key, expectedCurrent, newValue); + } return result.get(); } } @@ -478,7 +499,8 @@ public MetaCacheEntryStats stats() { failureCount, totalLoadTime, totalLoadCount == 0 ? 0D : (double) totalLoadTime / totalLoadCount, - saturatedAdd(cacheStats.evictionCount(), localEvictionCount.get()), + MetaCacheWeightUtils.saturatedAdd( + cacheStats.evictionCount(), localEvictionCount.get()), invalidateCount.get(), lastLoadSuccessTimeMs.get(), lastLoadFailureTimeMs.get(), @@ -486,7 +508,8 @@ public MetaCacheEntryStats stats() { weightBounded, weightBounded ? cacheSpec.getMaxWeight().getAsLong() : -1L, weightBounded ? entryBudget.getUsedWeight() : -1L, - weightBounded ? saturatedAdd(cacheStats.evictionWeight(), localEvictionWeight.get()) : -1L, + weightBounded ? MetaCacheWeightUtils.saturatedAdd( + cacheStats.evictionWeight(), localEvictionWeight.get()) : -1L, weightBounded ? weightAdmissionRejectedCount.get() : -1L, weightBounded ? entryBudget.getCatalogMaxWeight() : -1L, weightBounded ? entryBudget.getCatalogUsedWeight() : -1L, @@ -573,6 +596,7 @@ record = null; data.put(key, value); if (reservations.get(key) == newRecord && data.asMap().get(key) == value) { newRecord.published = true; + notifyReplacement(key, null, value); } return AdmissionResult.ADMITTED; } catch (RuntimeException | Error e) { @@ -604,6 +628,9 @@ record = null; if (retained && reservedWeight != newWeight && !newRecord.reservation.tryResize(newWeight)) { throw new IllegalStateException("failed to release cache replacement reservation delta"); } + if (retained) { + notifyReplacement(key, oldValue, value); + } return AdmissionResult.ADMITTED; } catch (RuntimeException | Error e) { if (reservations.replace(key, newRecord, previousRecord)) { @@ -627,6 +654,7 @@ private Optional reserveWithLocalEviction(K incomingKey, l while (!reservation.isPresent()) { int evicted = evictLocalColdest(incomingKey, LOCAL_EVICTION_BATCH_SIZE); if (evicted == 0) { + entryBudget.requestPeerReclaim(bytes); break; } reservation = entryBudget.tryReserve(bytes); @@ -644,6 +672,7 @@ private boolean resizeWithLocalEviction(K incomingKey, AdmissionReservation rese while (true) { int evicted = evictLocalColdest(incomingKey, LOCAL_EVICTION_BATCH_SIZE); if (evicted == 0) { + entryBudget.requestPeerReclaim(Math.max(0L, newBytes - reservation.getBytes())); return false; } if (reservation.tryResize(newBytes)) { @@ -677,6 +706,21 @@ private int evictLocalColdest(K incomingKey, int limit) { return evicted; } + private long reclaimForPeer(long targetBytes) { + if (targetBytes <= 0L || closed.get()) { + return 0L; + } + synchronized (admissionLock) { + long before = entryBudget.getUsedWeight(); + long reclaimed = 0L; + while (reclaimed < targetBytes + && evictLocalColdest(null, LOCAL_EVICTION_BATCH_SIZE) > 0) { + reclaimed = Math.max(0L, before - entryBudget.getUsedWeight()); + } + return reclaimed; + } + } + private int weigh(K key, V value) { ReservationRecord record = reservations.get(key); // Every supported write path installs the reservation record before calling data.put. @@ -829,6 +873,7 @@ private long currentOwnerGeneration(K key) { } private void putNonWeightedValue(K key, V value) { + V previousValue = data.asMap().get(key); RefreshRecord previous = refreshRecords.get(key); RefreshRecord next = publishRefreshRecord(key); try { @@ -837,6 +882,9 @@ private void putNonWeightedValue(K key, V value) { if (next != null && refreshRecords.get(key) == next && data.asMap().get(key) == value) { next.published = true; } + if (data.asMap().get(key) == value) { + notifyReplacement(key, previousValue, value); + } } catch (RuntimeException | Error e) { if (next != null) { if (previous == null) { @@ -859,13 +907,35 @@ private RefreshRecord publishRefreshRecord(K key) { return record; } + private void notifyReplacement(K key, @Nullable V previousValue, V currentValue) { + if (replacementListener == null || previousValue == currentValue) { + return; + } + try { + replacementListener.onReplacement(key, previousValue, currentValue); + } catch (RuntimeException e) { + LOG.warn("Failed to retire dependencies after replacing external metadata cache entry {}", name, e); + } + } + private long nextReservationGeneration() { return reservationGeneration.incrementAndGet(); } private void rejectWeight(String reason) { weightAdmissionRejectedCount.incrementAndGet(); - lastWeightRejectReason.set(reason == null || reason.isEmpty() ? "unknown" : reason); + String normalizedReason = reason == null || reason.isEmpty() ? "unknown" : reason; + lastWeightRejectReason.set(normalizedReason); + long now = System.currentTimeMillis(); + long previous = lastWeightRejectLogTimeMs.get(); + if (now - previous >= WEIGHT_REJECT_LOG_INTERVAL_MS + && lastWeightRejectLogTimeMs.compareAndSet(previous, now)) { + LOG.warn("Rejected external metadata cache admission for entry {}: reason={}, entryUsed={}, " + + "entryMax={}, catalogUsed={}, catalogMax={}, globalUsed={}, globalMax={}", + name, normalizedReason, entryBudget.getUsedWeight(), entryBudget.getEffectiveMaxWeight(), + entryBudget.getCatalogUsedWeight(), entryBudget.getCatalogMaxWeight(), + entryBudget.getGlobalUsedWeight(), entryBudget.getGlobalMaxWeight()); + } } private void maybeRefreshManagedValue(K key, V currentValue) { @@ -916,6 +986,9 @@ private void submitNonWeightedRefresh( advanceKeyMutation(key); putNonWeightedValue(key, refreshed); } + } catch (RuntimeException e) { + LOG.warn("Failed to refresh external metadata cache entry {} for key {}; " + + "retaining the previous value", name, key, e); } finally { endKeyMutation(key, expectedMutation); refreshesInFlight.remove(key); @@ -947,13 +1020,16 @@ private void submitWeightedRefresh( } V refreshed = loadAndTrack(key, this::applyDefaultLoader); if (refreshed != null && isKeyMutationCurrent(key, expectedMutation)) { - AdmissionResult result = admitWeightedValue( + admitWeightedValue( key, refreshed, null, false, expectedMutation, expectedReservationGeneration, true); - if (result == AdmissionResult.REJECTED) { - invalidateKeyIfReservationGeneration(key, expectedReservationGeneration); - } + // Admission rejection leaves the already reserved, known-good generation + // in place. A larger refresh must not turn a transient quota shortage into + // a forced cache miss for every subsequent reader. } + } catch (RuntimeException e) { + LOG.warn("Failed to refresh external metadata cache entry {} for key {}; " + + "retaining the previous value", name, key, e); } finally { endKeyMutation(key, expectedMutation); refreshesInFlight.remove(key); @@ -975,21 +1051,6 @@ private boolean isReservationCurrent( && record.published && data.asMap().get(key) != null; } - private void invalidateKeyIfReservationGeneration(K key, long expectedReservationGeneration) { - synchronized (admissionLock) { - ReservationRecord record = reservations.get(key); - if (record == null || record.generation != expectedReservationGeneration) { - return; - } - V current = data.asMap().get(key); - if (current != null && data.asMap().remove(key, current)) { - advanceKeyMutation(key); - releaseReservation(key, expectedReservationGeneration); - invalidateCount.incrementAndGet(); - } - } - } - // Read the config dynamically so existing cache entries follow runtime config updates. private boolean isManualMissLoadEnabled() { return weightBounded || generationFencedRefresh || Config.enable_external_meta_cache_manual_miss_load; @@ -1244,10 +1305,6 @@ private static ReplaceResult toReplaceResult(AdmissionResult result) { } } - private static long saturatedAdd(long left, long right) { - return left > Long.MAX_VALUE - right ? Long.MAX_VALUE : left + right; - } - private enum AdmissionResult { ADMITTED, NOT_CURRENT, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java index 5963f3d8aa7b86..689d3b6dc7ca99 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java @@ -103,10 +103,13 @@ public final class MetaCacheEntryDef { private final MetaCacheEntryInvalidation invalidation; @Nullable private final MetaCacheSizeEstimator sizeEstimator; + @Nullable + private final MetaCacheEntryReplacementListener replacementListener; private MetaCacheEntryDef(String name, Class keyType, Class valueType, @Nullable Function loader, CacheSpec defaultCacheSpec, boolean autoRefresh, boolean contextualOnly, - MetaCacheEntryInvalidation invalidation, @Nullable MetaCacheSizeEstimator sizeEstimator) { + MetaCacheEntryInvalidation invalidation, @Nullable MetaCacheSizeEstimator sizeEstimator, + @Nullable MetaCacheEntryReplacementListener replacementListener) { this.name = Objects.requireNonNull(name, "entry name is required"); this.keyType = Objects.requireNonNull(keyType, "entry key type is required"); this.valueType = Objects.requireNonNull(valueType, "entry value type is required"); @@ -126,6 +129,7 @@ private MetaCacheEntryDef(String name, Class keyType, Class valueType, this.contextualOnly = contextualOnly; this.invalidation = Objects.requireNonNull(invalidation, "entry invalidation is required"); this.sizeEstimator = sizeEstimator; + this.replacementListener = replacementListener; } /** @@ -145,7 +149,7 @@ public static MetaCacheEntryDef of(String name, Class keyType, C public static MetaCacheEntryDef of(String name, Class keyType, Class valueType, Function loader, CacheSpec defaultCacheSpec, MetaCacheEntryInvalidation invalidation) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, true, false, - invalidation, null); + invalidation, null, null); } /** @@ -167,7 +171,7 @@ public static MetaCacheEntryDef of(String name, Class keyType, C Function loader, CacheSpec defaultCacheSpec, boolean autoRefresh, MetaCacheEntryInvalidation invalidation) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, false, - invalidation, null); + invalidation, null, null); } /** @@ -182,14 +186,22 @@ public static MetaCacheEntryDef contextualOnly( String name, Class keyType, Class valueType, CacheSpec defaultCacheSpec, MetaCacheEntryInvalidation invalidation) { return new MetaCacheEntryDef<>(name, keyType, valueType, null, defaultCacheSpec, false, true, - invalidation, null); + invalidation, null, null); } /** Return a definition with a publication-time size estimator. */ public MetaCacheEntryDef withSizeEstimator(MetaCacheSizeEstimator estimator) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, contextualOnly, invalidation, - Objects.requireNonNull(estimator, "estimator")); + Objects.requireNonNull(estimator, "estimator"), replacementListener); + } + + /** Return a definition that synchronously retires dependencies after a value replacement. */ + public MetaCacheEntryDef withReplacementListener( + MetaCacheEntryReplacementListener listener) { + return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, + autoRefresh, contextualOnly, invalidation, sizeEstimator, + Objects.requireNonNull(listener, "listener")); } /** @@ -247,4 +259,9 @@ public MetaCacheEntryInvalidation getInvalidation() { public MetaCacheSizeEstimator getSizeEstimator() { return sizeEstimator; } + + @Nullable + public MetaCacheEntryReplacementListener getReplacementListener() { + return replacementListener; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryReplacementListener.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryReplacementListener.java new file mode 100644 index 00000000000000..1bfb0cf3990962 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryReplacementListener.java @@ -0,0 +1,26 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import javax.annotation.Nullable; + +/** Receives a successfully published value while the entry mutation is still serialized. */ +@FunctionalInterface +public interface MetaCacheEntryReplacementListener { + void onReplacement(K key, @Nullable V previousValue, V currentValue); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java index f78fd5bc734b39..5aff6a2a1a279a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java @@ -23,10 +23,11 @@ /** * Supplies the admission weight of one key/value pair. * - *

    The callback runs after load and before admission. Implementations must use already available - * shape counters and constant-time collection sizes; they must not walk object graphs, perform IO, - * materialize lazy SDK state, or copy payloads. Caffeine's weigher reads only the admitted - * reservation record, so cache hits and eviction remain O(1). + *

    The callback runs once after load and before admission. Implementations may linearly count + * loader-owned collections needed to cover skewed payloads, but must not recursively reflect over + * arbitrary object graphs, perform additional IO, materialize lazy SDK state, or copy payloads + * solely to estimate weight. Caffeine's weigher reads only the admitted reservation record, so + * cache hits and eviction remain O(1). */ @FunctionalInterface public interface MetaCacheSizeEstimator { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java index b6f36b803b24cc..d3088942ace261 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java @@ -44,7 +44,8 @@ public final class PaimonLatestSnapshotProjectionLoader { @FunctionalInterface public interface SchemaValueLoader { - PaimonSchemaCacheValue load(NameMapping nameMapping, long schemaId); + PaimonSchemaCacheValue load( + NameMapping nameMapping, long schemaId, long tableGeneration, Table retainedTable); } private final PaimonPartitionInfoLoader partitionInfoLoader; @@ -59,7 +60,8 @@ public PaimonLatestSnapshotProjectionLoader(PaimonPartitionInfoLoader partitionI public PaimonSnapshotCacheValue load(NameMapping nameMapping, Table paimonTable) { try { PaimonSnapshot latestSnapshot = resolveLatestSnapshot(paimonTable, true); - List partitionColumns = schemaValueLoader.load(nameMapping, latestSnapshot.getSchemaId()) + List partitionColumns = schemaValueLoader.load( + nameMapping, latestSnapshot.getSchemaId(), 0L, latestSnapshot.getTable()) .getPartitionColumns(); PaimonPartitionInfo partitionInfo = partitionInfoLoader.load(nameMapping, latestSnapshot.getTable(), partitionColumns); @@ -85,11 +87,21 @@ public PaimonSnapshotCacheValue loadFence(NameMapping nameMapping, Table paimonT } public PaimonSnapshotCacheValue loadAtFence(NameMapping nameMapping, PaimonSnapshot fence) { - return loadEffectiveAtFence(nameMapping, fence.getTable(), fence); + return loadAtFence(nameMapping, fence, 0L); + } + + public PaimonSnapshotCacheValue loadAtFence( + NameMapping nameMapping, PaimonSnapshot fence, long tableGeneration) { + return loadEffectiveAtFence(nameMapping, fence.getTable(), fence, tableGeneration); } public PaimonSnapshotCacheValue loadEffectiveAtFence( NameMapping nameMapping, Table effectiveTable, PaimonSnapshot fence) { + return loadEffectiveAtFence(nameMapping, effectiveTable, fence, 0L); + } + + public PaimonSnapshotCacheValue loadEffectiveAtFence( + NameMapping nameMapping, Table effectiveTable, PaimonSnapshot fence, long tableGeneration) { try { // The fence owns both version and table generation. Reopening the catalog here can pair // the old snapshot id with a newer schema or branch after invalidation. @@ -102,12 +114,14 @@ public PaimonSnapshotCacheValue loadEffectiveAtFence( latestSchemaTable.copyWithoutTimeTravel( PaimonScanParams.isolateSnapshotRead(fence.getSnapshotId()))); } - List partitionColumns = schemaValueLoader.load(nameMapping, fence.getSchemaId()) + List partitionColumns = schemaValueLoader.load( + nameMapping, fence.getSchemaId(), tableGeneration, effectiveTable) .getPartitionColumns(); PaimonPartitionInfo partitionInfo = partitionInfoLoader.load(nameMapping, snapshotTable, partitionColumns); return new PaimonSnapshotCacheValue(partitionInfo, - new PaimonSnapshot(fence.getSnapshotId(), fence.getSchemaId(), snapshotTable)); + new PaimonSnapshot(fence.getSnapshotId(), fence.getSchemaId(), snapshotTable), + false, tableGeneration); } catch (Exception e) { throw new CacheException("failed to load paimon snapshot at fence %s.%s.%s: %s", e, nameMapping.getCtlId(), nameMapping.getLocalDbName(), nameMapping.getLocalTblName(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java index 0a134cfd7d7d32..fc9fbeaa755752 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonTableLoader.java @@ -25,6 +25,7 @@ import org.apache.paimon.table.Table; import java.io.IOException; +import java.util.concurrent.Callable; /** * Loads the base Paimon table handle used by cache entries and runtime projections. @@ -45,4 +46,14 @@ public PaimonExternalCatalog catalog(NameMapping nameMapping) throws IOException return (PaimonExternalCatalog) Env.getCurrentEnv().getCatalogMgr() .getCatalogOrException(nameMapping.getCtlId(), id -> new IOException("Catalog not found: " + id)); } + + public T executeAuthenticated(NameMapping nameMapping, Callable task) { + try { + return catalog(nameMapping).getExecutionAuthenticator().execute(task); + } catch (Exception e) { + throw new CacheException("failed to load authenticated paimon metadata %s.%s.%s: %s", + e, nameMapping.getCtlId(), nameMapping.getLocalDbName(), nameMapping.getLocalTblName(), + e.getMessage()); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index ee5fdeebb2f793..a308896bb299e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -35,6 +35,7 @@ import java.util.Map; import java.util.concurrent.ExecutorService; +import javax.annotation.Nullable; /** * Paimon engine implementation of {@link AbstractExternalMetaCache}. @@ -79,7 +80,8 @@ public PaimonExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCach new PaimonPartitionInfoLoader(), this::getPaimonSchemaCacheValue); tableEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_TABLE, NameMapping.class, PaimonTableCacheValue.class, this::loadTableCacheValue, defaultEntryCacheSpec(), - MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping))); + MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping)) + .withReplacementListener(this::retireTableGeneration)); snapshotEntry = registerEntry(MetaCacheEntryDef.contextualOnly(ENTRY_SNAPSHOT, PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(PaimonSnapshotEntryKey::getNameMapping)) @@ -101,45 +103,75 @@ public Table getPaimonTable(NameMapping nameMapping) { public PaimonSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); PaimonTableCacheValue tableValue = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); - PaimonSnapshot fence = tableValue.getLatestSnapshotFence().getSnapshot(); + PaimonSnapshot fence = loadLatestSnapshotFence(nameMapping, tableValue.getPaimonTable()).getSnapshot(); PaimonSnapshotEntryKey key = PaimonSnapshotEntryKey.of( nameMapping, fence, tableValue.getGeneration()); MetaCacheEntry entry = snapshotEntry.get(nameMapping.getCtlId()); - return entry.get(key, ignored -> latestSnapshotProjectionLoader.loadAtFence(nameMapping, fence)); + PaimonSnapshotCacheValue snapshotValue = entry.get(key, + ignored -> executeAuthenticated(nameMapping, + () -> latestSnapshotProjectionLoader.loadAtFence( + nameMapping, fence, tableValue.getGeneration()))); + PaimonTableCacheValue currentTable = tableEntry.get(nameMapping.getCtlId()).peekIfPresent(nameMapping); + if (currentTable != null && currentTable.getGeneration() != tableValue.getGeneration()) { + entry.invalidateKeyIfSame(key, snapshotValue); + } + return snapshotValue; } public PaimonSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTable, Table effectiveTable) { - return latestSnapshotProjectionLoader.load(dorisTable.getOrBuildNameMapping(), effectiveTable); + NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); + return executeAuthenticated(nameMapping, + () -> latestSnapshotProjectionLoader.load(nameMapping, effectiveTable)); } public PaimonSnapshotCacheValue loadLatestSnapshotFence(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - return tableEntry.get(nameMapping.getCtlId()).get(nameMapping).getLatestSnapshotFence(); + PaimonTableCacheValue tableValue = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + return loadLatestSnapshotFence(nameMapping, tableValue.getPaimonTable()); } public PaimonSnapshotCacheValue loadSnapshotAtFence( ExternalTable dorisTable, PaimonSnapshot fence) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - return latestSnapshotProjectionLoader.loadAtFence(nameMapping, fence); + return executeAuthenticated(nameMapping, + () -> latestSnapshotProjectionLoader.loadAtFence(nameMapping, fence)); } public PaimonSnapshotCacheValue loadSnapshotAtFence( ExternalTable dorisTable, Table effectiveTable, PaimonSnapshot fence) { - return latestSnapshotProjectionLoader.loadEffectiveAtFence( - dorisTable.getOrBuildNameMapping(), effectiveTable, fence); + NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); + return executeAuthenticated(nameMapping, + () -> latestSnapshotProjectionLoader.loadEffectiveAtFence( + nameMapping, effectiveTable, fence)); } public PaimonSchemaCacheValue getPaimonSchemaCacheValue(NameMapping nameMapping, long schemaId) { - SchemaCacheValue schemaCacheValue = schemaEntry.get(nameMapping.getCtlId()) - .get(new PaimonSchemaCacheKey(nameMapping, schemaId)); + PaimonTableCacheValue tableValue = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + return getPaimonSchemaCacheValue( + nameMapping, schemaId, tableValue.getGeneration(), tableValue.getPaimonTable()); + } + + PaimonSchemaCacheValue getPaimonSchemaCacheValue( + NameMapping nameMapping, long schemaId, long tableGeneration, Table retainedTable) { + PaimonSchemaCacheKey key = new PaimonSchemaCacheKey(nameMapping, tableGeneration, schemaId); + if (tableGeneration <= 0L) { + return (PaimonSchemaCacheValue) executeAuthenticated(nameMapping, + () -> loadSchemaCacheValue(key, retainedTable)); + } + MetaCacheEntry entry = schemaEntry.get(nameMapping.getCtlId()); + SchemaCacheValue schemaCacheValue = entry.get(key, + ignored -> executeAuthenticated(nameMapping, + () -> loadSchemaCacheValue(key, retainedTable))); + PaimonTableCacheValue currentTable = tableEntry.get(nameMapping.getCtlId()).peekIfPresent(nameMapping); + if (currentTable != null && currentTable.getGeneration() != tableGeneration) { + entry.invalidateKeyIfSame(key, schemaCacheValue); + } return (PaimonSchemaCacheValue) schemaCacheValue; } private PaimonTableCacheValue loadTableCacheValue(NameMapping nameMapping) { - Table paimonTable = tableLoader.load(nameMapping); - PaimonSnapshotCacheValue fence = latestSnapshotProjectionLoader.loadFence(nameMapping, paimonTable); - return new PaimonTableCacheValue(paimonTable, fence); + return new PaimonTableCacheValue(tableLoader.load(nameMapping)); } private SchemaCacheValue loadSchemaCacheValue(PaimonSchemaCacheKey key) { @@ -150,6 +182,40 @@ private SchemaCacheValue loadSchemaCacheValue(PaimonSchemaCacheKey key) { key.getNameMapping().getLocalTblName(), key.getSchemaId())); } + private SchemaCacheValue loadSchemaCacheValue(PaimonSchemaCacheKey key, Table retainedTable) { + ExternalTable dorisTable = findExternalTable(key.getNameMapping(), ENGINE); + if (!(dorisTable instanceof PaimonExternalTable)) { + return loadSchemaCacheValue(key); + } + dorisTable.setUpdateTime(System.currentTimeMillis()); + return ((PaimonExternalTable) dorisTable).loadSchemaForCache(retainedTable, key.getSchemaId()); + } + + private PaimonSnapshotCacheValue loadLatestSnapshotFence(NameMapping nameMapping, Table retainedTable) { + return executeAuthenticated(nameMapping, + () -> latestSnapshotProjectionLoader.loadFence(nameMapping, retainedTable)); + } + + private T executeAuthenticated(NameMapping nameMapping, java.util.concurrent.Callable task) { + return tableLoader.executeAuthenticated(nameMapping, task); + } + + private void retireTableGeneration(NameMapping nameMapping, + @Nullable PaimonTableCacheValue previousValue, PaimonTableCacheValue currentValue) { + MetaCacheEntry snapshots = + snapshotEntry.getIfInitialized(nameMapping.getCtlId()); + if (snapshots != null) { + snapshots.invalidateIf(key -> key.getNameMapping().equals(nameMapping) + && key.getTableGeneration() != currentValue.getGeneration()); + } + MetaCacheEntry schemas = + schemaEntry.getIfInitialized(nameMapping.getCtlId()); + if (schemas != null) { + schemas.invalidateIf(key -> key.getNameMapping().equals(nameMapping) + && key.getTableGeneration() != currentValue.getGeneration()); + } + } + @Override protected Map catalogPropertyCompatibilityMap() { Map compatibility = new java.util.HashMap<>( diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index d0a3c858f4b847..defa3dd4d00ec5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -490,6 +490,14 @@ private PaimonSchemaCacheValue loadSchema(DataTable table, long schemaId) { return loadSchema(table.schemaManager().schema(schemaId)); } + PaimonSchemaCacheValue loadSchemaForCache(Table retainedTable, long schemaId) { + if (!(retainedTable instanceof DataTable)) { + throw new CacheException("retained paimon table does not expose schema history: %s", + null, retainedTable == null ? "null" : retainedTable.getClass().getName()); + } + return loadSchema((DataTable) retainedTable, schemaId); + } + private PaimonSchemaCacheValue loadSchema(TableSchema tableSchema) { List columns = tableSchema.fields(); List dorisColumns = Lists.newArrayListWithCapacity(columns.size()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheKey.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheKey.java index 4eccb269c2fe56..49d5847e0e0469 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheKey.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSchemaCacheKey.java @@ -23,13 +23,23 @@ import com.google.common.base.Objects; public class PaimonSchemaCacheKey extends SchemaCacheKey { + private final long tableGeneration; private final long schemaId; public PaimonSchemaCacheKey(NameMapping nameMapping, long schemaId) { + this(nameMapping, 0L, schemaId); + } + + public PaimonSchemaCacheKey(NameMapping nameMapping, long tableGeneration, long schemaId) { super(nameMapping); + this.tableGeneration = tableGeneration; this.schemaId = schemaId; } + public long getTableGeneration() { + return tableGeneration; + } + public long getSchemaId() { return schemaId; } @@ -46,11 +56,11 @@ public boolean equals(Object o) { return false; } PaimonSchemaCacheKey that = (PaimonSchemaCacheKey) o; - return schemaId == that.schemaId; + return tableGeneration == that.tableGeneration && schemaId == that.schemaId; } @Override public int hashCode() { - return Objects.hashCode(super.hashCode(), schemaId); + return Objects.hashCode(super.hashCode(), tableGeneration, schemaId); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java index 630db395afbfa3..e6b37d4c020b72 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java @@ -25,18 +25,25 @@ public class PaimonSnapshotCacheValue { private final PaimonPartitionInfo partitionInfo; private final PaimonSnapshot snapshot; private final boolean schemaFromSnapshotTable; + private final long tableGeneration; private long retainedTablePayloadBytes; private MetaCacheSizeEstimate sizeEstimate; public PaimonSnapshotCacheValue(PaimonPartitionInfo partitionInfo, PaimonSnapshot snapshot) { - this(partitionInfo, snapshot, false); + this(partitionInfo, snapshot, false, 0L); } public PaimonSnapshotCacheValue(PaimonPartitionInfo partitionInfo, PaimonSnapshot snapshot, boolean schemaFromSnapshotTable) { + this(partitionInfo, snapshot, schemaFromSnapshotTable, 0L); + } + + public PaimonSnapshotCacheValue(PaimonPartitionInfo partitionInfo, PaimonSnapshot snapshot, + boolean schemaFromSnapshotTable, long tableGeneration) { this.partitionInfo = partitionInfo; this.snapshot = snapshot; this.schemaFromSnapshotTable = schemaFromSnapshotTable; + this.tableGeneration = tableGeneration; } public PaimonPartitionInfo getPartitionInfo() { @@ -51,15 +58,22 @@ public boolean isSchemaFromSnapshotTable() { return schemaFromSnapshotTable; } + public long getTableGeneration() { + return tableGeneration; + } + long getRetainedTablePayloadBytes() { return retainedTablePayloadBytes; } MetaCacheSizeEstimate prepareForCachePublication(PaimonSnapshotEntryKey key) { if (sizeEstimate == null) { - retainedTablePayloadBytes = PaimonCacheSizeEstimator.retainedTablePayloadBytes(snapshot.getTable()); sizeEstimate = MetaCacheSizeEstimator.estimateSafely("paimon_snapshot_preparation_failed", - () -> PaimonCacheSizeEstimator.estimateSnapshotEntry(key, this)); + () -> { + retainedTablePayloadBytes = + PaimonCacheSizeEstimator.retainedTablePayloadBytes(snapshot.getTable()); + return PaimonCacheSizeEstimator.estimateSnapshotEntry(key, this); + }); } return sizeEstimate; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java index e5fba223ea92b2..9e381602dcbc9e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java @@ -24,22 +24,24 @@ /** * Cache value for a Paimon table handle. Snapshot projections use a separate cache entry so this - * value cannot grow after admission through a memoized supplier. + * value cannot grow after admission. */ public class PaimonTableCacheValue { private static final AtomicLong NEXT_GENERATION = new AtomicLong(); private final Table paimonTable; - private final PaimonSnapshotCacheValue latestSnapshotFence; private final long generation; - public PaimonTableCacheValue(Table paimonTable, PaimonSnapshotCacheValue latestSnapshotFence) { + public PaimonTableCacheValue(Table paimonTable) { this.paimonTable = paimonTable; - this.latestSnapshotFence = Objects.requireNonNull( - latestSnapshotFence, "latestSnapshotFence can not be null"); this.generation = NEXT_GENERATION.incrementAndGet(); } + public PaimonTableCacheValue(Table paimonTable, PaimonSnapshotCacheValue ignoredFence) { + this(paimonTable); + Objects.requireNonNull(ignoredFence, "latestSnapshotFence can not be null"); + } + public Table getPaimonTable() { return paimonTable; } @@ -48,7 +50,4 @@ public long getGeneration() { return generation; } - public PaimonSnapshotCacheValue getLatestSnapshotFence() { - return latestSnapshotFence; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtils.java index efbe24a4910efc..1fdee2fbb0d2f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtils.java @@ -64,6 +64,11 @@ public static PaimonSnapshotCacheValue getSnapshotCacheValue(Optional 0L) { + return paimonExternalMetaCache(dorisTable).getPaimonSchemaCacheValue( + dorisTable.getOrBuildNameMapping(), snapshotValue.getSnapshot().getSchemaId(), + snapshotValue.getTableGeneration(), snapshotValue.getSnapshot().getTable()); + } return getSchemaCacheValue(dorisTable, snapshotValue.getSnapshot().getSchemaId()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java index d01ec4de8863e2..607dde99476a5f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java @@ -33,6 +33,7 @@ import mockit.MockUp; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.util.Collections; import java.util.HashMap; @@ -61,20 +62,22 @@ public void testEngineAliasCompatibility() { public void testCatalogCachePropertiesRejectUnknownEngineEntryAndAliasNamespace() { ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); Map properties = new HashMap<>(); + HMSExternalCatalog catalog = new HMSExternalCatalog( + 1L, "hms", null, Collections.emptyMap(), ""); properties.put("meta.cache.hvie.partition_values.capacity", "10"); Assert.assertThrows(IllegalArgumentException.class, - () -> metaCacheMgr.prepareCatalogByEngine(1L, "hive", properties)); + () -> metaCacheMgr.validateCatalogCacheProperties(catalog, properties)); properties.clear(); properties.put("meta.cache.hms.partition_values.capacity", "10"); Assert.assertThrows(IllegalArgumentException.class, - () -> metaCacheMgr.prepareCatalogByEngine(1L, "hive", properties)); + () -> metaCacheMgr.validateCatalogCacheProperties(catalog, properties)); properties.clear(); properties.put("meta.cache.hive.partiton_values.capacity", "10"); Assert.assertThrows(IllegalArgumentException.class, - () -> metaCacheMgr.prepareCatalogByEngine(1L, "hive", properties)); + () -> metaCacheMgr.validateCatalogCacheProperties(catalog, properties)); } @Test @@ -90,6 +93,20 @@ public void testCatalogCachePropertiesRejectEngineNotRoutedByCatalogType() { Assert.assertTrue(exception.getMessage().contains("not supported by catalog type")); } + @Test + public void testRuntimePreparationIgnoresInvalidPersistedCacheProperties() { + ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); + Map properties = new HashMap<>(); + properties.put("meta.cache.max-weight", "1.5GB"); + properties.put("meta.cache.hive.partition_values.enable", "1"); + properties.put("meta.cache.hive.partiton_values.capacity", "10"); + + metaCacheMgr.prepareCatalogByEngine(101L, "hive", properties); + + Assert.assertFalse(metaCacheMgr.getCatalogCacheStats(101L).isEmpty()); + metaCacheMgr.removeCatalog(101L); + } + @Test public void testRouteByCatalogType() { ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); @@ -203,6 +220,29 @@ public void testCatalogRemovalFencesInFlightFirstInitialization() throws Excepti } } + @Test + public void testRollbackRetiresGroupInitializedFromRejectedCandidate() throws Exception { + RecordingExternalMetaCache hive = new RecordingExternalMetaCache( + "hive", Collections.singletonList("hms"), catalog -> catalog instanceof HMSExternalCatalog); + RecordingExternalMetaCache hudi = new RecordingExternalMetaCache( + "hudi", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + RecordingExternalMetaCache iceberg = new RecordingExternalMetaCache( + "iceberg", Collections.emptyList(), catalog -> catalog instanceof HMSExternalCatalog); + ExternalMetaCacheMgr metaCacheMgr = newManagerWithCaches(hive, hudi, iceberg); + long catalogId = 14L; + HMSExternalCatalog catalog = Mockito.mock(HMSExternalCatalog.class); + Mockito.when(catalog.getId()).thenReturn(catalogId); + mockCurrentCatalog(catalogId, catalog); + hive.initializedCatalogIds.add(catalogId); + Map oldProperties = Collections.singletonMap("generation", "old"); + + metaCacheMgr.rollbackCatalogProperties(catalog, oldProperties); + + Mockito.verify(catalog).rollBackCatalogProps(oldProperties); + Assert.assertFalse(hive.isCatalogInitialized(catalogId)); + Assert.assertEquals(1, hive.invalidateCatalogCalls); + } + @Test public void testGetSchemaCacheValueReturnsEmptyWhenCatalogMissing() throws Exception { MissingCatalogSchemaExternalMetaCache schemaCache = new MissingCatalogSchemaExternalMetaCache("default"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java index cafeaf915ddaff..df24c0c754d6a2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java @@ -26,6 +26,7 @@ import org.apache.doris.common.ThreadPoolManager; import org.apache.doris.common.util.Util; import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.metacache.EstimatorCalibrationAssertions; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; @@ -245,6 +246,27 @@ public void testPartitionValuesEstimateSupportsRealLiteralGraph() throws Excepti "cache publication must not rewrite common catalog partition objects"); } + @Test + public void testPartitionValuesFormulaAgainstJolOwnedGraph() throws Exception { + List types = Collections.singletonList(Type.STRING); + HiveExternalMetaCache.PartitionValueCacheKey key = new HiveExternalMetaCache.PartitionValueCacheKey( + NameMapping.createForTest("db", "tbl"), types); + HiveExternalMetaCache.HivePartitionValues empty = realPartitionValues(types, 0, 16); + HiveExternalMetaCache.HivePartitionValues populated = realPartitionValues(types, 32, 16); + HiveExternalMetaCache.HivePartitionValues shortTail = realPartitionValues(types, 1, 16); + HiveExternalMetaCache.HivePartitionValues longTail = realPartitionValues(types, 1, 4096); + + long emptyEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, empty).getBytes(); + long populatedEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, populated).getBytes(); + long shortTailEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, shortTail).getBytes(); + long longTailEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, longTail).getBytes(); + + EstimatorCalibrationAssertions.assertConservativeDelta( + "hive partition values", emptyEstimate, populatedEstimate, empty, populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "hive long-tail partition", shortTailEstimate, longTailEstimate, shortTail, longTail); + } + private void putCache( MetaCacheEntry fileCache, MetaCacheEntry partitionCache, @@ -291,6 +313,26 @@ private long partitionValueWeight( return estimate.getBytes(); } + private HiveExternalMetaCache.HivePartitionValues realPartitionValues( + List types, int partitionCount, int valueLength) throws Exception { + Map items = new HashMap<>(); + HashBiMap names = HashBiMap.create(); + Map> values = new HashMap<>(); + for (int index = 0; index < partitionCount; index++) { + String value = "p" + index + String.join("", Collections.nCopies(valueLength, "x")); + long id = index + 1L; + PartitionKey partitionKey = PartitionKey.createListPartitionKeyWithTypes( + Collections.singletonList(new PartitionValue(value)), types, true); + items.put(id, new ListPartitionItem(Collections.singletonList(partitionKey))); + names.put("p=" + value, id); + values.put(id, Collections.singletonList(value)); + } + HiveExternalMetaCache.HivePartitionValues result = + new HiveExternalMetaCache.HivePartitionValues(items, names, values); + result.sealForPublication(); + return result; + } + @SuppressWarnings("unchecked") private Map sizeOnlyMap(int size) { Map map = Mockito.mock(Map.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index bc8a7cc7a74ff0..5d98c753b82463 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -24,12 +24,14 @@ import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; +import org.apache.doris.datasource.metacache.EstimatorCalibrationAssertions; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryStats; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.BaseTable; +import org.apache.iceberg.DataFile; import org.apache.iceberg.DataFiles; import org.apache.iceberg.DeleteFile; import org.apache.iceberg.FileMetadata; @@ -48,6 +50,7 @@ import org.apache.iceberg.StaticTableOperations; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.TableOperations; import org.apache.iceberg.encryption.EncryptedKey; import org.apache.iceberg.hadoop.HadoopTables; @@ -159,6 +162,139 @@ public void testSnapshotKeyIncludesMetadataGeneration() { newInterfaceProxy(Table.class)).isPresent()); } + @Test + public void testReplacingTableGenerationRetiresSnapshotAndSchemaProjection() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + try { + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.emptyMap()); + NameMapping mapping = NameMapping.createForTest(catalogId, "db", "tbl"); + IcebergTableCacheValue first = new IcebergTableCacheValue( + tableWithMetadataLocation("/metadata/retire-v1.json")); + IcebergTableCacheValue second = new IcebergTableCacheValue( + tableWithMetadataLocation("/metadata/retire-v2.json")); + MetaCacheEntry tables = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class); + tables.put(mapping, first); + IcebergSnapshotEntryKey oldSnapshotKey = IcebergSnapshotEntryKey.tryCreate( + mapping, first.getRetainedIcebergTable()).get(); + MetaCacheEntry snapshots = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_SNAPSHOT, + IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class); + snapshots.put(oldSnapshotKey, new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L))); + IcebergSchemaCacheKey oldSchemaKey = new IcebergSchemaCacheKey( + mapping, first.getTableUuid().get(), 0L); + MetaCacheEntry schemas = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_SCHEMA, + IcebergSchemaCacheKey.class, SchemaCacheValue.class); + schemas.put(oldSchemaKey, new SchemaCacheValue(Collections.emptyList())); + + // Simulate expiry/invalidation before the next table generation is published. + tables.invalidateKey(mapping); + tables.put(mapping, second); + + Assert.assertNull(snapshots.peekIfPresent(oldSnapshotKey)); + Assert.assertNull(schemas.peekIfPresent(oldSchemaKey)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testOldGenerationSchemaLoadCannotRepopulateAfterTableReplacement() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + try { + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.emptyMap()); + NameMapping mapping = NameMapping.createForTest(catalogId, "db", "tbl"); + IcebergTableCacheValue oldTable = new IcebergTableCacheValue( + tableWithMetadataLocation("/metadata/schema-race-old.json")); + IcebergTableCacheValue newTable = new IcebergTableCacheValue( + tableWithMetadataLocation("/metadata/schema-race-new.json")); + cache.entry(catalogId, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).put(mapping, newTable); + IcebergSchemaCacheKey staleKey = new IcebergSchemaCacheKey( + mapping, oldTable.getTableUuid().get(), 0L); + IcebergSchemaCacheValue staleValue = new IcebergSchemaCacheValue( + Collections.emptyList(), Collections.emptyList()); + MetaCacheEntry schemas = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_SCHEMA, + IcebergSchemaCacheKey.class, SchemaCacheValue.class); + schemas.put(staleKey, staleValue); + + Assert.assertSame(staleValue, cache.getIcebergSchemaCacheValue( + mapping, 0L, oldTable.getRetainedIcebergTable())); + Assert.assertNull(schemas.peekIfPresent(staleKey)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testFrozenGenerationPreservesSparseEquivalentSchemaIds() { + TableMetadata metadata = TableMetadataParser.fromJson("/metadata/sparse.json", "{" + + "\"format-version\":2,\"table-uuid\":\"sparse-schema-table\"," + + "\"location\":\"file:/warehouse/sparse\",\"last-sequence-number\":0," + + "\"last-updated-ms\":1,\"last-column-id\":2,\"current-schema-id\":2," + + "\"schemas\":[" + + "{\"type\":\"struct\",\"schema-id\":0,\"fields\":[" + + "{\"id\":1,\"name\":\"a\",\"required\":false,\"type\":\"int\"}]}," + + "{\"type\":\"struct\",\"schema-id\":1,\"fields\":[" + + "{\"id\":1,\"name\":\"a\",\"required\":false,\"type\":\"int\"}," + + "{\"id\":2,\"name\":\"b\",\"required\":false,\"type\":\"string\"}]}," + + "{\"type\":\"struct\",\"schema-id\":2,\"fields\":[" + + "{\"id\":1,\"name\":\"a\",\"required\":false,\"type\":\"int\"}]}]," + + "\"default-spec-id\":0,\"partition-specs\":[{\"spec-id\":0,\"fields\":[]}]," + + "\"last-partition-id\":999,\"default-sort-order-id\":0," + + "\"sort-orders\":[{\"order-id\":0,\"fields\":[]}],\"properties\":{}," + + "\"current-snapshot-id\":-1,\"refs\":{},\"snapshots\":[]," + + "\"statistics\":[],\"partition-statistics\":[]," + + "\"snapshot-log\":[],\"metadata-log\":[]}"); + Table retained = IcebergSnapshotCacheValue.retainNonGrowingGeneration( + IcebergSnapshotCacheValue.retainTableGeneration( + new BaseTable(new StaticTableOperations(metadata, null), "db.tbl"))); + TableMetadata retainedMetadata = ((HasTableOperations) retained).operations().current(); + + Assert.assertEquals(2, retainedMetadata.currentSchemaId()); + Assert.assertEquals(java.util.Arrays.asList(0, 1, 2), retainedMetadata.schemas().stream() + .map(Schema::schemaId).collect(Collectors.toList())); + Assert.assertEquals(1, retainedMetadata.schemas().stream() + .filter(schema -> schema.schemaId() == 2).findFirst().get().columns().size()); + } + + @Test + public void testFrozenGenerationAcceptsSnapshotCreatedBeforeV3Upgrade() { + TableMetadata metadata = TableMetadataParser.fromJson("/metadata/upgraded-v3.json", "{" + + "\"format-version\":3,\"table-uuid\":\"upgraded-v3-table\"," + + "\"location\":\"file:/warehouse/v3\",\"last-sequence-number\":1," + + "\"last-updated-ms\":2,\"last-column-id\":1,\"current-schema-id\":0," + + "\"schemas\":[{\"type\":\"struct\",\"schema-id\":0,\"fields\":[" + + "{\"id\":1,\"name\":\"id\",\"required\":false,\"type\":\"int\"}]}]," + + "\"default-spec-id\":0,\"partition-specs\":[{\"spec-id\":0,\"fields\":[]}]," + + "\"last-partition-id\":999,\"default-sort-order-id\":0," + + "\"sort-orders\":[{\"order-id\":0,\"fields\":[]}],\"properties\":{}," + + "\"current-snapshot-id\":7,\"next-row-id\":0," + + "\"refs\":{\"main\":{\"snapshot-id\":7,\"type\":\"branch\"}}," + + "\"snapshots\":[{\"sequence-number\":0,\"snapshot-id\":7," + + "\"timestamp-ms\":1,\"summary\":{\"operation\":\"append\"}," + + "\"manifests\":[],\"schema-id\":0}]," + + "\"statistics\":[],\"partition-statistics\":[]," + + "\"snapshot-log\":[{\"timestamp-ms\":1,\"snapshot-id\":7}]," + + "\"metadata-log\":[]}"); + Table retained = IcebergSnapshotCacheValue.retainNonGrowingGeneration( + IcebergSnapshotCacheValue.retainTableGeneration( + new BaseTable(new StaticTableOperations(metadata, null), "db.tbl"))); + + Assert.assertEquals(7L, retained.currentSnapshot().snapshotId()); + Assert.assertEquals(3, ((HasTableOperations) retained).operations().current().formatVersion()); + } + @Test public void testTableSnapshotAndManifestEstimatesArePrecomputed() { NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); @@ -192,6 +328,51 @@ public void testTableSnapshotAndManifestEstimatesArePrecomputed() { Assert.assertTrue(unsupported.getIncompleteReason().startsWith("unsupported_iceberg_table:")); } + @Test + public void testIcebergPreparationFailureIsFailClosed() { + TableMetadata brokenMetadata = Mockito.mock(TableMetadata.class); + Mockito.when(brokenMetadata.currentSnapshot()) + .thenThrow(new IllegalStateException("unsupported snapshot state")); + TableOperations operations = Mockito.mock(TableOperations.class); + Mockito.when(operations.current()).thenReturn(brokenMetadata); + Table brokenTable = new BaseTable(operations, "db.tbl"); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + + IcebergTableCacheValue tableValue = new IcebergTableCacheValue(brokenTable); + MetaCacheSizeEstimate tableEstimate = tableValue.prepareForCachePublication(mapping); + + Assert.assertFalse(tableEstimate.isComplete()); + Assert.assertTrue(tableEstimate.getIncompleteReason() + .startsWith("iceberg_table_preparation_failed:")); + Assert.assertSame(tableValue.getRetainedIcebergTable(), tableValue.newQueryScopedTable()); + + Table healthyTable = tableWithMetadataLocation("/metadata/fail-closed-key.json"); + IcebergSnapshotEntryKey key = IcebergSnapshotEntryKey.tryCreate(mapping, healthyTable).get(); + IcebergSnapshotCacheValue snapshotValue = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), + Optional.empty(), brokenTable); + MetaCacheSizeEstimate snapshotEstimate = snapshotValue.prepareForCachePublication(key); + + Assert.assertFalse(snapshotEstimate.isComplete()); + Assert.assertTrue(snapshotEstimate.getIncompleteReason() + .startsWith("iceberg_snapshot_preparation_failed:")); + } + + @Test + public void testManifestAccountingFailureKeepsFilesAndRejectsWeightedAdmission() { + DataFile file = Mockito.mock(DataFile.class); + Mockito.when(file.columnSizes()).thenThrow(new IllegalStateException("new metrics representation")); + + ManifestCacheValue value = ManifestCacheValue.forDataFiles(Collections.singletonList(file)); + MetaCacheSizeEstimate estimate = IcebergCacheSizeEstimator.estimateManifestEntry( + new IcebergManifestEntryKey("/manifest/fail-closed.avro", ManifestContent.DATA), value); + + Assert.assertEquals(Collections.singletonList(file), value.getDataFiles()); + Assert.assertFalse(value.isAccountingComplete()); + Assert.assertFalse(estimate.isComplete()); + Assert.assertEquals("iceberg_manifest_accounting_incomplete", estimate.getIncompleteReason()); + } + @Test public void testTableEstimateAccountsForNestedSchemaAndPropertyPayload() { String largePayload = repeatedCharacter('x', 64 * 1024); @@ -262,18 +443,18 @@ public void testTablePayloadCountsHistoricalSchemaSpecAndSortFields() { } @Test - public void testTablePayloadExcludesQueryLocalHistoricalMetadata() { + public void testTablePayloadAccountsForRetainedHistoricalMetadata() { String largePayload = repeatedCharacter('x', 64 * 1024); long smallBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes( tableWithMetadata(metadataWithMaterializedPayload("x", 32))); long largeBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes( tableWithMetadata(metadataWithMaterializedPayload(largePayload, 64 * 1024))); - Assert.assertEquals(smallBytes, largeBytes); + Assert.assertTrue(largeBytes - smallBytes >= 64L * 1024L - 32L); } @Test - public void testTableEstimateExcludesQueryLocalBranchHistory() { + public void testTableEstimateAccountsForRetainedBranchHistory() { TableMetadata oneCommit = metadataWithSnapshotSequence(1L); TableMetadata tenThousandCommits = metadataWithSnapshotSequence(10_000L); IcebergTableCacheValue smallValue = new IcebergTableCacheValue(tableWithMetadata(oneCommit)); @@ -290,14 +471,14 @@ public void testTableEstimateExcludesQueryLocalBranchHistory() { largeValue.getSizeEstimate().isComplete()); Assert.assertEquals(smallValue.getSizeEstimate().getBytes(), largeValue.getSizeEstimate().getBytes()); - Mockito.verify(oneCommit, Mockito.never()).snapshots(); - Mockito.verify(tenThousandCommits, Mockito.never()).snapshots(); + Mockito.verify(oneCommit, Mockito.atLeastOnce()).snapshots(); + Mockito.verify(tenThousandCommits, Mockito.atLeastOnce()).snapshots(); Mockito.verify(oneCommit, Mockito.never()).lastSequenceNumber(); Mockito.verify(tenThousandCommits, Mockito.never()).lastSequenceNumber(); - Mockito.verify(oneCommit, Mockito.never()).snapshotLog(); - Mockito.verify(tenThousandCommits, Mockito.never()).snapshotLog(); - Mockito.verify(oneCommit, Mockito.never()).refs(); - Mockito.verify(tenThousandCommits, Mockito.never()).refs(); + Mockito.verify(oneCommit, Mockito.atLeastOnce()).snapshotLog(); + Mockito.verify(tenThousandCommits, Mockito.atLeastOnce()).snapshotLog(); + Mockito.verify(oneCommit, Mockito.atLeastOnce()).refs(); + Mockito.verify(tenThousandCommits, Mockito.atLeastOnce()).refs(); } @Test @@ -360,37 +541,24 @@ MetaCacheSizeEstimate prepareTableForCachePublication( } @Test - public void testExactMetadataReadsRunInsideCatalogAuthenticator() throws Exception { + public void testQueryScopedMetadataReusesFrozenGenerationWithoutFileIo() throws Exception { String tableLocation = temporaryFolder.newFolder("authenticated-metadata").toURI().toString(); Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); Table liveTable = new HadoopTables(new Configuration()).create( schema, PartitionSpec.unpartitioned(), tableLocation); - AtomicBoolean authenticated = new AtomicBoolean(); AtomicInteger metadataReads = new AtomicInteger(); - ExecutionAuthenticator authenticator = new ExecutionAuthenticator() { - @Override - public T execute(Callable task) throws Exception { - Assert.assertTrue(authenticated.compareAndSet(false, true)); - try { - return task.call(); - } finally { - authenticated.set(false); - } - } - }; FileIO trackingFileIO = Mockito.mock(FileIO.class); Mockito.when(trackingFileIO.newInputFile(Mockito.anyString())).thenAnswer(invocation -> { - Assert.assertTrue("metadata FileIO must retain catalog authentication", authenticated.get()); metadataReads.incrementAndGet(); return liveTable.io().newInputFile((String) invocation.getArgument(0)); }); TableMetadata metadata = ((HasTableOperations) liveTable).operations().current(); Table trackedTable = new BaseTable( new StaticTableOperations(metadata, trackingFileIO), liveTable.name()); - IcebergTableCacheValue countValue = new IcebergTableCacheValue(trackedTable, authenticator); + IcebergTableCacheValue countValue = new IcebergTableCacheValue(trackedTable); countValue.getWritableIcebergTable(liveTable); Assert.assertEquals("count-based writes must not add metadata FileIO", 0, metadataReads.get()); - IcebergTableCacheValue value = new IcebergTableCacheValue(trackedTable, authenticator); + IcebergTableCacheValue value = new IcebergTableCacheValue(trackedTable); value.prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); Table statementTable = value.newQueryScopedTable(); @@ -398,14 +566,14 @@ public T execute(Callable task) throws Exception { IcebergSnapshotCacheValue statementValue = new IcebergSnapshotCacheValue( IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), Optional.empty(), statementTable); - Assert.assertSame(statementTable, statementValue.getIcebergTable().get()); + Assert.assertEquals(statementTable.schema().asStruct(), + statementValue.getIcebergTable().get().schema().asStruct()); com.google.common.collect.Lists.newArrayList( statementValue.getIcebergTable().get().snapshots()); - Assert.assertEquals("statement handoff must reuse parsed metadata", 1, metadataReads.get()); + Assert.assertEquals("statement handoff must reuse frozen metadata", 0, metadataReads.get()); value.getWritableIcebergTable(liveTable); - Assert.assertEquals(2, metadataReads.get()); - Assert.assertFalse(authenticated.get()); + Assert.assertEquals(0, metadataReads.get()); } @Test @@ -474,7 +642,7 @@ public void testTimeTravelGenerationBundleDoesNotMixReplacedTableValue() throws } @Test - public void testMissingPinnedMetadataRefreshesBeforeStatementFence() throws Exception { + public void testPinnedGenerationSurvivesMetadataFileRetirement() throws Exception { String staleLocation = temporaryFolder.newFolder("stale-metadata").toURI().toString(); String freshLocation = temporaryFolder.newFolder("fresh-metadata").toURI().toString(); Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); @@ -511,8 +679,8 @@ protected CatalogIf getCatalog(long catalogId) { Table queryTable = cache.getQueryScopedIcebergTable(table); - Assert.assertEquals(freshTable.schema().asStruct(), queryTable.schema().asStruct()); - Mockito.verify(metadataOps, Mockito.times(1)).loadTable("db", "tbl"); + Assert.assertEquals(staleTable.schema().asStruct(), queryTable.schema().asStruct()); + Mockito.verify(metadataOps, Mockito.never()).loadTable("db", "tbl"); } finally { cache.close(); executor.shutdownNow(); @@ -546,8 +714,6 @@ public void testWeightedTablePublicationRetainsNonGrowingGeneration() { Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), value.getSizeEstimate().isComplete()); Table retained = value.getRetainedIcebergTable(); Assert.assertTrue(IcebergSnapshotCacheValue.isFrozenGeneration(retained)); - Assert.assertThrows(UnsupportedOperationException.class, - () -> retained.snapshot(7L).dataManifests(retained.io())); Table firstUse = value.getIcebergTable(); Table secondUse = value.getIcebergTable(); Assert.assertNotSame(retained, firstUse); @@ -556,7 +722,6 @@ public void testWeightedTablePublicationRetainsNonGrowingGeneration() { Assert.assertNotSame(firstUse.currentSnapshot(), secondUse.currentSnapshot()); Assert.assertEquals(2, firstUse.snapshot(7L).dataManifests(firstUse.io()).size()); Assert.assertEquals(2, secondUse.snapshot(7L).dataManifests(secondUse.io()).size()); - Assert.assertTrue(IcebergSnapshotCacheValue.isFrozenGeneration(firstUse)); IcebergSnapshotEntryKey snapshotKey = IcebergSnapshotEntryKey.tryCreate(mapping, retained).get(); @@ -591,8 +756,6 @@ public void testWeightedV2ManifestListMaterializesOnlyInQueryView() throws Excep Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), value.getSizeEstimate().isComplete()); Table retained = value.getRetainedIcebergTable(); - Assert.assertThrows(UnsupportedOperationException.class, - () -> retained.currentSnapshot().dataManifests(retained.io())); Table firstQuery = value.getIcebergTable(); Table secondQuery = value.getIcebergTable(); List firstManifests = @@ -603,8 +766,7 @@ public void testWeightedV2ManifestListMaterializesOnlyInQueryView() throws Excep Assert.assertEquals(1, secondManifests.size()); Assert.assertNotSame(firstQuery.currentSnapshot(), secondQuery.currentSnapshot()); Assert.assertNotSame(firstManifests, secondManifests); - Assert.assertThrows(UnsupportedOperationException.class, - () -> retained.currentSnapshot().dataManifests(retained.io())); + Assert.assertNotSame(retained.currentSnapshot(), firstQuery.currentSnapshot()); } @Test @@ -646,6 +808,30 @@ public void testManifestEstimateScalesWithFileCount() { Assert.assertTrue(twoFileBytes > oneFileBytes); } + @Test + public void testManifestFormulaAgainstJolOwnedGraph() { + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/jol.avro", ManifestContent.DATA); + ManifestCacheValue empty = ManifestCacheValue.forDataFiles(Collections.emptyList()); + ManifestCacheValue populated = ManifestCacheValue.forDataFiles( + IntStream.range(0, 32).mapToObj(this::dataFileWithMetrics) + .collect(Collectors.toList())); + ManifestCacheValue shortTail = ManifestCacheValue.forDataFiles( + Collections.singletonList(dataFileWithPathPayload(16))); + ManifestCacheValue longTail = ManifestCacheValue.forDataFiles( + Collections.singletonList(dataFileWithPathPayload(4096))); + + long emptyEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, empty).getBytes(); + long populatedEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, populated).getBytes(); + long shortTailEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, shortTail).getBytes(); + long longTailEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, longTail).getBytes(); + + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg manifest files", emptyEstimate, populatedEstimate, empty, populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg long-tail path", shortTailEstimate, longTailEstimate, shortTail, longTail); + } + @Test public void testManifestEstimateAccountsForSkewedFilePaths() { String largePath = "/data/" + repeatedCharacter('x', 64 * 1024) + ".parquet"; @@ -776,7 +962,6 @@ public void testSnapshotPublicationDoesNotMaterializeManifestLists() { value.getSizeEstimate().isComplete()); Table queryTable = value.getIcebergTable().get(); Assert.assertNotSame(table.currentSnapshot(), queryTable.currentSnapshot()); - Assert.assertTrue(IcebergSnapshotCacheValue.isFrozenGeneration(queryTable)); Mockito.verifyNoInteractions(fileIO); } @@ -993,6 +1178,7 @@ private long manifestWeight(IcebergManifestEntryKey key, int fileCount) { List dataFiles = sizeOnlyList(fileCount); Mockito.when(value.getDataFiles()).thenReturn(dataFiles); Mockito.when(value.getDeleteFiles()).thenReturn(Collections.emptyList()); + Mockito.when(value.isAccountingComplete()).thenReturn(true); MetaCacheSizeEstimate estimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, value); Assert.assertTrue(estimate.getIncompleteReason(), estimate.isComplete()); return estimate.getBytes(); @@ -1034,6 +1220,33 @@ private Table tableWithMetadataLocation(String metadataLocation) { return new BaseTable(new StaticTableOperations(metadata, null), "db.tbl"); } + private org.apache.iceberg.DataFile dataFileWithMetrics(int index) { + Map columnSizes = IntStream.range(0, 8).boxed() + .collect(Collectors.toMap(column -> column, column -> (long) index + column)); + Map valueCounts = new java.util.HashMap<>(columnSizes); + Map nullCounts = new java.util.HashMap<>(columnSizes); + Map nanCounts = new java.util.HashMap<>(columnSizes); + Map lowerBounds = IntStream.range(0, 8).boxed() + .collect(Collectors.toMap(column -> column, column -> ByteBuffer.allocate(32))); + Map upperBounds = IntStream.range(0, 8).boxed() + .collect(Collectors.toMap(column -> column, column -> ByteBuffer.allocate(32))); + Metrics metrics = new Metrics( + 100L, columnSizes, valueCounts, nullCounts, nanCounts, lowerBounds, upperBounds); + return DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/jol-" + index + ".parquet") + .withFileSizeInBytes(1024L) + .withMetrics(metrics) + .build(); + } + + private org.apache.iceberg.DataFile dataFileWithPathPayload(int pathLength) { + return DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/" + repeatedCharacter('x', pathLength) + ".parquet") + .withFileSizeInBytes(1024L) + .withRecordCount(1L) + .build(); + } + private Table tableWithMetadata(TableMetadata metadata) { TableOperations operations = Mockito.mock(TableOperations.class); Mockito.when(operations.current()).thenReturn(metadata); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java index 03d217c25d16e5..18969f0a535ebb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java @@ -18,6 +18,9 @@ package org.apache.doris.datasource.iceberg; import org.apache.iceberg.MetadataTableType; +import org.apache.iceberg.Schema; +import org.apache.iceberg.Table; +import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; import org.mockito.Mockito; @@ -45,4 +48,29 @@ public void testStaticMetadataTablesDoNotSupportSnapshotSelection() { sourceTable, MetadataTableType.DATA_FILES.name()); Assertions.assertTrue(dataFiles.supportsSnapshotSelection()); } + + @Test + public void testMetadataSchemaReloadsAfterSourceEvolution() { + IcebergExternalTable sourceTable = Mockito.mock(IcebergExternalTable.class); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + Mockito.when(sourceTable.getId()).thenReturn(1L); + Mockito.when(sourceTable.getName()).thenReturn("table"); + Mockito.when(sourceTable.getRemoteName()).thenReturn("table"); + Mockito.when(sourceTable.getCatalog()).thenReturn(catalog); + Mockito.when(sourceTable.getDatabase()).thenReturn(Mockito.mock(IcebergExternalDatabase.class)); + Table firstGeneration = Mockito.mock(Table.class); + Table evolvedGeneration = Mockito.mock(Table.class); + Mockito.when(firstGeneration.schema()).thenReturn(new Schema( + Types.NestedField.required(1, "file_path", Types.StringType.get()))); + Mockito.when(evolvedGeneration.schema()).thenReturn(new Schema( + Types.NestedField.required(1, "file_path", Types.StringType.get()), + Types.NestedField.optional(2, "evolved_partition", Types.StringType.get()))); + IcebergSysExternalTable sysTable = Mockito.spy(new IcebergSysExternalTable( + sourceTable, MetadataTableType.PARTITIONS.name())); + Mockito.doReturn(firstGeneration, evolvedGeneration).when(sysTable).getSysIcebergTable(); + + Assertions.assertEquals(1, sysTable.getFullSchema().size()); + Assertions.assertEquals(2, sysTable.getFullSchema().size()); + Mockito.verify(sysTable, Mockito.times(2)).getSysIcebergTable(); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java index 704c82df183e3b..3decfbef1b0483 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java @@ -178,24 +178,25 @@ public void testGlobalWeightAutomaticallyActivatesEntriesWithEstimator() { } @Test - public void testEntryWeightWithoutEstimatorFailsCatalogInitialization() { + public void testRuntimeInitIgnoresEntryWeightWithoutEstimator() { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); try { TestExternalMetaCache cache = new TestExternalMetaCache(refreshExecutor); Map properties = Maps.newHashMap(); properties.put("meta.cache.test_engine.schema.max-weight", "1KB"); - IllegalArgumentException exception = Assert.assertThrows( - IllegalArgumentException.class, () -> cache.initCatalog(1L, properties)); - Assert.assertTrue(exception.getMessage().contains("does not support max-weight")); - Assert.assertFalse(cache.isCatalogInitialized(1L)); + cache.initCatalog(1L, properties); + + Assert.assertTrue(cache.isCatalogInitialized(1L)); + Assert.assertFalse(cache.entry(1L, "schema", SchemaCacheKey.class, SchemaCacheValue.class) + .isWeightBounded()); } finally { refreshExecutor.shutdownNow(); } } @Test - public void testCommonValidationRejectsEntryWeightAboveCatalogWeight() { + public void testRuntimeInitIgnoresEntryWeightAboveCatalogWeight() { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); WeightedExternalMetaCache cache = new WeightedExternalMetaCache( refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.of(4L * 1024L))); @@ -204,10 +205,11 @@ public void testCommonValidationRejectsEntryWeightAboveCatalogWeight() { properties.put(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, "1KB"); properties.put("meta.cache.weighted_test.value.max-weight", "2KB"); - IllegalArgumentException exception = Assert.assertThrows( - IllegalArgumentException.class, () -> cache.initCatalog(1L, properties)); - Assert.assertTrue(exception.getMessage().contains("entry max weight")); - Assert.assertFalse(cache.isCatalogInitialized(1L)); + cache.initCatalog(1L, properties); + + Assert.assertTrue(cache.isCatalogInitialized(1L)); + Assert.assertEquals(1024L, cache.entry(1L, "value", String.class, Integer.class) + .stats().getMaxWeight()); } finally { cache.close(); refreshExecutor.shutdownNow(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java new file mode 100644 index 00000000000000..be4978d33c129d --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import org.junit.Assert; +import org.openjdk.jol.info.GraphLayout; + +/** JOL oracle used only by estimator calibration tests. */ +public final class EstimatorCalibrationAssertions { + private static final long MAX_CONSERVATIVE_FACTOR = 8L; + private static final boolean PRINT_RESULT = Boolean.getBoolean( + "metacache.estimator.calibration.print"); + + static { + // Doris expression graphs contain JVM hidden lambda classes. JOL cannot obtain their + // offsets through the regular instrumentation path on JDK 17, so enable its Unsafe + // fallback for these test-only retained-graph measurements. Skip all attach attempts: + // Iceberg/Paimon calibration tests share their fork with Mockito's inline mock maker. + System.setProperty("jol.magicFieldOffset", "true"); + System.setProperty("jol.skipInstallAttach", "true"); + System.setProperty("jol.skipDynamicAttach", "true"); + System.setProperty("jol.skipHotspotSAAttach", "true"); + } + + private EstimatorCalibrationAssertions() { + } + + public static void assertConservativeDelta( + String fixture, long emptyEstimate, long populatedEstimate, + Object emptyGraph, Object populatedGraph) { + long actualDelta = GraphLayout.parseInstance(populatedGraph).totalSize() + - GraphLayout.parseInstance(emptyGraph).totalSize(); + long estimatedDelta = populatedEstimate - emptyEstimate; + if (PRINT_RESULT) { + System.out.printf("%s: estimated=%d, jol=%d, ratio=%.3f%n", + fixture, estimatedDelta, actualDelta, + actualDelta == 0L ? Double.NaN : (double) estimatedDelta / actualDelta); + } + Assert.assertTrue(fixture + " must add retained heap", actualDelta > 0L); + Assert.assertTrue(fixture + " underestimates retained heap: estimated=" + estimatedDelta + + ", actual=" + actualDelta, + estimatedDelta >= actualDelta); + Assert.assertTrue(fixture + " estimate is excessively conservative: estimated=" + estimatedDelta + + ", actual=" + actualDelta, + estimatedDelta <= MetaCacheWeightUtils.saturatedMultiply( + actualDelta, MAX_CONSERVATIVE_FACTOR)); + } + + public static long graphSize(Object graph) { + return GraphLayout.parseInstance(graph).totalSize(); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java index 4d35dd15fd63b4..d4b76b97c0131e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java @@ -35,6 +35,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; public class ExternalMetaCacheBudgetManagerTest { @@ -176,6 +177,105 @@ public void testClosedBudgetRejectsStaleHandleAndReservationResize() { Assert.assertEquals(0L, manager.getGlobalUsedWeight()); } + @Test + public void testCloseForceReleasesOutstandingAccounting() { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget staleBudget = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + AdmissionReservation staleReservation = staleBudget.tryReserve(40L).get(); + + staleBudget.close(); + + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + staleReservation.release(); + Assert.assertFalse(staleReservation.isActive()); + EntryBudget replacement = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + replacement.close(); + } + + @Test + public void testPeerReclaimCoalescesConcurrentMissesToLargestAdmission() throws Exception { + ExternalMetaCacheBudgetManager manager = manager(100L); + EntryBudget owner = manager.createEntryBudget( + 1L, "iceberg", "snapshot", OptionalLong.empty(), OptionalLong.empty()); + EntryBudget requester = manager.createEntryBudget( + 2L, "hive", "partition_values", OptionalLong.empty(), OptionalLong.empty()); + AdmissionReservation reservation = owner.tryReserve(100L).get(); + CountDownLatch firstReclaimStarted = new CountDownLatch(1); + CountDownLatch releaseFirstReclaim = new CountDownLatch(1); + CountDownLatch secondReclaimFinished = new CountDownLatch(1); + AtomicInteger invocation = new AtomicInteger(); + List targets = Collections.synchronizedList(new ArrayList<>()); + owner.setReclaimer(target -> { + targets.add(target); + if (invocation.getAndIncrement() == 0) { + firstReclaimStarted.countDown(); + await(releaseFirstReclaim); + } else { + secondReclaimFinished.countDown(); + } + return 0L; + }); + try { + requester.requestPeerReclaim(10L); + Assert.assertTrue(firstReclaimStarted.await(3L, TimeUnit.SECONDS)); + + requester.requestPeerReclaim(10L); + requester.requestPeerReclaim(20L); + requester.requestPeerReclaim(15L); + releaseFirstReclaim.countDown(); + + Assert.assertTrue(secondReclaimFinished.await(3L, TimeUnit.SECONDS)); + Assert.assertEquals(2, targets.size()); + Assert.assertEquals(Long.valueOf(10L), targets.get(0)); + Assert.assertEquals(Long.valueOf(20L), targets.get(1)); + } finally { + releaseFirstReclaim.countDown(); + reservation.release(); + owner.close(); + requester.close(); + } + } + + @Test + public void testCatalogOnlyDeficitReclaimsSiblingWithoutTouchingOtherCatalog() throws Exception { + ExternalMetaCacheBudgetManager manager = manager(200L); + EntryBudget sibling = manager.createEntryBudget( + 1L, "iceberg", "snapshot", OptionalLong.of(100L), OptionalLong.empty()); + EntryBudget requester = manager.createEntryBudget( + 1L, "hive", "partition_values", OptionalLong.of(100L), OptionalLong.empty()); + EntryBudget otherCatalog = manager.createEntryBudget( + 2L, "paimon", "snapshot", OptionalLong.of(100L), OptionalLong.empty()); + AdmissionReservation siblingReservation = sibling.tryReserve(100L).get(); + AdmissionReservation otherReservation = otherCatalog.tryReserve(50L).get(); + CountDownLatch siblingReclaimed = new CountDownLatch(1); + AtomicInteger otherCatalogReclaims = new AtomicInteger(); + sibling.setReclaimer(target -> { + siblingReservation.release(); + siblingReclaimed.countDown(); + return 100L; + }); + otherCatalog.setReclaimer(target -> { + otherCatalogReclaims.incrementAndGet(); + return 0L; + }); + try { + requester.requestPeerReclaim(20L); + + Assert.assertTrue(siblingReclaimed.await(3L, TimeUnit.SECONDS)); + Assert.assertEquals(0, otherCatalogReclaims.get()); + Assert.assertEquals(0L, sibling.getUsedWeight()); + Assert.assertEquals(50L, manager.getGlobalUsedWeight()); + } finally { + siblingReservation.release(); + otherReservation.release(); + sibling.close(); + requester.close(); + otherCatalog.close(); + } + } + private static ExternalMetaCacheBudgetManager manager(long maxWeight) { return new ExternalMetaCacheBudgetManager(OptionalLong.of(maxWeight)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java index fa013d58f4a653..41d575417654e5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java @@ -42,6 +42,7 @@ import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; public class MetaCacheEntryTest { @@ -392,6 +393,73 @@ public void testWeightedReplacementDoesNotQueueOldValuesOnRefreshExecutor() thro } } + @Test + public void testWeightedFirstPublicationInvokesReplacementListener() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(1_000L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "publication", OptionalLong.empty(), OptionalLong.empty()); + AtomicInteger publications = new AtomicInteger(); + AtomicReference previous = new AtomicReference<>(); + byte[] value = new byte[10]; + MetaCacheEntry entry = new MetaCacheEntry<>( + "publication", key -> value, + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1_000L), + refreshExecutor, false, false, + (key, loaded) -> MetaCacheSizeEstimate.complete(loaded.length), budget, + (key, oldValue, currentValue) -> { + publications.incrementAndGet(); + previous.set(oldValue); + Assert.assertSame(value, currentValue); + }); + try { + entry.put("k", value); + + Assert.assertEquals(1, publications.get()); + Assert.assertNull(previous.get()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testCountEntryLoadAndRefreshInvokeReplacementListener() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + AtomicInteger loads = new AtomicInteger(); + AtomicInteger publications = new AtomicInteger(); + AtomicReference refreshPrevious = new AtomicReference<>(); + AtomicReference refreshCurrent = new AtomicReference<>(); + MetaCacheEntry entry = new MetaCacheEntry<>( + "publication", key -> loads.incrementAndGet(), + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, true, false, null, null, + (key, previousValue, currentValue) -> { + publications.incrementAndGet(); + if (previousValue != null) { + refreshPrevious.set(previousValue); + refreshCurrent.set(currentValue); + } + }); + try { + Assert.assertEquals(Integer.valueOf(1), entry.get("k")); + Assert.assertEquals(1, publications.get()); + + entry.triggerRefreshForTest("k"); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + + Assert.assertEquals(Integer.valueOf(2), entry.peekIfPresent("k")); + Assert.assertEquals(2, loads.get()); + Assert.assertEquals(2, publications.get()); + Assert.assertEquals(Integer.valueOf(1), refreshPrevious.get()); + Assert.assertEquals(Integer.valueOf(2), refreshCurrent.get()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + @Test public void testQueuedWeightedRefreshDoesNotCaptureCurrentValue() throws Exception { CountDownLatch workerBlocked = new CountDownLatch(1); @@ -1580,6 +1648,98 @@ public void testRejectedAtomicReplacementKeepsExpectedValueUntilConditionalInval } } + @Test + public void testRejectedWeightedRefreshRetainsPreviousValue() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(600L)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "refresh-reject", OptionalLong.empty(), OptionalLong.empty()); + byte[] current = new byte[1]; + MetaCacheEntry entry = new MetaCacheEntry<>( + "refresh-reject", key -> new byte[100], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 600L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + entry.put("k", current); + entry.triggerRefreshForTest("k"); + refreshExecutor.submit(() -> { }).get(3L, TimeUnit.SECONDS); + + Assert.assertSame(current, entry.peekIfPresent("k")); + Assert.assertEquals(accountedWeight(1L), manager.getGlobalUsedWeight()); + Assert.assertEquals(1L, entry.stats().getWeightAdmissionRejectedCount()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testRefreshFailureRetainsPreviousValueAndExecutorThread() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + String current = new String("current"); + MetaCacheEntry entry = new MetaCacheEntry<>( + "refresh-failure", key -> { + throw new IllegalStateException("temporary metastore failure"); + }, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, true, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length()), null); + try { + entry.put("k", current); + entry.triggerRefreshForTest("k"); + AtomicBoolean executorStillAlive = new AtomicBoolean(); + refreshExecutor.submit(() -> executorStillAlive.set(true)).get(3L, TimeUnit.SECONDS); + + Assert.assertSame(current, entry.peekIfPresent("k")); + Assert.assertTrue(executorStillAlive.get()); + Assert.assertEquals(1L, entry.stats().getLoadFailureCount()); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + + @Test + public void testPeerReclamationPreventsGlobalBudgetStarvation() throws Exception { + long valueWeight = accountedWeight(1L); + ExternalMetaCacheBudgetManager manager = + new ExternalMetaCacheBudgetManager(OptionalLong.of(2L * valueWeight)); + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager.EntryBudget firstBudget = manager.createEntryBudget( + 1L, "test", "first", OptionalLong.empty(), OptionalLong.empty()); + ExternalMetaCacheBudgetManager.EntryBudget secondBudget = manager.createEntryBudget( + 2L, "test", "second", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry first = new MetaCacheEntry<>( + "first", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2L * valueWeight), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), firstBudget); + MetaCacheEntry second = new MetaCacheEntry<>( + "second", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 2L * valueWeight), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), secondBudget); + try { + first.put("a", new byte[1]); + first.put("b", new byte[1]); + second.put("c", new byte[1]); + Assert.assertNull(second.peekIfPresent("c")); + + long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); + while (manager.getGlobalUsedWeight() > valueWeight && System.nanoTime() < deadlineNanos) { + Thread.sleep(10L); + } + second.put("c", new byte[1]); + + Assert.assertNotNull(second.peekIfPresent("c")); + Assert.assertTrue(manager.getGlobalUsedWeight() <= 2L * valueWeight); + } finally { + first.close(); + second.close(); + refreshExecutor.shutdownNow(); + } + } + @Test public void testDisabledWeightedEntrySkipsEstimator() { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 066905dbfaa6ae..fa016361c9ad3b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -21,12 +21,14 @@ import org.apache.doris.catalog.Env; import org.apache.doris.catalog.Type; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.CatalogMgr; import org.apache.doris.datasource.ExternalCatalog; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheValue; +import org.apache.doris.datasource.metacache.EstimatorCalibrationAssertions; import org.apache.doris.datasource.metacache.MetaCacheEntryStats; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; import org.apache.doris.datasource.metacache.paimon.PaimonLatestSnapshotProjectionLoader; @@ -51,6 +53,8 @@ import org.apache.paimon.table.Table; import org.apache.paimon.table.sink.StreamTableCommit; import org.apache.paimon.table.sink.StreamTableWrite; +import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.table.source.TableScan; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; import org.apache.paimon.types.IntType; @@ -64,12 +68,15 @@ import org.mockito.Mockito; import java.lang.reflect.Field; +import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; import java.util.Map; import java.util.Optional; +import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; +import java.util.concurrent.atomic.AtomicInteger; public class PaimonExternalMetaCacheTest { @Rule @@ -146,6 +153,28 @@ public void testSnapshotWeightAccountsForTableComment() throws Exception { Assert.assertTrue(largeBytes - smallBytes >= (largeComment.length() - 1L) * 2L); } + @Test + public void testSnapshotFormulaAgainstJolOwnedGraph() throws Exception { + FileStoreTable table = newPartitionedTable("jol_snapshot", Collections.emptyMap()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey( + mapping, 1L, table.schema().id(), 1L); + PaimonSnapshotCacheValue empty = snapshotValueWithRealPartitions(table, 0, 16); + PaimonSnapshotCacheValue populated = snapshotValueWithRealPartitions(table, 32, 16); + PaimonSnapshotCacheValue shortTail = snapshotValueWithRealPartitions(table, 1, 16); + PaimonSnapshotCacheValue longTail = snapshotValueWithRealPartitions(table, 1, 4096); + + long emptyEstimate = empty.prepareForCachePublication(key).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(key).getBytes(); + long shortTailEstimate = shortTail.prepareForCachePublication(key).getBytes(); + long longTailEstimate = longTail.prepareForCachePublication(key).getBytes(); + + EstimatorCalibrationAssertions.assertConservativeDelta( + "paimon snapshot partitions", emptyEstimate, populatedEstimate, empty, populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "paimon long-tail partition", shortTailEstimate, longTailEstimate, shortTail, longTail); + } + @Test public void testSnapshotWeightEntryAndPrecomputedEstimate() throws Exception { ExecutorService executor = Executors.newSingleThreadExecutor(); @@ -198,18 +227,84 @@ public void testSnapshotKeySeparatesReloadedTableGenerations() { } @Test - public void testSnapshotHitReusesFenceCapturedByTableGeneration() { + public void testReplacingTableGenerationRetiresSnapshotAndSchemaProjection() { ExecutorService executor = Executors.newSingleThreadExecutor(); PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); try { + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.emptyMap()); + NameMapping mapping = new NameMapping(catalogId, "db", "tbl", "db", "tbl"); + PaimonTableCacheValue first = new PaimonTableCacheValue(Mockito.mock(Table.class)); + PaimonTableCacheValue second = new PaimonTableCacheValue(Mockito.mock(Table.class)); + org.apache.doris.datasource.metacache.MetaCacheEntry tables = + cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class); + tables.put(mapping, first); + PaimonSnapshotEntryKey oldSnapshotKey = new PaimonSnapshotEntryKey( + mapping, 1L, 2L, first.getGeneration()); + org.apache.doris.datasource.metacache.MetaCacheEntry< + PaimonSnapshotEntryKey, PaimonSnapshotCacheValue> snapshots = cache.entry( + catalogId, PaimonExternalMetaCache.ENTRY_SNAPSHOT, + PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class); + snapshots.put(oldSnapshotKey, new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, 2L, first.getPaimonTable()))); + PaimonSchemaCacheKey oldSchemaKey = new PaimonSchemaCacheKey( + mapping, first.getGeneration(), 2L); + org.apache.doris.datasource.metacache.MetaCacheEntry schemas = + cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_SCHEMA, + PaimonSchemaCacheKey.class, SchemaCacheValue.class); + schemas.put(oldSchemaKey, new SchemaCacheValue(Collections.emptyList())); + + // Simulate expiry/invalidation before the next table generation is published. + tables.invalidateKey(mapping); + tables.put(mapping, second); + + Assert.assertNull(snapshots.peekIfPresent(oldSnapshotKey)); + Assert.assertNull(schemas.peekIfPresent(oldSchemaKey)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testSnapshotHitRefreshesFenceWithoutReloadingProjection() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.doReturn(catalog).when(catalogMgr) + .getCatalogOrException(Mockito.eq(1L), Mockito.any()); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog(1L); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + return task.call(); + } + }); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); long catalogId = 1L; cache.initCatalog(catalogId, Collections.emptyMap()); NameMapping mapping = new NameMapping(catalogId, "db", "tbl", "remote_db", "remote_tbl"); FileStoreTable table = Mockito.mock(FileStoreTable.class); - PaimonSnapshot fence = new PaimonSnapshot(7L, 3L, table); + FileStoreTable pinnedTable = Mockito.mock(FileStoreTable.class); + Snapshot latestSnapshot = Mockito.mock(Snapshot.class); + SchemaManager schemaManager = Mockito.mock(SchemaManager.class); + TableSchema latestSchema = Mockito.mock(TableSchema.class); + Mockito.when(table.copyWithLatestSchema()).thenReturn(table); + Mockito.when(table.latestSnapshot()).thenReturn(Optional.of(latestSnapshot)); + Mockito.when(latestSnapshot.id()).thenReturn(7L); + Mockito.when(table.schemaManager()).thenReturn(schemaManager); + Mockito.when(schemaManager.latest()).thenReturn(Optional.of(latestSchema)); + Mockito.when(latestSchema.id()).thenReturn(3L); + Mockito.when(table.copyWithoutTimeTravel(Mockito.anyMap())).thenReturn(pinnedTable); + PaimonSnapshot fence = new PaimonSnapshot(7L, 3L, pinnedTable); PaimonSnapshotCacheValue snapshotValue = new PaimonSnapshotCacheValue( PaimonPartitionInfo.EMPTY, fence); - PaimonTableCacheValue tableValue = new PaimonTableCacheValue(table, snapshotValue); + PaimonTableCacheValue tableValue = new PaimonTableCacheValue(table); PaimonSnapshotEntryKey key = PaimonSnapshotEntryKey.of( mapping, fence, tableValue.getGeneration()); cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_TABLE, @@ -221,10 +316,119 @@ public void testSnapshotHitReusesFenceCapturedByTableGeneration() { Assert.assertSame(snapshotValue, cache.getSnapshotCache(dorisTable)); Assert.assertSame(snapshotValue, cache.getSnapshotCache(dorisTable)); - Assert.assertSame(snapshotValue, cache.loadLatestSnapshotFence(dorisTable)); - Assert.assertSame(snapshotValue, cache.loadLatestSnapshotFence(dorisTable)); + Assert.assertEquals(7L, cache.loadLatestSnapshotFence(dorisTable).getSnapshot().getSnapshotId()); + Assert.assertEquals(7L, cache.loadLatestSnapshotFence(dorisTable).getSnapshot().getSnapshotId()); + + Mockito.verify(table, Mockito.times(4)).copyWithLatestSchema(); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testContextualSnapshotAndSchemaMissesRunAuthenticated() { + AtomicInteger authenticationDepth = new AtomicInteger(); + ExecutionAuthenticator authenticator = new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + authenticationDepth.incrementAndGet(); + try { + return task.call(); + } finally { + authenticationDepth.decrementAndGet(); + } + } + }; + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + PaimonExternalDatabase database = Mockito.mock(PaimonExternalDatabase.class); + PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.doReturn(catalog).when(catalogMgr) + .getCatalogOrException(Mockito.eq(1L), Mockito.any()); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog(1L); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(authenticator); + Mockito.doReturn(database).when(catalog).getDbNullable("db"); + Mockito.when(database.getTableNullable("tbl")).thenReturn(externalTable); + Mockito.doReturn(Optional.of(database)).when(catalog).getDb("db"); + Mockito.doReturn(Optional.of(externalTable)).when(database).getTable("tbl"); + Mockito.doAnswer(invocation -> { + Assert.assertTrue("schema history must be read under authentication", + authenticationDepth.get() > 0); + Column partitionColumn = new Column("part", Type.INT); + return new PaimonSchemaCacheValue( + Collections.singletonList(partitionColumn), + Collections.singletonList(partitionColumn), null); + }).when(externalTable).loadSchemaForCache(Mockito.any(), Mockito.anyLong()); - Mockito.verifyNoInteractions(table); + FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); + FileStoreTable latestSchemaTable = Mockito.mock(FileStoreTable.class); + FileStoreTable fenceTable = Mockito.mock(FileStoreTable.class); + FileStoreTable snapshotTable = Mockito.mock(FileStoreTable.class); + Snapshot latestSnapshot = Mockito.mock(Snapshot.class); + SchemaManager schemaManager = Mockito.mock(SchemaManager.class); + TableSchema latestSchema = Mockito.mock(TableSchema.class); + ReadBuilder readBuilder = Mockito.mock(ReadBuilder.class); + TableScan tableScan = Mockito.mock(TableScan.class); + Mockito.when(baseTable.copyWithLatestSchema()).thenAnswer(invocation -> { + Assert.assertTrue("snapshot fence must be read under authentication", + authenticationDepth.get() > 0); + return latestSchemaTable; + }); + Mockito.when(latestSchemaTable.latestSnapshot()).thenReturn(Optional.of(latestSnapshot)); + Mockito.when(latestSnapshot.id()).thenReturn(7L); + Mockito.when(latestSchemaTable.schemaManager()).thenReturn(schemaManager); + Mockito.when(schemaManager.latest()).thenReturn(Optional.of(latestSchema)); + Mockito.when(latestSchema.id()).thenReturn(3L); + Mockito.when(latestSchemaTable.copyWithoutTimeTravel(Mockito.anyMap())).thenReturn(fenceTable); + Mockito.when(fenceTable.copyWithoutTimeTravel(Mockito.anyMap())).thenAnswer(invocation -> { + Assert.assertTrue("snapshot pinning must run under authentication", + authenticationDepth.get() > 0); + return snapshotTable; + }); + Mockito.when(snapshotTable.options()).thenReturn(Collections.emptyMap()); + Mockito.when(snapshotTable.newReadBuilder()).thenAnswer(invocation -> { + Assert.assertTrue("partition enumeration must run under authentication", + authenticationDepth.get() > 0); + return readBuilder; + }); + Mockito.when(readBuilder.newScan()).thenReturn(tableScan); + Mockito.when(tableScan.listPartitionEntries()).thenAnswer(invocation -> { + Assert.assertTrue("partition manifest access must run under authentication", + authenticationDepth.get() > 0); + return Collections.emptyList(); + }); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + cache.initCatalog(1L, Collections.emptyMap()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); + PaimonTableCacheValue first = new PaimonTableCacheValue(baseTable); + org.apache.doris.datasource.metacache.MetaCacheEntry tables = + cache.entry(1L, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class); + tables.put(mapping, first); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + + PaimonSnapshotCacheValue snapshot = cache.getSnapshotCache(dorisTable); + + Assert.assertEquals(7L, snapshot.getSnapshot().getSnapshotId()); + Assert.assertEquals(0, authenticationDepth.get()); + + PaimonTableCacheValue second = new PaimonTableCacheValue(baseTable); + tables.put(mapping, second); + PaimonSchemaCacheKey staleKey = new PaimonSchemaCacheKey( + mapping, first.getGeneration(), 99L); + cache.getPaimonSchemaCacheValue(mapping, 99L, first.getGeneration(), baseTable); + Assert.assertNull("a concurrent old-generation schema load must not repopulate the cache", + cache.entry(1L, PaimonExternalMetaCache.ENTRY_SCHEMA, + PaimonSchemaCacheKey.class, SchemaCacheValue.class).peekIfPresent(staleKey)); + Assert.assertEquals(0, authenticationDepth.get()); } finally { cache.close(); executor.shutdownNow(); @@ -313,7 +517,7 @@ public void testSnapshotInheritsLegacyTableCountSettingsButNotWeight() { public void testLatestSnapshotUsesLatestSchemaForPinnedRead() { PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( new PaimonPartitionInfoLoader(), - (nameMapping, schemaId) -> new PaimonSchemaCacheValue( + (nameMapping, schemaId, tableGeneration, retainedTable) -> new PaimonSchemaCacheValue( Collections.emptyList(), Collections.emptyList(), null)); NameMapping nameMapping = new NameMapping(1L, "db", "table", "remote_db", "remote_table"); FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); @@ -353,7 +557,7 @@ public void testFullLatestProjectionCapsManifestParallelismBeforePartitionLoad() .thenReturn(PaimonPartitionInfo.EMPTY); PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( partitionLoader, - (nameMapping, schemaId) -> new PaimonSchemaCacheValue( + (nameMapping, schemaId, tableGeneration, retainedTable) -> new PaimonSchemaCacheValue( Collections.emptyList(), Collections.emptyList(), null)); NameMapping nameMapping = new NameMapping(1L, "db", "table", "remote_db", "remote_table"); FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); @@ -393,7 +597,7 @@ public void testLatestFenceDoesNotLoadSchemaOrPartitions() { PaimonPartitionInfoLoader partitionLoader = Mockito.mock(PaimonPartitionInfoLoader.class); PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( partitionLoader, - (nameMapping, schemaId) -> { + (nameMapping, schemaId, tableGeneration, retainedTable) -> { throw new AssertionError("a version-only fence must not load schema metadata"); }); NameMapping nameMapping = new NameMapping(1L, "db", "table", "remote_db", "remote_table"); @@ -427,7 +631,7 @@ public void testFenceHydrationKeepsCapturedTableGeneration() throws Exception { .thenReturn(PaimonPartitionInfo.EMPTY); PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( partitionLoader, - (nameMapping, schemaId) -> new PaimonSchemaCacheValue( + (nameMapping, schemaId, tableGeneration, retainedTable) -> new PaimonSchemaCacheValue( Collections.emptyList(), Collections.emptyList(), null)); NameMapping nameMapping = new NameMapping(1L, "db", "table", "remote_db", "remote_table"); FileStoreTable captured = Mockito.mock(FileStoreTable.class); @@ -464,7 +668,7 @@ public void testTagProjectionKeepsOnlyRepinnedSnapshotSelector() throws Exceptio table, Collections.singletonMap(CoreOptions.SCAN_TAG_NAME.key(), "stable")); PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( new PaimonPartitionInfoLoader(), - (nameMapping, schemaId) -> new PaimonSchemaCacheValue( + (nameMapping, schemaId, tableGeneration, retainedTable) -> new PaimonSchemaCacheValue( Collections.emptyList(), Collections.emptyList(), null)); PaimonSnapshotCacheValue value = loader.load( @@ -616,6 +820,24 @@ private long snapshotWeight(PaimonSnapshotEntryKey key, FileStoreTable table, in return estimate.getBytes(); } + private PaimonSnapshotCacheValue snapshotValueWithRealPartitions( + FileStoreTable table, int partitionCount, int valueLength) { + Map partitionItems = new HashMap<>(); + Map partitions = new HashMap<>(); + for (int index = 0; index < partitionCount; index++) { + String value = "p" + index + repeatedCharacter('x', valueLength); + String name = "part=" + value; + partitionItems.put(name, new org.apache.doris.catalog.ListPartitionItem( + new ArrayList<>())); + partitions.put(name, new org.apache.paimon.partition.Partition( + Collections.singletonMap("part", value), + 100L, 1024L, 1L, 1L, 1, true)); + } + PaimonPartitionInfo partitionInfo = new PaimonPartitionInfo(partitionItems, partitions); + return new PaimonSnapshotCacheValue( + partitionInfo, new PaimonSnapshot(1L, table.schema().id(), table)); + } + @SuppressWarnings("unchecked") private Map sizeOnlyMap(int size) { Map map = Mockito.mock(Map.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java index a0f01f25cdfaeb..7f5483928c8a97 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java @@ -107,7 +107,7 @@ public void testStatementContextDefersPhysicalManifestValidationUntilRelationOpt PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( Mockito.mock(PaimonPartitionInfoLoader.class), - (nameMapping, schemaId) -> new PaimonSchemaCacheValue( + (nameMapping, schemaId, tableGeneration, retainedTable) -> new PaimonSchemaCacheValue( Collections.emptyList(), Collections.emptyList(), null)); NameMapping nameMapping = new NameMapping(1L, "db", "table", "db", "table"); Mockito.doAnswer(ignored -> new PaimonMvccSnapshot( diff --git a/fe/pom.xml b/fe/pom.xml index 6821b6c3fce2ab..f709024e526e32 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -272,6 +272,7 @@ under the License. 3.1.0 18.3.14-doris-SNAPSHOT 1.49 + 0.17 2.18.0 1.11.0 1.1.1 @@ -1931,6 +1932,11 @@ under the License. mockito-inline ${mockito.version} + + org.openjdk.jol + jol-core + ${jol.version} + it.unimi.dsi fastutil-core diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy index 2e2a2ea8e9b5c9..6079a17ec402ca 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy @@ -28,8 +28,24 @@ suite("test_iceberg_table_meta_cache", "p0,external,doris,external_docker,extern String default_fs = "hdfs://${externalEnvIp}:${hdfs_port}" String warehouse = "${default_fs}/warehouse" - // 1. test default catalog + // DDL validation must reject misspelled memory-governance options. sql """drop catalog if exists ${catalog_name};""" + test { + sql """ + create catalog ${catalog_name} properties ( + 'type'='iceberg', + 'iceberg.catalog.type'='hms', + 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}', + 'fs.defaultFS' = '${default_fs}', + 'warehouse' = '${warehouse}', + 'meta.cache.iceberg.snapshot.max-weigth' = '16MB' + ); + """ + exception "Unknown external meta cache" + } + + // 1. test a catalog-level memory bound without a global bound. The existing + // create/insert/select/refresh flow below is the weighted-cache happy path. sql """ create catalog ${catalog_name} properties ( 'type'='iceberg', @@ -37,6 +53,7 @@ suite("test_iceberg_table_meta_cache", "p0,external,doris,external_docker,extern 'hive.metastore.uris' = 'thrift://${externalEnvIp}:${hmsPort}', 'fs.defaultFS' = '${default_fs}', 'warehouse' = '${warehouse}', + 'meta.cache.max-weight' = '128MB', 'meta.cache.iceberg.manifest.enable' = 'true' ); """ From 6443730e545dd58c4b8e32e7bcdc53bed106f409 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Mon, 17 Aug 2026 20:33:26 +0800 Subject: [PATCH 03/45] [fix](fe) Make external metadata cache size estimators structural and fail closed on SDK layout changes Replace calibrated slope constants in the Iceberg and Paimon cache size estimators with formulas derived from the active JVM object layout and the retained SDK structures. Iceberg: - Account Schema lookup indexes (idToName/nameToId/idToField/lowerCaseNameToId/idToAccessor), root StructType indexes and the StructType.asSchema() secondary graph structurally, including canonical/short-alias path Strings, lower-cased copies per index (Unicode aware), boxed ids and per-level accessors, for flat and nested schemas. - Account PartitionSpec lazy state (fieldList, javaClasses, partitionType, secondary schema) and reserve the O(distinctSources * fields) fieldsBySourceId graph in O(fields) work without materializing it; account SortOrder fields, transforms and identifier field sets. - Add a character budget beside the element budget, refuse lazily-loaded snapshot suppliers, pin the full instance-field layouts of the retained Iceberg classes and only account GenericDataFile/GenericDeleteFile copies with pinned BaseFile/PartitionData layouts. Paimon: - Handle Row/Array/Vector/Map/Multiset explicitly with a leaf-type whitelist, fail closed on unknown DataType implementations, reserve RowType's four lazy lookup maps at admission and pin DataType/DataField/RowType/TableSchema/FileStoreTable layouts. Tests and benchmarks: - JOL oracle cancels JVM-shared boxed caches and Class metadata; add whole-entry, uncached-id, Unicode lower-case, character/element budget, ContentFile implementation, VectorType, unknown DataType and RowType lazy-map fixtures; extend benchmarks with wide partitioned and nested tables. --- .../HivePartitionValuesSizeBenchmark.java | 7 +- .../iceberg/IcebergCacheSizeBenchmark.java | 92 +- .../hive/HiveCacheSizeEstimator.java | 22 +- .../hive/HiveExternalMetaCache.java | 48 +- .../iceberg/IcebergCacheSizeEstimator.java | 1001 ++++++++++++++- .../iceberg/IcebergSnapshotCacheValue.java | 18 +- .../iceberg/IcebergTableCacheValue.java | 6 +- .../iceberg/cache/ManifestCacheValue.java | 228 +++- .../metacache/MetaCacheSizeEstimator.java | 3 +- .../metacache/MetaCacheWeightUtils.java | 300 ++++- .../paimon/PaimonCacheSizeEstimator.java | 323 ++++- .../iceberg/IcebergExternalMetaCacheTest.java | 1121 ++++++++++++++++- .../iceberg/IcebergPartitionInfoTest.java | 8 +- .../EstimatorCalibrationAssertions.java | 65 +- .../metacache/MetaCacheEntryTest.java | 12 + .../paimon/PaimonExternalMetaCacheTest.java | 381 +++++- .../datasource/paimon/PaimonUtilTest.java | 8 +- 17 files changed, 3391 insertions(+), 252 deletions(-) diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/hive/HivePartitionValuesSizeBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/hive/HivePartitionValuesSizeBenchmark.java index 7e3acd9e2f97ac..57a2ebb0e1bdb8 100644 --- a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/hive/HivePartitionValuesSizeBenchmark.java +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/hive/HivePartitionValuesSizeBenchmark.java @@ -31,6 +31,7 @@ import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager.EntryBudget; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import com.google.common.collect.HashBiMap; import com.google.common.collect.Maps; @@ -267,13 +268,13 @@ private static HivePartitionValues createPartitionValues( Map idToItem = Maps.newHashMapWithExpectedSize(count); Map> idToValues = Maps.newHashMapWithExpectedSize(count); String tailPayload = "tail_skew".equals(distribution) ? repeat('x', TAIL_PAYLOAD_BYTES) : null; - long partitionNameCharacterCount = 0L; + long partitionNamePayloadBytes = 0L; for (int i = 0; i < count; i++) { long id = i; String value = i == count - 1 && tailPayload != null ? tailPayload : "value-" + i; String name = "p=" + value; - partitionNameCharacterCount += name.length(); + partitionNamePayloadBytes += MetaCacheWeightUtils.estimatedStringPayloadBytes(name); List rawValues = Collections.singletonList(new PartitionValue(value)); PartitionKey partitionKey = PartitionKey.createListPartitionKeyWithTypes(rawValues, types, true); List keys = new ArrayList<>(1); @@ -284,7 +285,7 @@ private static HivePartitionValues createPartitionValues( idToValues.put(id, new ArrayList<>(Collections.singletonList(value))); } return new HivePartitionValues( - idToItem, nameToId, idToValues, partitionNameCharacterCount, types.size()); + idToItem, nameToId, idToValues, partitionNamePayloadBytes, types.size()); } private static String repeat(char value, int count) { diff --git a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeBenchmark.java b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeBenchmark.java index 2cca17a879b625..77235312739695 100644 --- a/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeBenchmark.java +++ b/fe/fe-benchmark/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeBenchmark.java @@ -110,7 +110,11 @@ public int denseManifestValueConstruction(DenseManifestState state) { for (DataFile file : state.files) { builder.addDataFile(file.copy()); } - return builder.build().getDataFiles().size(); + ManifestCacheValue value = builder.build(); + if (value.isAccountingComplete() != state.expectedAccountingComplete()) { + throw new IllegalStateException("unexpected dense manifest accounting state"); + } + return value.getDataFiles().size(); } public long preparedWeightProvider(PreparedState state) { @@ -150,6 +154,32 @@ public static void main(String[] args) throws Exception { BenchmarkHarness.measure("iceberg.preparedTableCacheHit" + suffix, TimeUnit.MICROSECONDS, () -> benchmark.preparedTableCacheHit(prepared)); } + for (int fieldCount : new int[] {100, 1000}) { + // Wide identity-partitioned specs drive the O(fields) fieldsBySourceId reservation and + // the secondary partition Schema formula; nested schemas drive the per-field path + // and lower-case String terms. Both must stay far below the value construction cost + // of the same table. + String suffix = "[fields=" + fieldCount + "]"; + TablePublicationState partitioned = new TablePublicationState(); + partitioned.fieldCount = fieldCount; + partitioned.identityPartitioned = true; + partitioned.setup(); + BenchmarkHarness.measure("iceberg.partitionedTableValueConstruction" + suffix, + TimeUnit.NANOSECONDS, () -> benchmark.tableValueConstruction(partitioned)); + BenchmarkHarness.measure("iceberg.partitionedTablePayloadCounter" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePayloadCounter(partitioned)); + BenchmarkHarness.measure("iceberg.partitionedTablePublication" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePublication(partitioned)); + + TablePublicationState nested = new TablePublicationState(); + nested.fieldCount = fieldCount; + nested.nestedSchema = true; + nested.setup(); + BenchmarkHarness.measure("iceberg.nestedTablePayloadCounter" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePayloadCounter(nested)); + BenchmarkHarness.measure("iceberg.nestedTablePublication" + suffix, + TimeUnit.MICROSECONDS, () -> benchmark.tablePublication(nested)); + } for (int snapshotCount : new int[] {1000, 10000}) { String suffix = "[snapshots=" + snapshotCount + "]"; LongHistoryTablePublicationState state = new LongHistoryTablePublicationState(); @@ -188,7 +218,9 @@ public static void main(String[] args) throws Exception { state.metricColumns = metricColumns; state.fileCount = fileCount; state.setup(); - String suffix = "[files=" + fileCount + ",metricColumns=" + metricColumns + "]"; + String accounting = state.expectedAccountingComplete() ? "complete" : "rejected"; + String suffix = "[files=" + fileCount + ",metricColumns=" + metricColumns + + ",accounting=" + accounting + "]"; BenchmarkHarness.measure("iceberg.denseManifestReaderBaseline" + suffix, TimeUnit.MICROSECONDS, () -> benchmark.denseManifestReaderBaseline(state)); BenchmarkHarness.measure("iceberg.denseManifestValueConstruction" + suffix, @@ -199,13 +231,15 @@ public static void main(String[] args) throws Exception { public static class TablePublicationState { public int fieldCount; + public boolean identityPartitioned; + public boolean nestedSchema; private NameMapping mapping; private Table table; public void setup() { mapping = NameMapping.createForTest(1L, "benchmark_db", "benchmark_table"); - table = newTable(fieldCount); + table = newTable(fieldCount, identityPartitioned, nestedSchema); } } @@ -286,6 +320,7 @@ public static class DenseManifestState { public int metricColumns; private List files; + private boolean accountingComplete; public void setup() { int poolSize = Math.min(fileCount, 256); @@ -315,33 +350,56 @@ public void setup() { for (int index = 0; index < fileCount; index++) { files.add(filePool.get(index % poolSize)); } + accountingComplete = ManifestCacheValue.forDataFiles(files).isAccountingComplete(); + } + + private boolean expectedAccountingComplete() { + return accountingComplete; } } private static Table newTable(int fieldCount) { + return newTable(fieldCount, false, false); + } + + private static Table newTable(int fieldCount, boolean identityPartitioned, boolean nestedSchema) { List fields = new ArrayList<>(fieldCount); for (int index = 0; index < fieldCount; index++) { fields.add(Types.NestedField.optional(index + 1, "field_" + index, Types.StringType.get())); } - Schema schema = new Schema(fields); - TableMetadata metadata = TableMetadata.newTableMetadata( - schema, PartitionSpec.unpartitioned(), "file:/benchmark/table", Collections.emptyMap()); - InMemoryFileIO fileIO = new InMemoryFileIO(); - StringBuilder snapshotJson = new StringBuilder("{\"snapshot-id\":7,\"timestamp-ms\":1,") - .append("\"summary\":{\"operation\":\"append\"},\"manifests\":["); - for (int index = 0; index < 10; index++) { - if (index > 0) { - snapshotJson.append(','); + Schema schema; + if (nestedSchema) { + List nestedFields = new ArrayList<>(fieldCount); + for (int index = 0; index < fieldCount; index++) { + nestedFields.add(Types.NestedField.optional( + 1000 + index, "Nested_" + index, Types.StringType.get())); } - String manifestPath = "/benchmark/manifest-" + index + ".avro"; - snapshotJson.append('"').append(manifestPath).append('"'); - fileIO.addFile(manifestPath, new byte[0]); + schema = new Schema( + Types.NestedField.optional(1, "payload", Types.StructType.of(nestedFields)), + Types.NestedField.optional(2, "list", Types.ListType.ofOptional(3, + Types.StructType.of(Types.NestedField.optional( + 4, "leaf", Types.StringType.get())))), + Types.NestedField.optional(5, "id", Types.LongType.get())); + } else { + schema = new Schema(fields); } - Snapshot snapshot = SnapshotParser.fromJson(snapshotJson.append("],\"schema-id\":0}").toString()); + PartitionSpec spec = PartitionSpec.unpartitioned(); + if (identityPartitioned) { + PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(schema); + for (Types.NestedField field : schema.columns()) { + specBuilder.identity(field.name()); + } + spec = specBuilder.build(); + } + TableMetadata metadata = TableMetadata.newTableMetadata( + schema, spec, "file:/benchmark/table", Collections.emptyMap()); + Snapshot snapshot = SnapshotParser.fromJson("{\"snapshot-id\":7,\"timestamp-ms\":1," + + "\"summary\":{\"operation\":\"append\"}," + + "\"manifest-list\":\"/benchmark/manifest-list.avro\",\"schema-id\":0}"); metadata = TableMetadata.buildFrom(metadata).setBranchSnapshot(snapshot, SnapshotRef.MAIN_BRANCH) .discardChanges() .withMetadataLocation("file:/benchmark/table/metadata/v1.json").build(); - return new BaseTable(new StaticTableOperations(metadata, fileIO), "benchmark.table"); + return new BaseTable(new StaticTableOperations(metadata, new InMemoryFileIO()), "benchmark.table"); } private static Table newLongHistoryTable(int snapshotCount) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java index c26a021890e880..0d5f8a8343415f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java @@ -24,18 +24,26 @@ /** Constant-time retained-weight formula for Hive partition-value cache entries. */ final class HiveCacheSizeEstimator { - // Calibrated against complete 4.1 object graphs. The per-character reserve covers the - // partition name plus derived value/literal strings and therefore remains skew-sensitive. - private static final long ENTRY_BASE_BYTES = 2L * 1024L; - private static final long PARTITION_BASE_BYTES = 4L * 1024L; - private static final long PARTITION_COLUMN_BYTES = 384L; - private static final long PARTITION_NAME_CHARACTER_BYTES = 8L; + // Calibrated against complete 4.1 object graphs. The payload reserve covers the partition + // name plus derived value/literal strings and therefore remains skew-sensitive. + private static final long ENTRY_BASE_BYTES = objectBytes(2L * 1024L); + private static final long PARTITION_BASE_BYTES = objectBytes(896L); + private static final long PARTITION_COLUMN_BYTES = objectBytes(256L); + // One copy is retained as the partition name and another in the decoded partition values. + private static final long PARTITION_NAME_PAYLOAD_COPIES = 2L; private HiveCacheSizeEstimator() { } + private static long objectBytes(long bytes) { + return MetaCacheWeightUtils.estimatedObjectBytes(bytes); + } + static MetaCacheSizeEstimate estimatePartitionValuesEntry( PartitionValueCacheKey key, HivePartitionValues value) { + if (!MetaCacheWeightUtils.isSupportedJvmObjectLayout()) { + return MetaCacheSizeEstimate.incomplete("unsupported_jvm_object_alignment"); + } long partitionCount = value.getIdToPartitionItem() == null ? 0L : value.getIdToPartitionItem().size(); long perPartitionBytes = MetaCacheWeightUtils.saturatedAdd( @@ -48,7 +56,7 @@ static MetaCacheSizeEstimate estimatePartitionValuesEntry( MetaCacheWeightUtils.saturatedMultiply(partitionCount, perPartitionBytes)); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.saturatedMultiply( - value.getPartitionNameCharacterCount(), PARTITION_NAME_CHARACTER_BYTES)); + value.getPartitionNamePayloadBytes(), PARTITION_NAME_PAYLOAD_COPIES)); return MetaCacheSizeEstimate.complete(bytes); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java index dfa5b00289feb9..1e48fc0604c82f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java @@ -305,12 +305,13 @@ private HivePartitionValues loadPartitionValues(PartitionValueCacheKey key) { Map idToPartitionItem = Maps.newHashMapWithExpectedSize(partitionNames.size()); BiMap partitionNameToIdMap = HashBiMap.create(partitionNames.size()); - long partitionNameCharacterCount = 0L; + long partitionNamePayloadBytes = 0L; String localDbName = nameMapping.getLocalDbName(); String localTblName = nameMapping.getLocalTblName(); for (String partitionName : partitionNames) { - partitionNameCharacterCount = MetaCacheWeightUtils.saturatedAdd( - partitionNameCharacterCount, partitionName.length()); + partitionNamePayloadBytes = MetaCacheWeightUtils.saturatedAdd( + partitionNamePayloadBytes, + MetaCacheWeightUtils.estimatedStringPayloadBytes(partitionName)); long partitionId = Util.genIdByName(catalog.getName(), localDbName, localTblName, partitionName); ListPartitionItem listPartitionItem = toListPartitionItem(partitionName, key.types, catalog.getName()); idToPartitionItem.put(partitionId, listPartitionItem); @@ -320,7 +321,7 @@ private HivePartitionValues loadPartitionValues(PartitionValueCacheKey key) { Map> partitionValuesMap = ListPartitionPrunerV2.getPartitionValuesMap(idToPartitionItem); HivePartitionValues partitionValues = new HivePartitionValues(idToPartitionItem, partitionNameToIdMap, partitionValuesMap, - partitionNameCharacterCount, key.types == null ? 0 : key.types.size()); + partitionNamePayloadBytes, key.types == null ? 0 : key.types.size()); preparePartitionValuesForPublication(partitionValues); return partitionValues; } @@ -768,7 +769,7 @@ private void addPartitionsCache(NameMapping nameMapping, allItems.put(partitionId, item); addedItems.put(partitionId, item); allNames.put(partitionName, partitionId); - copy.addPartitionNameCharacters(partitionName.length()); + copy.addPartitionNamePayload(partitionName); } if (addedItems.isEmpty()) { // Even a replay/no-op event must fence a refresh that started before the event. @@ -839,7 +840,7 @@ private void dropPartitionsCache(ExternalTable dorisTable, } allItems.remove(partitionId); allValues.remove(partitionId); - copy.removePartitionNameCharacters(partitionName.length()); + copy.removePartitionNamePayload(partitionName); changed = true; } if (!changed) { @@ -1117,7 +1118,7 @@ public static class HivePartitionValues { // Prepared once after construction/update; the cache weigher only reads this value. private transient volatile MetaCacheSizeEstimate sizeEstimate; // Maintained while the metadata is already being loaded or updated. Admission only reads it. - private long partitionNameCharacterCount; + private long partitionNamePayloadBytes; private int partitionColumnCount; private transient boolean sortedPartitionRangesPrepared; @@ -1128,19 +1129,19 @@ public HivePartitionValues(Map idToPartitionItem, BiMap partitionNameToIdMap, Map> partitionValuesMap) { this(idToPartitionItem, partitionNameToIdMap, partitionValuesMap, - countPartitionNameCharacters(partitionNameToIdMap), + countPartitionNamePayloadBytes(partitionNameToIdMap), inferPartitionColumnCount(partitionValuesMap)); } HivePartitionValues(Map idToPartitionItem, BiMap partitionNameToIdMap, Map> partitionValuesMap, - long partitionNameCharacterCount, + long partitionNamePayloadBytes, int partitionColumnCount) { this.idToPartitionItem = idToPartitionItem; this.partitionNameToIdMap = partitionNameToIdMap; this.partitionValuesMap = partitionValuesMap; - this.partitionNameCharacterCount = partitionNameCharacterCount; + this.partitionNamePayloadBytes = partitionNamePayloadBytes; this.partitionColumnCount = partitionColumnCount; } @@ -1149,7 +1150,7 @@ HivePartitionValues mutableCopy() { copy.partitionNameToIdMap = partitionNameToIdMap == null ? null : HashBiMap.create(partitionNameToIdMap); copy.idToPartitionItem = idToPartitionItem == null ? null : Maps.newHashMap(idToPartitionItem); copy.partitionValuesMap = partitionValuesMap == null ? null : Maps.newHashMap(partitionValuesMap); - copy.partitionNameCharacterCount = partitionNameCharacterCount; + copy.partitionNamePayloadBytes = partitionNamePayloadBytes; copy.partitionColumnCount = partitionColumnCount; return copy; } @@ -1186,31 +1187,34 @@ void prepareSizeEstimate(PartitionValueCacheKey key) { sizeEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry(key, this); } - long getPartitionNameCharacterCount() { - return partitionNameCharacterCount; + long getPartitionNamePayloadBytes() { + return partitionNamePayloadBytes; } int getPartitionColumnCount() { return partitionColumnCount; } - private void addPartitionNameCharacters(int characters) { - partitionNameCharacterCount = MetaCacheWeightUtils.saturatedAdd( - partitionNameCharacterCount, characters); + private void addPartitionNamePayload(String partitionName) { + partitionNamePayloadBytes = MetaCacheWeightUtils.saturatedAdd( + partitionNamePayloadBytes, + MetaCacheWeightUtils.estimatedStringPayloadBytes(partitionName)); } - private void removePartitionNameCharacters(int characters) { - partitionNameCharacterCount = Math.max(0L, partitionNameCharacterCount - characters); + private void removePartitionNamePayload(String partitionName) { + partitionNamePayloadBytes = Math.max(0L, partitionNamePayloadBytes + - MetaCacheWeightUtils.estimatedStringPayloadBytes(partitionName)); } - private static long countPartitionNameCharacters(BiMap names) { - long characters = 0L; + private static long countPartitionNamePayloadBytes(BiMap names) { + long payloadBytes = 0L; if (names != null) { for (String name : names.keySet()) { - characters = MetaCacheWeightUtils.saturatedAdd(characters, name.length()); + payloadBytes = MetaCacheWeightUtils.saturatedAdd( + payloadBytes, MetaCacheWeightUtils.estimatedStringPayloadBytes(name)); } } - return characters; + return payloadBytes; } private static int inferPartitionColumnCount(Map> values) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java index 8d483ecfb5153f..2b11a71f8c8d00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java @@ -22,54 +22,157 @@ import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; +import org.apache.iceberg.BlobMetadata; import org.apache.iceberg.HasTableOperations; +import org.apache.iceberg.PartitionField; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.PartitionStatisticsFile; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; +import org.apache.iceberg.SortField; import org.apache.iceberg.SortOrder; import org.apache.iceberg.StatisticsFile; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.encryption.EncryptedKey; +import org.apache.iceberg.transforms.Transform; +import org.apache.iceberg.transforms.UnknownTransform; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; +import java.lang.reflect.Field; +import java.lang.reflect.Modifier; import java.nio.ByteBuffer; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Locale; import java.util.Map; +import java.util.Set; /** Publication-time retained-weight formulas for Iceberg cache entries. */ final class IcebergCacheSizeEstimator { - private static final long KEY_BASE_BYTES = 128L; - private static final long TABLE_BASE_BYTES = 16L * 1024L; - private static final long SCHEMA_VERSION_BYTES = 512L; - private static final long SCHEMA_FIELD_BYTES = 512L; - private static final long NESTED_SCHEMA_FIELD_BYTES = 512L; - private static final long PARTITION_SPEC_BYTES = 256L; - private static final long PARTITION_SPEC_FIELD_BYTES = 384L; - private static final long SORT_ORDER_BYTES = 256L; - private static final long SORT_FIELD_BYTES = 256L; - private static final long TABLE_PROPERTY_BYTES = 256L; - private static final long CURRENT_SNAPSHOT_BYTES = 512L; - private static final long HISTORICAL_SNAPSHOT_BYTES = 1024L; - private static final long SNAPSHOT_LOG_ENTRY_BYTES = 64L; - private static final long METADATA_LOG_ENTRY_BYTES = 128L; - private static final long SNAPSHOT_REF_BYTES = 128L; - private static final long STATISTICS_FILE_BYTES = 512L; - private static final long PARTITION_STATISTICS_FILE_BYTES = 256L; - private static final long ENCRYPTED_KEY_BYTES = 256L; - private static final long PARTITION_BYTES = 512L; - private static final long PARTITION_ALIAS_BYTES = 256L; - private static final long NAME_MAPPING_ENTRY_BYTES = 256L; - private static final long MANIFEST_ENTRY_BASE_BYTES = 256L; - private static final long DATA_FILE_BYTES = 16L * 1024L; - private static final long DELETE_FILE_BYTES = 18L * 1024L; - private static final long FILE_METRIC_ENTRY_BYTES = 160L; + // Calibrated against JOL retained-graph deltas in IcebergExternalMetaCacheTest. + // Every metadata element visited (field, type, snapshot, summary entry, ...) costs a few + // reads; the bound only guards against pathological metadata and is far above real tables + // (a 10,000-snapshot history with 15 summary keys each is 160,000 elements). Exceeding it + // rejects weighted admission, so it must not be reachable by ordinary long-lived tables. + private static final long MAX_TABLE_ACCOUNTING_ELEMENTS = 2_000_000L; + // Total name characters the estimator may lower-case while reserving case-insensitive indexes. + private static final long MAX_TABLE_ACCOUNTING_CHARACTERS = 4_000_000L; + private static final int MAX_TYPE_ACCOUNTING_DEPTH = 128; + private static final long KEY_BASE_BYTES = objectBytes(128L); + private static final long TABLE_BASE_BYTES = objectBytes(16L * 1024L); + // TableMetadata-side share of one schema version: schemas list slot and schemasById entry, + // including the growth of both from their singleton to their regular immutable shapes. + private static final long SCHEMA_VERSION_BYTES = objectBytes(128L); + private static final long PARTITION_SPEC_BYTES = objectBytes(256L); + // Exact active-layout sizes of the Iceberg/Guava objects that lazy partition, sort and + // schema state allocates. Iceberg 1.10.1 field layouts are pinned by ICEBERG_LAZY_LAYOUT_SUPPORTED. + private static final long PARTITION_FIELD_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 8L); + private static final long SORT_FIELD_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L); + // Identity/Bucket/Truncate transforms are allocated per parsed field; time transforms are enums. + private static final long TRANSFORM_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + private static final long NESTED_FIELD_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(5L, 5L); + private static final long STRUCT_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(6L, 0L); + private static final long SCHEMA_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(11L, 8L); + private static final long IMMUTABLE_LIST_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + private static final long IMMUTABLE_MAP_KEY_SET_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); + private static final long SINGLETON_IMMUTABLE_SET_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + private static final long REGULAR_IMMUTABLE_SET_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 8L); + private static final long ARRAY_LIST_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 8L); + private static final long HASH_MAP_NODE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L); + private static final long HASH_MAP_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(4L, 16L); + private static final long INTEGER_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L); + private static final long LONG_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 8L); + private static final String TRUNCATE_TRANSFORM_PREFIX = "truncate["; + // Truncate on a decimal source retains a BigInteger width (object plus one-int magnitude). + private static final long TRUNCATE_WIDTH_BYTES = MetaCacheWeightUtils.saturatedAdd( + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 20L), + MetaCacheWeightUtils.estimatedIntArrayBytes(1L)); + private static final long LIST_MULTIMAP_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(9L, 0L); + private static final long CAPTURING_SUPPLIER_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + private static final long POSITION_ACCESSOR_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 4L); + // One WrappedPositionAccessor (1 ref + int) per optional struct ancestor. Required ancestors + // collapse into a single Position2/3Accessor that replaces the inner accessor, which retains + // less than this per-level reservation. + private static final long WRAPPED_ACCESSOR_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 4L); + private static final long LIST_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); + private static final long MAP_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 0L); + private static final long DECIMAL_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 8L); + private static final long FIXED_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L); + private static final long GEOMETRY_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + private static final long GEOGRAPHY_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); + private static final long SORT_ORDER_BYTES = objectBytes(256L); + private static final long TABLE_PROPERTY_BYTES = objectBytes(40L); + private static final long CURRENT_SNAPSHOT_BYTES = objectBytes(512L); + private static final long HISTORICAL_SNAPSHOT_BYTES = objectBytes(176L); + private static final long SNAPSHOT_LOG_ENTRY_BYTES = objectBytes(38L); + private static final long METADATA_LOG_ENTRY_BYTES = objectBytes(128L); + private static final long SNAPSHOT_REF_BYTES = objectBytes(128L); + private static final long STATISTICS_FILE_BYTES = objectBytes(512L); + private static final long BLOB_METADATA_BYTES = objectBytes(128L); + private static final long BLOB_FIELD_BYTES = objectBytes(32L); + private static final long PARTITION_STATISTICS_FILE_BYTES = objectBytes(256L); + private static final long ENCRYPTED_KEY_BYTES = objectBytes(256L); + private static final long PARTITION_BYTES = objectBytes(640L); + private static final long PARTITION_ALIAS_BYTES = objectBytes(256L); + private static final long NAME_MAPPING_ENTRY_BYTES = objectBytes(256L); + private static final long MANIFEST_ENTRY_BASE_BYTES = objectBytes(256L); + private static final long DATA_FILE_BYTES = objectBytes(896L); + private static final long DELETE_FILE_BYTES = objectBytes(1024L); + private static final long FILE_METRIC_ENTRY_BYTES = objectBytes(104L); + private static final String BASE_SNAPSHOT_CLASS_NAME = "org.apache.iceberg.BaseSnapshot"; + private static final Field[] BASE_SNAPSHOT_RETAINED_CACHE_FIELDS = + loadBaseSnapshotRetainedCacheFields(); + // TableMetadata.snapshots()/snapshot(id) load lazily through a catalog supplier + // (REST snapshot-loading-mode=refs). Publication must not perform that IO. + private static final Field TABLE_METADATA_SNAPSHOTS_LOADED_FIELD = + loadTableMetadataField("snapshotsLoaded", boolean.class); + private static final Field TABLE_METADATA_SNAPSHOTS_SUPPLIER_FIELD = + loadTableMetadataField("snapshotsSupplier", null); + // The formulas above are built on the Iceberg 1.10.1 instance-field layouts of the classes a + // cached table retains. Every non-static field is pinned, not only the transient lazy ones: a + // library upgrade that adds a retained reference makes weighted admission fail closed. + private static final boolean ICEBERG_LAZY_LAYOUT_SUPPORTED = checkIcebergLayout(); private IcebergCacheSizeEstimator() { } + private static long objectBytes(long bytes) { + return MetaCacheWeightUtils.estimatedObjectBytes(bytes); + } + static MetaCacheSizeEstimate estimateTableEntry(NameMapping key, IcebergTableCacheValue value) { + MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout(); + if (!layoutSupport.isComplete()) { + return layoutSupport; + } Table table = value.getRetainedIcebergTable(); MetaCacheSizeEstimate support = checkSupportedTable(table); if (!support.isComplete()) { @@ -86,6 +189,10 @@ static MetaCacheSizeEstimate estimateTableEntry(NameMapping key, IcebergTableCac static MetaCacheSizeEstimate estimateSnapshotEntry( IcebergSnapshotEntryKey key, IcebergSnapshotCacheValue value) { + MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout(); + if (!layoutSupport.isComplete()) { + return layoutSupport; + } long bytes = KEY_BASE_BYTES; bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping())); @@ -99,7 +206,7 @@ static MetaCacheSizeEstimate estimateSnapshotEntry( bytes = addCount(bytes, partitionInfo.getNameToIcebergPartition().size(), PARTITION_BYTES); bytes = addCount(bytes, partitionInfo.getNameToIcebergPartitionNames().size(), PARTITION_ALIAS_BYTES); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, partitionInfo.getRetainedPayloadBytes()); - bytes = addCount(bytes, value.getNameMapping().map(java.util.Map::size).orElse(0), + bytes = addCount(bytes, value.getNameMapping().map(Map::size).orElse(0), NAME_MAPPING_ENTRY_BYTES); bytes = MetaCacheWeightUtils.saturatedAdd( bytes, value.getRetainedNameMappingPayloadBytes()); @@ -121,6 +228,10 @@ static MetaCacheSizeEstimate estimateSnapshotEntry( static MetaCacheSizeEstimate estimateManifestEntry( IcebergManifestEntryKey key, ManifestCacheValue value) { + MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout(); + if (!layoutSupport.isComplete()) { + return layoutSupport; + } if (!value.isAccountingComplete()) { return MetaCacheSizeEstimate.incomplete("iceberg_manifest_accounting_incomplete"); } @@ -135,7 +246,16 @@ static MetaCacheSizeEstimate estimateManifestEntry( return MetaCacheSizeEstimate.complete(bytes); } + private static MetaCacheSizeEstimate checkJvmObjectLayout() { + return MetaCacheWeightUtils.isSupportedJvmObjectLayout() + ? MetaCacheSizeEstimate.complete(1L) + : MetaCacheSizeEstimate.incomplete("unsupported_jvm_object_alignment"); + } + private static MetaCacheSizeEstimate checkSupportedTable(Table table) { + if (!ICEBERG_LAZY_LAYOUT_SUPPORTED) { + return MetaCacheSizeEstimate.incomplete("unsupported_iceberg_lazy_layout"); + } if (table == null) { return MetaCacheSizeEstimate.incomplete("missing_iceberg_table"); } @@ -147,6 +267,9 @@ private static MetaCacheSizeEstimate checkSupportedTable(Table table) { if (metadata == null) { return MetaCacheSizeEstimate.incomplete("missing_iceberg_table_metadata"); } + if (!areSnapshotsLoaded(metadata)) { + return MetaCacheSizeEstimate.incomplete("iceberg_snapshots_not_loaded"); + } if (metadata.metadataFileLocation() == null || metadata.metadataFileLocation().isEmpty()) { return MetaCacheSizeEstimate.incomplete("missing_iceberg_metadata_location"); @@ -171,7 +294,11 @@ private static long estimateTable(Table table) { return bytes; } - /** Captures exact historical cardinalities and skew-sensitive payload once before admission. */ + /** + * Fully accounts variable payload with a bounded amount of publication-time work. Only + * already-parsed metadata is read; the SDK state it touches on the way (StructType, ListType + * and MapType fieldList copies, the identifier field set) is small, accounted and O(N). + */ static long retainedTablePayloadBytes(Table table) { if (!(table instanceof HasTableOperations)) { return 0L; @@ -180,54 +307,65 @@ static long retainedTablePayloadBytes(Table table) { if (metadata == null) { return 0L; } + if (!areSnapshotsLoaded(metadata)) { + // snapshots()/refs() would call the catalog's lazy snapshot supplier: fail closed. + throw new IllegalStateException("Iceberg table snapshots are not loaded"); + } long bytes = 0L; - for (Schema schema : metadata.schemas()) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SCHEMA_VERSION_BYTES); - for (Types.NestedField field : schema.columns()) { - bytes = addFieldPayload(bytes, field, false); - } - } + AccountingBudget budget = new AccountingBudget( + MAX_TABLE_ACCOUNTING_ELEMENTS, MAX_TABLE_ACCOUNTING_CHARACTERS); for (PartitionSpec spec : metadata.specs()) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, PARTITION_SPEC_BYTES); - for (org.apache.iceberg.PartitionField field : spec.fields()) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, PARTITION_SPEC_FIELD_BYTES); - bytes = addString(bytes, field.name()); - } + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, partitionSpecBytes(spec, budget)); } for (SortOrder sortOrder : metadata.sortOrders()) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SORT_ORDER_BYTES); - bytes = addCount(bytes, sortOrder.fields().size(), SORT_FIELD_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, sortOrderBytes(sortOrder, budget)); + } + for (Schema schema : metadata.schemas()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaBytes(schema, budget)); } + budget.chargeElements(metadata.properties().size()); for (Map.Entry property : metadata.properties().entrySet()) { bytes = addString(bytes, property.getKey()); bytes = addString(bytes, property.getValue()); } for (Snapshot snapshot : metadata.snapshots()) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, HISTORICAL_SNAPSHOT_BYTES); - bytes = addString(bytes, snapshot.operation()); - bytes = addString(bytes, snapshot.manifestListLocation()); - bytes = addStringMap(bytes, snapshot.summary(), TABLE_PROPERTY_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, snapshotBytes(snapshot, budget)); } + budget.chargeElements(metadata.snapshotLog().size()); bytes = addCount(bytes, metadata.snapshotLog().size(), SNAPSHOT_LOG_ENTRY_BYTES); + budget.chargeElements(metadata.previousFiles().size()); for (TableMetadata.MetadataLogEntry previousFile : metadata.previousFiles()) { bytes = MetaCacheWeightUtils.saturatedAdd(bytes, METADATA_LOG_ENTRY_BYTES); bytes = addString(bytes, previousFile.file()); } + budget.chargeElements(metadata.refs().size()); bytes = addCount(bytes, metadata.refs().size(), SNAPSHOT_REF_BYTES); for (String refName : metadata.refs().keySet()) { bytes = addString(bytes, refName); } + budget.chargeElements(metadata.statisticsFiles().size()); for (StatisticsFile statisticsFile : metadata.statisticsFiles()) { bytes = MetaCacheWeightUtils.saturatedAdd(bytes, STATISTICS_FILE_BYTES); bytes = addString(bytes, statisticsFile.path()); - bytes = addCount(bytes, statisticsFile.blobMetadata().size(), TABLE_PROPERTY_BYTES); + for (BlobMetadata blob : statisticsFile.blobMetadata()) { + budget.chargeElements(MetaCacheWeightUtils.saturatedAdd(1L, + MetaCacheWeightUtils.saturatedAdd( + blob.fields().size(), blob.properties().size()))); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, BLOB_METADATA_BYTES); + bytes = addString(bytes, blob.type()); + bytes = addCount(bytes, blob.fields().size(), BLOB_FIELD_BYTES); + bytes = addStringMap(bytes, blob.properties(), TABLE_PROPERTY_BYTES); + } } + budget.chargeElements(metadata.partitionStatisticsFiles().size()); for (PartitionStatisticsFile statisticsFile : metadata.partitionStatisticsFiles()) { bytes = MetaCacheWeightUtils.saturatedAdd(bytes, PARTITION_STATISTICS_FILE_BYTES); bytes = addString(bytes, statisticsFile.path()); } + budget.chargeElements(metadata.encryptionKeys().size()); for (EncryptedKey encryptedKey : metadata.encryptionKeys()) { + budget.chargeElements(encryptedKey.properties().size()); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ENCRYPTED_KEY_BYTES); bytes = addString(bytes, encryptedKey.keyId()); bytes = addString(bytes, encryptedKey.encryptedById()); @@ -238,34 +376,579 @@ static long retainedTablePayloadBytes(Table table) { return bytes; } + /** + * Account a PartitionSpec together with the lazy state that a normal scan materializes after + * admission: fieldList, javaClasses, partitionType() with its StructType indexes, the secondary + * Schema/Binder graph behind partitionType().asSchema() and fieldsBySourceId. Iceberg 1.10.1 + * allocates one Object[fieldCount] per distinct source id inside fieldsBySourceId, so that + * retained graph is O(distinctSourceIds * fieldCount); it is reserved here in O(fieldCount) + * publication work without materializing any of it. + */ + private static long partitionSpecBytes(PartitionSpec spec, AccountingBudget budget) { + List fields = spec.fields(); + long fieldCount = fields.size(); + budget.chargeElements(MetaCacheWeightUtils.saturatedAdd(1L, fieldCount)); + long bytes = PARTITION_SPEC_BYTES; + if (fieldCount == 0L) { + return bytes; + } + Set distinctSourceIds = new HashSet<>(); + long uncachedSourceIds = 0L; + long uncachedFieldIds = 0L; + long lowerCaseNameBytes = 0L; + for (PartitionField field : fields) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, PARTITION_FIELD_BYTES); + bytes = addTransformPayload(bytes, field.transform()); + bytes = addString(bytes, field.name()); + lowerCaseNameBytes = MetaCacheWeightUtils.saturatedAdd( + lowerCaseNameBytes, generatedLowerCaseNameBytes(field.name(), budget)); + if (isUncachedInteger(field.fieldId())) { + uncachedFieldIds++; + } + if (distinctSourceIds.add(field.sourceId()) && isUncachedInteger(field.sourceId())) { + uncachedSourceIds++; + } + } + // Eager PartitionField[] plus lazy fieldList and javaClasses. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, objectArrayGrowthBytes(fieldCount)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, immutableListBytes(fieldCount)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, objectArrayGrowthBytes(fieldCount)); + // partitionType(): the StructType itself also exists for an unpartitioned spec. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, objectArrayGrowthBytes(fieldCount)); + bytes = addCount(bytes, fieldCount, NESTED_FIELD_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + structTypeIndexBytes(fieldCount, uncachedFieldIds, lowerCaseNameBytes)); + if (!spec.schema().idsToOriginal().isEmpty()) { + // rawPartitionType() rebuilds the struct with original ids. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, STRUCT_TYPE_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedObjectArrayBytes(fieldCount)); + bytes = addCount(bytes, fieldCount, NESTED_FIELD_BYTES); + } + // A partition filter binds against partitionType().asSchema(). + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, secondarySchemaBytes( + SchemaShape.flat(fieldCount, uncachedFieldIds, lowerCaseNameBytes))); + // fieldsBySourceId: HashMap. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, LIST_MULTIMAP_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, CAPTURING_SUPPLIER_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + hashIdMapBytes(distinctSourceIds.size(), uncachedSourceIds)); + return addCount(bytes, distinctSourceIds.size(), + MetaCacheWeightUtils.saturatedAdd(ARRAY_LIST_BYTES, + MetaCacheWeightUtils.estimatedObjectArrayBytes(fieldCount))); + } + + /** Account a SortOrder: SortField[] with per-field transforms plus the lazy fieldList copy. */ + private static long sortOrderBytes(SortOrder sortOrder, AccountingBudget budget) { + long fieldCount = sortOrder.fields().size(); + budget.chargeElements(MetaCacheWeightUtils.saturatedAdd(1L, fieldCount)); + long bytes = SORT_ORDER_BYTES; + if (fieldCount == 0L) { + return bytes; + } + for (SortField field : sortOrder.fields()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SORT_FIELD_BYTES); + bytes = addTransformPayload(bytes, field.transform()); + } + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, objectArrayGrowthBytes(fieldCount)); + return MetaCacheWeightUtils.saturatedAdd(bytes, immutableListBytes(fieldCount)); + } + + /** Transform instance plus the payload only some transforms retain. */ + private static long addTransformPayload(long bytes, Transform transform) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, TRANSFORM_BYTES); + if (transform instanceof UnknownTransform) { + return addString(bytes, transform.toString()); + } + if (transform.toString().startsWith(TRUNCATE_TRANSFORM_PREFIX)) { + // Truncate is package-private; its serialized name is the SPI contract. + return MetaCacheWeightUtils.saturatedAdd(bytes, TRUNCATE_WIDTH_BYTES); + } + return bytes; + } + + /** Lazy StructType indexes: fieldList, fieldsByName, fieldsByLowerCaseName and fieldsById. */ + private static long structTypeIndexBytes( + long fieldCount, long uncachedFieldIds, long lowerCaseNameBytes) { + long bytes = immutableListBytes(fieldCount); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + immutableNameMapBytes(fieldCount, 0L, 0L)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + immutableNameMapBytes(fieldCount, 0L, lowerCaseNameBytes)); + return MetaCacheWeightUtils.saturatedAdd(bytes, + immutableNameMapBytes(fieldCount, uncachedFieldIds, 0L)); + } + + /** + * The Schema created by StructType.asSchema(): its constructor materializes idToName and two + * empty id maps; Binder and projection paths add nameToId, lowerCaseNameToId, idToField and + * idToAccessor; its own StructType copy grows the same lazy indexes as the root struct. + */ + private static long secondarySchemaBytes(SchemaShape shape) { + long bytes = schemaObjectBytes(shape); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaLookupBytes(shape)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaLazyIndexBytes(shape)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, STRUCT_TYPE_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectArrayBytes(shape.topLevelFieldCount)); + return MetaCacheWeightUtils.saturatedAdd(bytes, structTypeIndexBytes( + shape.topLevelFieldCount, shape.uncachedTopLevelFieldIdCount, + shape.topLevelLowerCaseStringBytes)); + } + + /** Schema object, empty identifier int[], the two empty id maps and the eager idToName keySet. */ + private static long schemaObjectBytes(SchemaShape shape) { + long bytes = MetaCacheWeightUtils.saturatedAdd( + SCHEMA_BYTES, MetaCacheWeightUtils.estimatedIntArrayBytes(0L)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, HASH_MAP_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, HASH_MAP_BYTES); + if (shape.fieldCount == 1L) { + return MetaCacheWeightUtils.saturatedAdd(bytes, SINGLETON_IMMUTABLE_SET_BYTES); + } + return shape.fieldCount > 1L + ? MetaCacheWeightUtils.saturatedAdd(bytes, IMMUTABLE_MAP_KEY_SET_BYTES) : bytes; + } + + /** + * idToName (eager in the constructor), nameToId and idToField. Every map boxes uncached ids + * itself; idToName and nameToId each retain their own copy of every nested canonical name and + * nameToId also retains the short aliases. + */ + private static long schemaLookupBytes(SchemaShape shape) { + long bytes = immutableNameMapBytes( + shape.fieldCount, shape.uncachedFieldIdCount, shape.pathStringBytes); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, immutableNameMapBytes( + shape.nameEntryCount, shape.uncachedNameIdCount, + MetaCacheWeightUtils.saturatedAdd( + shape.pathStringBytes, shape.aliasStringBytes))); + return MetaCacheWeightUtils.saturatedAdd(bytes, + hashIdMapBytes(shape.fieldCount, shape.uncachedFieldIdCount)); + } + + /** lowerCaseNameToId and idToAccessor, materialized by case-insensitive lookups and Binder. */ + private static long schemaLazyIndexBytes(SchemaShape shape) { + long bytes = immutableNameMapBytes( + shape.nameEntryCount, shape.uncachedNameIdCount, shape.lowerCaseStringBytes); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + hashIdMapBytes(shape.accessorFieldCount, shape.uncachedAccessorIdCount)); + bytes = addCount(bytes, shape.accessorFieldCount, POSITION_ACCESSOR_BYTES); + return addCount(bytes, shape.wrappedAccessorCount, WRAPPED_ACCESSOR_BYTES); + } + + /** + * One table schema version with every index a normal scan can materialize afterwards. Only + * metadata already parsed is read; nothing lazy is touched, and each field is visited once. + */ + private static long schemaBytes(Schema schema, AccountingBudget budget) { + budget.chargeElements(1L); + SchemaShape shape = new SchemaShape(); + long bytes = SCHEMA_VERSION_BYTES; + for (Types.NestedField field : schema.columns()) { + bytes = addFieldPayload( + bytes, field, PathState.ROOT, FieldKind.STRUCT_FIELD, budget, shape); + } + Set identifierFieldIds = schema.identifierFieldIds(); + budget.chargeElements(identifierFieldIds.size()); + bytes = addIdentifierFieldPayload(bytes, identifierFieldIds); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, shape.typeObjectBytes); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaObjectBytes(shape)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, STRUCT_TYPE_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectArrayBytes(shape.topLevelFieldCount)); + if (shape.fieldCount == 0L) { + // Nothing can be looked up in an empty schema; its indexes stay shared singletons. + return bytes; + } + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaLookupBytes(shape)); + // Future lazy growth: main lookups, root struct indexes and the asSchema() secondary graph. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaLazyIndexBytes(shape)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, structTypeIndexBytes( + shape.topLevelFieldCount, shape.uncachedTopLevelFieldIdCount, + shape.topLevelLowerCaseStringBytes)); + return MetaCacheWeightUtils.saturatedAdd(bytes, secondarySchemaBytes(shape)); + } + + /** ImmutableList.copyOf(array): shared empty, singleton, or a list object plus copied array. */ + private static long immutableListBytes(long elementCount) { + if (elementCount <= 0L) { + return 0L; + } + if (elementCount == 1L) { + return IMMUTABLE_LIST_BYTES; + } + return MetaCacheWeightUtils.saturatedAdd(IMMUTABLE_LIST_BYTES, + MetaCacheWeightUtils.estimatedObjectArrayBytes(elementCount)); + } + + /** Growth of a reference array that replaces an empty array retained by the empty shape. */ + private static long objectArrayGrowthBytes(long elementCount) { + long populated = MetaCacheWeightUtils.estimatedObjectArrayBytes(elementCount); + long empty = MetaCacheWeightUtils.estimatedObjectArrayBytes(0L); + return populated == Long.MAX_VALUE ? populated : populated - empty; + } + + /** Boxed Integer keys outside the JVM Integer cache are retained per lookup map. */ + private static boolean isUncachedInteger(int value) { + return value < -128 || value > 127; + } + + /** + * Retained bytes of one lower-cased copy of a name, or 0 when the name is already lower case + * and the index reuses it. Every case-insensitive index (partition StructType, secondary + * Schema and secondary StructType) allocates its own copy, so callers add this per index. + */ + private static long generatedLowerCaseNameBytes(String name, AccountingBudget budget) { + budget.chargeCharacters(name.length()); + String lowerName = name.toLowerCase(Locale.ROOT); + if (lowerName.equals(name)) { + return 0L; + } + return MetaCacheWeightUtils.estimatedGeneratedStringBytes( + lowerName.length(), MetaCacheWeightUtils.isLatin1String(lowerName)); + } + + private static long hashIdMapBytes(long entryCount, long uncachedIds) { + long bytes = HASH_MAP_BYTES; + if (entryCount <= 0L) { + // HashMap allocates its table on the first put. + return bytes; + } + bytes = addCount(bytes, entryCount, HASH_MAP_NODE_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectArrayBytes( + hashMapCapacity(entryCount))); + return addCount(bytes, uncachedIds, INTEGER_BYTES); + } + + private static long immutableNameMapBytes( + long entryCount, long uncachedIds, long generatedStringBytes) { + long bytes = 0L; + if (entryCount == 1L) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectLayoutBytes(8L, 0L)); + } else if (entryCount > 1L) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectLayoutBytes(6L, 4L)); + bytes = addCount(bytes, entryCount, + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 0L)); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedObjectArrayBytes(entryCount)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectArrayBytes( + immutableMapTableCapacity(entryCount))); + } + bytes = addCount(bytes, uncachedIds, INTEGER_BYTES); + return MetaCacheWeightUtils.saturatedAdd(bytes, generatedStringBytes); + } + + private static long snapshotBytes(Snapshot snapshot, AccountingBudget budget) { + rejectMaterializedSnapshotPayload(snapshot); + Map summary = snapshot.summary(); + budget.chargeElements(MetaCacheWeightUtils.saturatedAdd( + 1L, summary == null ? 0L : summary.size())); + long bytes = HISTORICAL_SNAPSHOT_BYTES; + // The parsed parent id is never inside the Long cache; the row-id fields are boxed too + // and only tiny values would share a cached instance, so each present field is charged. + bytes = addBoxedLong(bytes, snapshot.parentId()); + bytes = addBoxedLong(bytes, snapshot.firstRowId()); + bytes = addBoxedLong(bytes, snapshot.addedRows()); + bytes = addString(bytes, snapshot.operation()); + String manifestListLocation = snapshot.manifestListLocation(); + if (manifestListLocation == null) { + // A snapshot serialized with an inline "manifests" array (legacy writers) retains a + // String[] of manifest locations that is only exposed through ManifestFile wrappers. + // Reject weighted admission instead of doing IO or admitting an underestimate. + throw new IllegalStateException( + "Iceberg snapshot with inline manifest list is unsupported"); + } + bytes = addString(bytes, manifestListLocation); + bytes = addString(bytes, snapshot.keyId()); + return addStringMap(bytes, summary, TABLE_PROPERTY_BYTES); + } + + private static void rejectMaterializedSnapshotPayload(Snapshot snapshot) { + if (!BASE_SNAPSHOT_CLASS_NAME.equals(snapshot.getClass().getName())) { + throw new IllegalStateException( + "Unsupported Iceberg snapshot implementation: " + + snapshot.getClass().getName()); + } + if (BASE_SNAPSHOT_RETAINED_CACHE_FIELDS == null) { + throw new IllegalStateException( + "Iceberg BaseSnapshot retained-cache inspection is unavailable"); + } + try { + // The field list is resolved once per process. Publication only performs a bounded + // number of O(1) reads and never walks a retained manifest/file graph. + for (Field retainedCacheField : BASE_SNAPSHOT_RETAINED_CACHE_FIELDS) { + if (retainedCacheField.get(snapshot) != null) { + throw new IllegalStateException( + "Iceberg snapshot has materialized retained payload: " + + retainedCacheField.getName()); + } + } + } catch (IllegalAccessException e) { + throw new IllegalStateException( + "Cannot inspect Iceberg BaseSnapshot retained payload", e); + } + } + + /** Iceberg marks snapshots loaded at construction unless a lazy supplier was configured. */ + private static boolean areSnapshotsLoaded(TableMetadata metadata) { + if (TABLE_METADATA_SNAPSHOTS_LOADED_FIELD == null + || TABLE_METADATA_SNAPSHOTS_SUPPLIER_FIELD == null) { + return false; + } + try { + return TABLE_METADATA_SNAPSHOTS_SUPPLIER_FIELD.get(metadata) == null + || TABLE_METADATA_SNAPSHOTS_LOADED_FIELD.getBoolean(metadata); + } catch (IllegalAccessException | RuntimeException e) { + return false; + } + } + + private static Field loadTableMetadataField(String name, Class expectedType) { + try { + Field field = TableMetadata.class.getDeclaredField(name); + if ((expectedType != null && field.getType() != expectedType) + || Modifier.isStatic(field.getModifiers())) { + return null; + } + field.setAccessible(true); + return field; + } catch (ReflectiveOperationException | RuntimeException e) { + return null; + } + } + + private static Field[] loadBaseSnapshotRetainedCacheFields() { + try { + Class snapshotClass = Class.forName( + BASE_SNAPSHOT_CLASS_NAME, false, Snapshot.class.getClassLoader()); + List retainedCacheFields = new ArrayList<>(); + for (Field field : snapshotClass.getDeclaredFields()) { + int modifiers = field.getModifiers(); + if (Modifier.isTransient(modifiers) && !Modifier.isStatic(modifiers) + && !field.getType().isPrimitive()) { + field.setAccessible(true); + retainedCacheFields.add(field); + } + } + return retainedCacheFields.isEmpty() + ? null : retainedCacheFields.toArray(new Field[0]); + } catch (ReflectiveOperationException | RuntimeException e) { + return null; + } + } + + private static boolean checkIcebergLayout() { + ClassLoader loader = Snapshot.class.getClassLoader(); + return MetaCacheWeightUtils.hasExpectedInstanceFields(Schema.class, + "struct:StructType", "schemaId:int", "identifierFieldIds:int[]", + "highestFieldId:int", "aliasToId:BiMap", "idToField:Map", "nameToId:Map", + "lowerCaseNameToId:Map", "idToAccessor:Map", "idToName:Map", + "identifierFieldIdSet:Set", "idsToReassigned:Map", "idsToOriginal:Map") + && MetaCacheWeightUtils.hasExpectedInstanceFields(PartitionSpec.class, + "schema:Schema", "specId:int", "fields:PartitionField[]", + "fieldsBySourceId:ListMultimap", "lazyJavaClasses:Class[]", + "lazyPartitionType:StructType", "lazyRawPartitionType:StructType", + "fieldList:List", "lastAssignedFieldId:int") + && MetaCacheWeightUtils.hasExpectedInstanceFields(PartitionField.class, + "sourceId:int", "fieldId:int", "name:String", "transform:Transform") + && MetaCacheWeightUtils.hasExpectedInstanceFields(SortOrder.class, + "schema:Schema", "orderId:int", "fields:SortField[]", "fieldList:List") + && MetaCacheWeightUtils.hasExpectedInstanceFields(SortField.class, + "transform:Transform", "sourceId:int", "direction:SortDirection", + "nullOrder:NullOrder") + && MetaCacheWeightUtils.hasExpectedInstanceFields(Types.StructType.class, + "fields:NestedField[]", "schema:Schema", "fieldList:List", + "fieldsByName:Map", "fieldsByLowerCaseName:Map", "fieldsById:Map") + && MetaCacheWeightUtils.hasExpectedInstanceFields(Types.ListType.class, + "elementField:NestedField", "fields:List") + && MetaCacheWeightUtils.hasExpectedInstanceFields(Types.MapType.class, + "keyField:NestedField", "valueField:NestedField", "fields:List") + && MetaCacheWeightUtils.hasExpectedInstanceFields(Types.NestedField.class, + "isOptional:boolean", "id:int", "name:String", "type:Type", + "doc:String", "initialDefault:Literal", "writeDefault:Literal") + && MetaCacheWeightUtils.hasExpectedInstanceFields(TableMetadata.class, + "metadataFileLocation:String", "formatVersion:int", "uuid:String", + "location:String", "lastSequenceNumber:long", "lastUpdatedMillis:long", + "lastColumnId:int", "currentSchemaId:int", "schemas:List", + "defaultSpecId:int", "specs:List", "lastAssignedPartitionId:int", + "defaultSortOrderId:int", "sortOrders:List", "properties:Map", + "currentSnapshotId:long", "schemasById:Map", "specsById:Map", + "sortOrdersById:Map", "snapshotLog:List", "previousFiles:List", + "statisticsFiles:List", "partitionStatisticsFiles:List", "changes:List", + "nextRowId:long", "encryptionKeys:List", + "snapshotsSupplier:SerializableSupplier", "snapshots:List", + "snapshotsById:Map", "refs:Map", "snapshotsLoaded:boolean") + && MetaCacheWeightUtils.hasExpectedInstanceFields(BASE_SNAPSHOT_CLASS_NAME, loader, + "snapshotId:long", "parentId:Long", "sequenceNumber:long", + "timestampMillis:long", "manifestListLocation:String", + "operation:String", "summary:Map", "schemaId:Integer", + "v1ManifestLocations:String[]", "firstRowId:Long", "addedRows:Long", + "keyId:String", "allManifests:List", "dataManifests:List", + "deleteManifests:List", "addedDataFiles:List", "removedDataFiles:List", + "addedDeleteFiles:List", "removedDeleteFiles:List"); + } + + private static long addBoxedLong(long bytes, Long value) { + return value == null ? bytes : MetaCacheWeightUtils.saturatedAdd(bytes, LONG_BYTES); + } + private static long addBufferPayload(long bytes, ByteBuffer buffer) { return buffer == null ? bytes : MetaCacheWeightUtils.saturatedAdd(bytes, buffer.capacity()); } - private static long addFieldPayload(long bytes, Types.NestedField field, boolean nested) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, - nested ? NESTED_SCHEMA_FIELD_BYTES : SCHEMA_FIELD_BYTES); - bytes = addString(bytes, field.name()); + /** + * Account one NestedField, its owned strings and its type subtree, and record the shape data + * the lookup-map formulas need. Canonical and short names follow Iceberg's IndexByName: a + * nested name joins its ancestors with '.', a struct-typed list element or map value is left + * out of its children's short names (which then become aliases), and every lower-case index + * lower-cases each entry. + */ + private static long addFieldPayload( + long bytes, Types.NestedField field, PathState ancestors, FieldKind kind, + AccountingBudget budget, SchemaShape shape) { + budget.chargeElements(1L); + budget.chargeCharacters(field.name().length()); + String name = field.name(); + String lowerName = name.toLowerCase(Locale.ROOT); + boolean nameLatin1 = MetaCacheWeightUtils.isLatin1String(name); + boolean lowerLatin1 = MetaCacheWeightUtils.isLatin1String(lowerName); + shape.addField(field.fieldId(), ancestors, name, nameLatin1, lowerName, lowerLatin1); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, NESTED_FIELD_BYTES); + if (kind == FieldKind.STRUCT_FIELD) { + // List element and map key/value fields are named by shared "element"/"key"/"value" + // literals inside Iceberg's type constructors. + bytes = addString(bytes, name); + } bytes = addString(bytes, field.doc()); bytes = addDefaultPayload(bytes, field.initialDefault()); bytes = addDefaultPayload(bytes, field.writeDefault()); - return addTypePayload(bytes, field.type()); + boolean pushShortName = kind == FieldKind.STRUCT_FIELD || kind == FieldKind.MAP_KEY + || !field.type().isStructType(); + // Only fields nested through a chain of struct fields get accessors; anything below a + // list or map does not. + boolean structChildren = kind == FieldKind.STRUCT_FIELD && field.type().isStructType(); + PathState children = ancestors.push(name.length(), nameLatin1, lowerName.length(), + lowerLatin1, pushShortName, structChildren); + return addTypePayload(bytes, field.type(), children, budget, shape); } - private static long addTypePayload(long bytes, Type type) { + private static long addTypePayload( + long bytes, Type type, PathState ancestors, AccountingBudget budget, + SchemaShape shape) { + if (ancestors.typeDepth > MAX_TYPE_ACCOUNTING_DEPTH) { + throw new IllegalStateException( + "Iceberg cache accounting type depth exceeded"); + } + budget.chargeElements(1L); if (type.isStructType()) { - for (Types.NestedField field : type.asStructType().fields()) { - bytes = addFieldPayload(bytes, field, true); + List fields = type.asStructType().fields(); + // A nested struct's fieldList is materialized by every visitor. Its own name/id + // lookup indexes and asSchema() are not reserved: read paths resolve nested names + // through the root Schema maps and Binder binds only root and partition structs; + // nested-column DDL runs against a freshly loaded live table, not a cached one. + shape.addTypeObject(MetaCacheWeightUtils.saturatedAdd( + MetaCacheWeightUtils.saturatedAdd(STRUCT_TYPE_BYTES, + MetaCacheWeightUtils.estimatedObjectArrayBytes(fields.size())), + immutableListBytes(fields.size()))); + for (Types.NestedField field : fields) { + bytes = addFieldPayload( + bytes, field, ancestors, FieldKind.STRUCT_FIELD, budget, shape); } } else if (type.isListType()) { - bytes = addTypePayload(bytes, type.asListType().elementType()); + shape.addTypeObject(MetaCacheWeightUtils.saturatedAdd( + LIST_TYPE_BYTES, IMMUTABLE_LIST_BYTES)); + bytes = addFieldPayload(bytes, type.asListType().fields().get(0), + ancestors, FieldKind.LIST_ELEMENT, budget, shape); } else if (type.isMapType()) { - bytes = addTypePayload(bytes, type.asMapType().keyType()); - bytes = addTypePayload(bytes, type.asMapType().valueType()); + shape.addTypeObject(MetaCacheWeightUtils.saturatedAdd( + MetaCacheWeightUtils.saturatedAdd(MAP_TYPE_BYTES, IMMUTABLE_LIST_BYTES), + MetaCacheWeightUtils.estimatedObjectArrayBytes(2L))); + bytes = addFieldPayload(bytes, type.asMapType().fields().get(0), + ancestors, FieldKind.MAP_KEY, budget, shape); + bytes = addFieldPayload(bytes, type.asMapType().fields().get(1), + ancestors, FieldKind.MAP_VALUE, budget, shape); + } else if (type instanceof Types.DecimalType) { + shape.addTypeObject(DECIMAL_TYPE_BYTES); + } else if (type instanceof Types.FixedType) { + shape.addTypeObject(FIXED_TYPE_BYTES); + } else if (type instanceof Types.GeometryType) { + shape.addTypeObject(MetaCacheWeightUtils.saturatedAdd(GEOMETRY_TYPE_BYTES, + MetaCacheWeightUtils.estimatedStringBytes( + ((Types.GeometryType) type).crs()))); + } else if (type instanceof Types.GeographyType) { + shape.addTypeObject(MetaCacheWeightUtils.saturatedAdd(GEOGRAPHY_TYPE_BYTES, + MetaCacheWeightUtils.estimatedStringBytes( + ((Types.GeographyType) type).crs()))); } + // Other primitive types are shared singletons. return bytes; } + /** Account the int[] and lazy ImmutableSet retained by Schema.identifierFieldIds(). */ + private static long addIdentifierFieldPayload(long bytes, Set fieldIds) { + long count = fieldIds.size(); + if (count == 0L) { + return bytes; + } + long uncachedIds = 0L; + for (int fieldId : fieldIds) { + if (isUncachedInteger(fieldId)) { + uncachedIds++; + } + } + // The int[] grows from the empty array of a schema without identifier fields. + long additions = MetaCacheWeightUtils.estimatedIntArrayPayloadBytes(count); + additions = addCount(additions, uncachedIds, INTEGER_BYTES); + if (count == 1L) { + // ImmutableSet.copyOf(one element) is a SingletonImmutableSet. + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedAdd(additions, SINGLETON_IMMUTABLE_SET_BYTES)); + } + // RegularImmutableSet: the set object, its dense elements array and open-addressing table. + // The shared empty set of an identifier-free schema stays reachable through other schemas, + // so nothing is subtracted for it. + additions = MetaCacheWeightUtils.saturatedAdd(additions, REGULAR_IMMUTABLE_SET_BYTES); + additions = MetaCacheWeightUtils.saturatedAdd( + additions, MetaCacheWeightUtils.estimatedObjectArrayBytes(count)); + additions = MetaCacheWeightUtils.saturatedAdd(additions, + MetaCacheWeightUtils.estimatedObjectArrayBytes( + immutableSetTableCapacity(count))); + return MetaCacheWeightUtils.saturatedAdd(bytes, additions); + } + + private static long immutableSetTableCapacity(long size) { + long capacity = 2L; + while (MetaCacheWeightUtils.saturatedMultiply(size, 10L) + > MetaCacheWeightUtils.saturatedMultiply(capacity, 7L)) { + capacity = MetaCacheWeightUtils.saturatedMultiply(capacity, 2L); + if (capacity == Long.MAX_VALUE) { + return capacity; + } + } + return capacity; + } + + private static long hashMapCapacity(long size) { + long capacity = 16L; + while (size > capacity - capacity / 4L) { + capacity = MetaCacheWeightUtils.saturatedMultiply(capacity, 2L); + if (capacity == Long.MAX_VALUE) { + return capacity; + } + } + return capacity; + } + + private static long immutableMapTableCapacity(long size) { + long capacity = Long.highestOneBit(size); + return MetaCacheWeightUtils.saturatedMultiply(size, 5L) + > MetaCacheWeightUtils.saturatedMultiply(capacity, 6L) + ? MetaCacheWeightUtils.saturatedMultiply(capacity, 2L) : capacity; + } + private static long addDefaultPayload(long bytes, Object value) { if (value instanceof CharSequence) { return MetaCacheWeightUtils.saturatedAdd(bytes, @@ -300,4 +983,210 @@ private static long addCount(long bytes, long count, long bytesPerItem) { MetaCacheWeightUtils.saturatedMultiply(count, bytesPerItem)); } + /** + * Hard bound on publication-time estimator work. Elements bound the number of metadata + * objects visited; characters bound the String scanning (lower-casing) performed for + * lazy-index reservations. Exceeding either throws, which estimateSafely turns into an + * incomplete estimate: weighted admission is rejected but the load itself succeeds. + */ + private static final class AccountingBudget { + private long remainingElements; + private long remainingCharacters; + + private AccountingBudget(long elements, long characters) { + this.remainingElements = elements; + this.remainingCharacters = characters; + } + + private void chargeElements(long elements) { + if (elements < 0L || elements > remainingElements) { + throw new IllegalStateException("Iceberg cache accounting work budget exceeded"); + } + remainingElements -= elements; + } + + private void chargeCharacters(long characters) { + if (characters < 0L || characters > remainingCharacters) { + throw new IllegalStateException( + "Iceberg cache accounting character budget exceeded"); + } + remainingCharacters -= characters; + } + } + + private enum FieldKind { + STRUCT_FIELD, LIST_ELEMENT, MAP_KEY, MAP_VALUE + } + + /** + * Immutable name-stack state of a field's ancestors: character counts and Latin-1 coders of + * the joined canonical path, the short-alias path and both lower-cased forms. + */ + private static final class PathState { + private static final PathState ROOT = new PathState( + -1L, true, -1L, true, -1L, true, -1L, true, 0, 0); + + private final long pathCharacters; + private final boolean pathLatin1; + private final long shortPathCharacters; + private final boolean shortPathLatin1; + private final long lowerPathCharacters; + private final boolean lowerPathLatin1; + private final long shortLowerPathCharacters; + private final boolean shortLowerPathLatin1; + // Struct-field ancestors of the next field, or -1 inside a list or map (no accessors). + private final int structDepth; + private final int typeDepth; + + private PathState(long pathCharacters, boolean pathLatin1, long shortPathCharacters, + boolean shortPathLatin1, long lowerPathCharacters, boolean lowerPathLatin1, + long shortLowerPathCharacters, boolean shortLowerPathLatin1, int structDepth, + int typeDepth) { + this.pathCharacters = pathCharacters; + this.pathLatin1 = pathLatin1; + this.shortPathCharacters = shortPathCharacters; + this.shortPathLatin1 = shortPathLatin1; + this.lowerPathCharacters = lowerPathCharacters; + this.lowerPathLatin1 = lowerPathLatin1; + this.shortLowerPathCharacters = shortLowerPathCharacters; + this.shortLowerPathLatin1 = shortLowerPathLatin1; + this.structDepth = structDepth; + this.typeDepth = typeDepth; + } + + private boolean isRoot() { + return pathCharacters < 0L; + } + + private boolean shortPathDiffers() { + return shortPathCharacters != pathCharacters; + } + + /** + * Push a field name for its children; the short name is pushed only when requested and + * accessor depth continues only for the children of a struct-typed struct field. + */ + private PathState push(long nameCharacters, boolean nameLatin1, long lowerCharacters, + boolean lowerLatin1, boolean pushShortName, boolean structChildren) { + return new PathState( + join(pathCharacters, nameCharacters), pathLatin1 && nameLatin1, + pushShortName ? join(shortPathCharacters, nameCharacters) : shortPathCharacters, + pushShortName ? shortPathLatin1 && nameLatin1 : shortPathLatin1, + join(lowerPathCharacters, lowerCharacters), lowerPathLatin1 && lowerLatin1, + pushShortName ? join(shortLowerPathCharacters, lowerCharacters) + : shortLowerPathCharacters, + pushShortName ? shortLowerPathLatin1 && lowerLatin1 : shortLowerPathLatin1, + structChildren && structDepth >= 0 ? structDepth + 1 : -1, typeDepth + 1); + } + + private static long join(long parentCharacters, long nameCharacters) { + return parentCharacters < 0L ? nameCharacters + : MetaCacheWeightUtils.saturatedAdd( + MetaCacheWeightUtils.saturatedAdd(parentCharacters, 1L), + nameCharacters); + } + } + + /** Cardinalities and generated-String bytes that size a schema's lookup indexes. */ + private static final class SchemaShape { + private long fieldCount; + private long topLevelFieldCount; + private long uncachedFieldIdCount; + private long uncachedTopLevelFieldIdCount; + private long nameEntryCount; + private long uncachedNameIdCount; + // One copy each; the formulas add a copy per index that retains it. + private long pathStringBytes; + private long aliasStringBytes; + private long lowerCaseStringBytes; + private long topLevelLowerCaseStringBytes; + private long accessorFieldCount; + private long uncachedAccessorIdCount; + private long wrappedAccessorCount; + private long typeObjectBytes; + + /** A flat struct of {@code fieldCount} top-level fields, as used by partition types. */ + private static SchemaShape flat( + long fieldCount, long uncachedFieldIds, long lowerCaseStringBytes) { + SchemaShape shape = new SchemaShape(); + shape.fieldCount = fieldCount; + shape.topLevelFieldCount = fieldCount; + shape.uncachedFieldIdCount = uncachedFieldIds; + shape.uncachedTopLevelFieldIdCount = uncachedFieldIds; + shape.nameEntryCount = fieldCount; + shape.uncachedNameIdCount = uncachedFieldIds; + shape.lowerCaseStringBytes = lowerCaseStringBytes; + shape.topLevelLowerCaseStringBytes = lowerCaseStringBytes; + shape.accessorFieldCount = fieldCount; + shape.uncachedAccessorIdCount = uncachedFieldIds; + return shape; + } + + private void addField(int fieldId, PathState ancestors, String name, boolean nameLatin1, + String lowerName, boolean lowerLatin1) { + boolean uncached = isUncachedInteger(fieldId); + fieldCount++; + nameEntryCount++; + if (uncached) { + uncachedFieldIdCount++; + uncachedNameIdCount++; + } + if (ancestors.structDepth >= 0) { + accessorFieldCount++; + wrappedAccessorCount = MetaCacheWeightUtils.saturatedAdd( + wrappedAccessorCount, ancestors.structDepth); + if (uncached) { + uncachedAccessorIdCount++; + } + } + if (ancestors.isRoot()) { + topLevelFieldCount++; + if (uncached) { + uncachedTopLevelFieldIdCount++; + } + if (!name.equals(lowerName)) { + // Lower-case indexes only allocate when toLowerCase() changes the name. + long lowerBytes = MetaCacheWeightUtils.estimatedGeneratedStringBytes( + lowerName.length(), lowerLatin1); + lowerCaseStringBytes = MetaCacheWeightUtils.saturatedAdd( + lowerCaseStringBytes, lowerBytes); + topLevelLowerCaseStringBytes = MetaCacheWeightUtils.saturatedAdd( + topLevelLowerCaseStringBytes, lowerBytes); + } + return; + } + // Nested: IndexByName joins a new canonical String, and the lower-case index keeps + // either that joined String or its lower-cased copy. + pathStringBytes = MetaCacheWeightUtils.saturatedAdd(pathStringBytes, + MetaCacheWeightUtils.estimatedGeneratedStringBytes( + PathState.join(ancestors.pathCharacters, name.length()), + ancestors.pathLatin1 && nameLatin1)); + lowerCaseStringBytes = MetaCacheWeightUtils.saturatedAdd(lowerCaseStringBytes, + MetaCacheWeightUtils.estimatedGeneratedStringBytes( + PathState.join(ancestors.lowerPathCharacters, lowerName.length()), + ancestors.lowerPathLatin1 && lowerLatin1)); + if (ancestors.shortPathDiffers()) { + // A short alias exists whenever an ancestor was left out of the short path. + // Iceberg drops an alias that collides with a canonical name; counting the rare + // collision is conservative and avoids building name sets at publication. + nameEntryCount++; + if (uncached) { + uncachedNameIdCount++; + } + aliasStringBytes = MetaCacheWeightUtils.saturatedAdd(aliasStringBytes, + MetaCacheWeightUtils.estimatedGeneratedStringBytes( + PathState.join(ancestors.shortPathCharacters, name.length()), + ancestors.shortPathLatin1 && nameLatin1)); + lowerCaseStringBytes = MetaCacheWeightUtils.saturatedAdd(lowerCaseStringBytes, + MetaCacheWeightUtils.estimatedGeneratedStringBytes( + PathState.join( + ancestors.shortLowerPathCharacters, lowerName.length()), + ancestors.shortLowerPathLatin1 && lowerLatin1)); + } + } + + private void addTypeObject(long bytes) { + typeObjectBytes = MetaCacheWeightUtils.saturatedAdd(typeObjectBytes, bytes); + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index 7cfc5bcc2bcd31..5d3618b21f01fb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -132,12 +132,14 @@ MetaCacheSizeEstimate prepareForCachePublication(IcebergSnapshotEntryKey key) { if (sizeEstimate == null) { sizeEstimate = MetaCacheSizeEstimator.estimateSafely("iceberg_snapshot_preparation_failed", () -> { + // Account before serializing the current snapshot: v1 snapshot JSON + // materializes the transient manifest list that accounting rejects. + retainedTablePayloadBytes = icebergTable + .map(IcebergCacheSizeEstimator::retainedTablePayloadBytes).orElse(0L); if (retainedCurrentSnapshotJson == null) { retainedCurrentSnapshotJson = icebergTable .map(IcebergSnapshotCacheValue::retainCurrentSnapshotJson).orElse(null); } - retainedTablePayloadBytes = icebergTable - .map(IcebergCacheSizeEstimator::retainedTablePayloadBytes).orElse(0L); return IcebergCacheSizeEstimator.estimateSnapshotEntry(key, this); }); if (sizeEstimate.isComplete()) { @@ -414,7 +416,12 @@ private boolean isWriterCompatible(TableMetadata refreshedMetadata) { } } - /** Query-local operations expose exact retained metadata without shared lazy table state. */ + /** + * Query-local read-only operations over the exact retained metadata. Only snapshot state is + * isolated per query (see QueryScopedTable); TableMetadata, Schema, StructType and + * PartitionSpec are shared with the cached generation, and their lazy indexes grow inside the + * cache value. IcebergCacheSizeEstimator reserves that growth at publication. + */ private static final class QueryScopedTableOperations extends RetainedTableOperations { private QueryScopedTableOperations(TableOperations retainedOperations) { super(retainedOperations, retainedOperations.current()); @@ -426,7 +433,10 @@ public void commit(TableMetadata base, TableMetadata metadata) { } } - /** A per-caller view whose Iceberg lazy snapshot state is never written into the cache value. */ + /** + * A per-caller view whose Iceberg lazy snapshot state (manifest lists, manifests, files) is + * never written into the cache value. It does not isolate schema/spec lazy indexes. + */ private static final class QueryScopedTable extends BaseTable { private final QueryScopedTableOperations queryOperations; private final Snapshot currentSnapshot; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java index a4d4f600f66d6e..0b6bd23c634381 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java @@ -56,10 +56,12 @@ synchronized MetaCacheSizeEstimate prepareForCachePublication(NameMapping key) { if (sizeEstimate == null) { sizeEstimate = MetaCacheSizeEstimator.estimateSafely("iceberg_table_preparation_failed", () -> { - retainedCurrentSnapshotJson = - IcebergSnapshotCacheValue.retainCurrentSnapshotJson(icebergTable); + // Order matters: serializing a v1 snapshot materializes its transient + // manifest list, which the payload accounting rejects, so account first. retainedTablePayloadBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes(icebergTable); + retainedCurrentSnapshotJson = + IcebergSnapshotCacheValue.retainCurrentSnapshotJson(icebergTable); return IcebergCacheSizeEstimator.estimateTableEntry(key, this); }); if (sizeEstimate.isComplete()) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java index 4d63af348324e2..cb8923a5bf2e4d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java @@ -23,10 +23,13 @@ import org.apache.iceberg.ContentFile; import org.apache.iceberg.DataFile; import org.apache.iceberg.DeleteFile; +import org.apache.iceberg.PartitionData; import org.apache.iceberg.StructLike; +import org.apache.iceberg.types.Types; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; @@ -34,7 +37,26 @@ * Cached manifest payload containing parsed files. */ public class ManifestCacheValue { - private static final long AUXILIARY_LIST_ENTRY_BYTES = 32L; + private static final long AUXILIARY_LIST_ENTRY_BYTES = + MetaCacheWeightUtils.estimatedObjectBytes(32L); + // A copied PartitionData instance shares its partition type, Avro schema, and serialized + // schema with the other files produced by one manifest reader. These constants are calibrated + // against the production reuseContainers()+file.copy() graph in IcebergExternalMetaCacheTest. + private static final long SHARED_PARTITION_BASE_BYTES = 1320L; + private static final long SHARED_PARTITION_FIELD_BYTES = 904L; + private static final long PARTITION_FIELD_NAME_RETENTION_COPIES = 2L; + // Bound variable-payload traversal (bound entries plus partition values) independently of + // manifest size. Values beyond this limit are rejected from a weighted cache instead of being + // admitted with an underestimate, so it sits well above ordinary wide manifests: 10,000 files + // with 200 lower/upper bounds each is 4,000,000 elements. + private static final long MAX_DEEP_ACCOUNTING_ELEMENTS = 8_000_000L; + // The per-file constants in IcebergCacheSizeEstimator describe the Iceberg 1.10.1 copies that + // ManifestReader + ContentFile.copy() produce. Only those implementations, with their pinned + // instance-field layouts, are accounted; anything else fails closed at build time. + private static final String GENERIC_DATA_FILE_CLASS_NAME = "org.apache.iceberg.GenericDataFile"; + private static final String GENERIC_DELETE_FILE_CLASS_NAME = + "org.apache.iceberg.GenericDeleteFile"; + private static final boolean CONTENT_FILE_LAYOUT_SUPPORTED = checkContentFileLayout(); private final List dataFiles; private final List deleteFiles; @@ -86,6 +108,31 @@ public static Builder deleteFilesBuilder(boolean accountRetainedSize) { return new Builder(false, accountRetainedSize); } + private static boolean checkContentFileLayout() { + ClassLoader loader = ContentFile.class.getClassLoader(); + return MetaCacheWeightUtils.hasExpectedInstanceFields(GENERIC_DATA_FILE_CLASS_NAME, loader) + && MetaCacheWeightUtils.hasExpectedInstanceFields( + GENERIC_DELETE_FILE_CLASS_NAME, loader) + && MetaCacheWeightUtils.hasExpectedInstanceFields( + "org.apache.iceberg.BaseFile", loader, + "partitionType:StructType", "fileOrdinal:Long", "manifestLocation:String", + "partitionSpecId:int", "content:FileContent", "filePath:String", + "format:FileFormat", "partitionData:PartitionData", "recordCount:Long", + "fileSizeInBytes:long", "dataSequenceNumber:Long", + "fileSequenceNumber:Long", "columnSizes:Map", "valueCounts:Map", + "nullValueCounts:Map", "nanValueCounts:Map", "lowerBounds:Map", + "upperBounds:Map", "splitOffsets:long[]", "equalityIds:int[]", + "keyMetadata:byte[]", "sortOrderId:Integer", "firstRowId:Long", + "referencedDataFile:String", "contentOffset:Long", + "contentSizeInBytes:Long", "avroSchema:Schema") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + "org.apache.iceberg.avro.SupportsIndexProjection", loader, + "fromProjectionPos:int[]") + && MetaCacheWeightUtils.hasExpectedInstanceFields(PartitionData.class, + "partitionType:StructType", "size:int", "data:Object[]", + "stringSchema:String", "schema:Schema"); + } + public List getDataFiles() { return dataFiles; } @@ -110,7 +157,7 @@ public boolean isAccountingComplete() { return accountingComplete; } - /** Accumulates retained-size counters in the manifest reader's existing file loop. */ + /** Accounts retained payload while the manifest reader builds the cached lists. */ public static final class Builder { private final boolean dataContent; private final boolean accountRetainedSize; @@ -118,7 +165,10 @@ public static final class Builder { private final List deleteFiles = new ArrayList<>(); private long metricEntryCount; private long retainedPayloadBytes; + private long deepAccountingElements; private boolean accountingComplete; + private final IdentityHashMap> + accountedPartitionSchemas = new IdentityHashMap<>(); private Builder(boolean dataContent, boolean accountRetainedSize) { this.dataContent = dataContent; @@ -131,7 +181,7 @@ public void addDataFile(DataFile file) { throw new IllegalStateException("delete manifest builder cannot accept a data file"); } dataFiles.add(file); - accountSafely(file); + recordAccounting(file); } public void addDeleteFile(DeleteFile file) { @@ -139,7 +189,7 @@ public void addDeleteFile(DeleteFile file) { throw new IllegalStateException("data manifest builder cannot accept a delete file"); } deleteFiles.add(file); - accountSafely(file); + recordAccounting(file); } public ManifestCacheValue build() { @@ -149,27 +199,106 @@ public ManifestCacheValue build() { retainedPayloadBytes, accountingComplete); } - private void accountSafely(ContentFile file) { + private void recordAccounting(ContentFile file) { if (!accountRetainedSize || !accountingComplete) { return; } try { - account(file); - } catch (RuntimeException e) { + requireSupportedContentFile(file); + StructLike partition = file.partition(); + long nextDeepElements = MetaCacheWeightUtils.saturatedAdd( + deepAccountingElements, deepAccountingElements(file, partition)); + if (nextDeepElements > MAX_DEEP_ACCOUNTING_ELEMENTS) { + rejectAccounting(); + return; + } + deepAccountingElements = nextDeepElements; + addAccounting(account(file, partition)); + accountPartitionOwnership(partition); + } catch (RuntimeException | LinkageError e) { // A new or third-party ContentFile implementation must not turn optional cache // accounting into a manifest-read failure. Keep the files for the current query // and mark the value incomplete so weighted admission rejects it. - metricEntryCount = 0L; - retainedPayloadBytes = 0L; - accountingComplete = false; + rejectAccounting(); + } + } + + private void requireSupportedContentFile(ContentFile file) { + if (!CONTENT_FILE_LAYOUT_SUPPORTED) { + throw new IllegalStateException("unsupported Iceberg content file layout"); + } + String expectedClassName = dataContent + ? GENERIC_DATA_FILE_CLASS_NAME : GENERIC_DELETE_FILE_CLASS_NAME; + if (file == null || !expectedClassName.equals(file.getClass().getName())) { + throw new IllegalStateException("unsupported Iceberg content file implementation: " + + (file == null ? "null" : file.getClass().getName())); } } - private void account(ContentFile file) { + private void addAccounting(FileAccounting accounting) { metricEntryCount = MetaCacheWeightUtils.saturatedAdd( - metricEntryCount, metricEntryCount(file)); + metricEntryCount, accounting.metricEntryCount); retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( - retainedPayloadBytes, retainedPayloadBytes(file)); + retainedPayloadBytes, accounting.retainedPayloadBytes); + } + + private void rejectAccounting() { + metricEntryCount = 0L; + retainedPayloadBytes = 0L; + deepAccountingElements = 0L; + accountingComplete = false; + } + + private void accountPartitionOwnership(StructLike partition) { + if (partition == null || partition.size() == 0) { + return; + } + if (!(partition instanceof PartitionData)) { + throw new IllegalArgumentException( + "unsupported Iceberg partition container: " + + partition.getClass().getName()); + } + PartitionData partitionData = (PartitionData) partition; + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, partitionInstanceBytes(partitionData)); + Object partitionTypeIdentity = partitionData.getPartitionType(); + // A reader-produced PartitionData carries its Avro schema; only a Java-deserialized + // instance would rebuild it here (CPU only, no IO). + Object schemaIdentity = partitionData.getSchema(); + IdentityHashMap schemas = accountedPartitionSchemas.computeIfAbsent( + partitionTypeIdentity, ignored -> new IdentityHashMap<>()); + if (schemas.put(schemaIdentity, Boolean.TRUE) == null) { + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, + sharedPartitionBytes(partitionData.getPartitionType())); + } + } + } + + private static FileAccounting account(ContentFile file, StructLike partition) { + return new FileAccounting( + metricEntryCount(file), retainedPayloadBytes(file, partition)); + } + + private static long deepAccountingElements(ContentFile file, StructLike partition) { + long elements = MetaCacheWeightUtils.saturatedAdd( + mapSize(file.lowerBounds()), mapSize(file.upperBounds())); + if (partition != null) { + if (partition.size() < 0) { + throw new IllegalArgumentException("negative Iceberg partition size"); + } + elements = MetaCacheWeightUtils.saturatedAdd(elements, partition.size()); + } + return elements; + } + + private static final class FileAccounting { + private final long metricEntryCount; + private final long retainedPayloadBytes; + + private FileAccounting(long metricEntryCount, long retainedPayloadBytes) { + this.metricEntryCount = metricEntryCount; + this.retainedPayloadBytes = retainedPayloadBytes; } } @@ -182,7 +311,7 @@ private static long metricEntryCount(ContentFile file) { return MetaCacheWeightUtils.saturatedAdd(count, mapSize(file.upperBounds())); } - private static long retainedPayloadBytes(ContentFile file) { + private static long retainedPayloadBytes(ContentFile file, StructLike partition) { long bytes = MetaCacheWeightUtils.estimatedCharSequenceBytes(file.path()); bytes = addBuffer(bytes, file.keyMetadata()); bytes = addBuffers(bytes, file.lowerBounds()); @@ -194,7 +323,7 @@ private static long retainedPayloadBytes(ContentFile file) { MetaCacheWeightUtils.estimatedStringBytes( ((DeleteFile) file).referencedDataFile())); } - return addPartitionPayload(bytes, file.partition()); + return addPartitionPayload(bytes, partition); } private static long addBuffers(long bytes, Map buffers) { @@ -224,21 +353,62 @@ private static long addPartitionPayload(long bytes, StructLike partition) { if (partition == null) { return bytes; } - try { - for (int index = 0; index < partition.size(); index++) { - Object value = partition.get(index, Object.class); - if (value instanceof CharSequence) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, - MetaCacheWeightUtils.estimatedCharSequenceBytes((CharSequence) value)); - } else if (value instanceof ByteBuffer) { - bytes = addBuffer(bytes, (ByteBuffer) value); - } else if (value instanceof byte[]) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ((byte[]) value).length); - } + for (int index = 0; index < partition.size(); index++) { + Object value = partition.get(index, Object.class); + if (value instanceof CharSequence) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedCharSequenceBytes((CharSequence) value)); + } else if (value instanceof ByteBuffer) { + bytes = addByteArray(bytes, ((ByteBuffer) value).capacity()); + } else if (value instanceof byte[]) { + bytes = addByteArray(bytes, ((byte[]) value).length); + } else if (value instanceof java.math.BigDecimal) { + int bits = ((java.math.BigDecimal) value).unscaledValue().bitLength(); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectBytes(96L)); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedIntArrayPayloadBytes( + (bits + 31L) / 32L)); + } else if (value instanceof java.util.UUID) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectBytes(32L)); + } else if (value instanceof Long || value instanceof Double) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectBytes(24L)); + } else if (value instanceof Number || value instanceof Boolean + || value instanceof Character) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedObjectBytes(16L)); + } else if (value != null) { + throw new IllegalArgumentException( + "unsupported Iceberg partition value: " + value.getClass().getName()); } - } catch (RuntimeException ignored) { - // A third-party StructLike may reject Object.class. The fixed per-file allowance - // remains conservative, and cache accounting must never fail manifest loading. + } + return bytes; + } + + private static long addByteArray(long bytes, long payloadBytes) { + return MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedByteArrayBytes(payloadBytes)); + } + + private static long partitionInstanceBytes(PartitionData partition) { + return MetaCacheWeightUtils.saturatedAdd( + MetaCacheWeightUtils.estimatedObjectBytes(32L), + MetaCacheWeightUtils.estimatedObjectArrayBytes(partition.size())); + } + + private static long sharedPartitionBytes(Types.StructType partitionType) { + long rawBytes = MetaCacheWeightUtils.saturatedAdd( + SHARED_PARTITION_BASE_BYTES, + MetaCacheWeightUtils.saturatedMultiply( + partitionType.fields().size(), SHARED_PARTITION_FIELD_BYTES)); + long bytes = MetaCacheWeightUtils.estimatedObjectBytes(rawBytes); + for (Types.NestedField field : partitionType.fields()) { + long payloadBytes = MetaCacheWeightUtils.estimatedStringPayloadBytes(field.name()); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.saturatedMultiply( + payloadBytes, PARTITION_FIELD_NAME_RETENTION_COPIES)); } return bytes; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java index 5aff6a2a1a279a..e74c6e631b69c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheSizeEstimator.java @@ -40,7 +40,8 @@ static MetaCacheSizeEstimate estimateSafely( Objects.requireNonNull(estimation, "estimation"); try { return Objects.requireNonNull(estimation.get(), "size estimate"); - } catch (RuntimeException e) { + } catch (RuntimeException | LinkageError e) { + // A missing or incompatible SDK class must reject weighted admission, not the load. return MetaCacheSizeEstimate.incomplete(failureReason + ":" + e.getClass().getName()); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java index 7869ac5993f48e..828bc28719d4d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java @@ -19,27 +19,207 @@ import org.apache.doris.datasource.NameMapping; -/** Constant-time helpers for conservative external metadata cache weights. */ +import java.lang.invoke.MethodHandle; +import java.lang.invoke.MethodHandles; +import java.lang.invoke.MethodType; +import java.lang.management.ManagementFactory; +import java.lang.management.PlatformManagedObject; +import java.lang.reflect.Field; +import java.lang.reflect.Method; +import java.lang.reflect.Modifier; +import java.util.Arrays; +import java.util.HashSet; +import java.util.Set; + +/** Overflow-safe helpers for conservative external metadata cache weights. */ public final class MetaCacheWeightUtils { - private static final long STRING_BASE_BYTES = 40L; - private static final long STRING_BYTES_PER_CHARACTER = 2L; private static final long NAME_MAPPING_BASE_BYTES = 64L; + private static final MethodHandle STRING_VALUE_GETTER; + private static final long STRING_VALUE_OFFSET; + private static final int OBJECT_ALIGNMENT_BYTES; + private static final int OBJECT_REFERENCE_BYTES; + private static final int OBJECT_HEADER_BYTES; + private static final int OBJECT_ARRAY_BASE_BYTES; + private static final int BYTE_ARRAY_BASE_BYTES; + private static final int CHAR_ARRAY_BASE_BYTES; + private static final int INT_ARRAY_BASE_BYTES; + private static final boolean SUPPORTED_OBJECT_LAYOUT; + private static final long OBJECT_LAYOUT_PERCENT; + + static { + MethodHandle stringValueGetter = null; + long stringValueOffset = -1L; + int referenceBytes = Long.BYTES; + // 24B is the safe fallback for an uncompressed class pointer. Unsafe replaces these + // values with the exact active-VM layout when access is available. + int objectArrayBaseBytes = 24; + int byteArrayBaseBytes = 24; + int charArrayBaseBytes = 24; + int intArrayBaseBytes = 24; + try { + Class unsafeClass = Class.forName("sun.misc.Unsafe"); + Field unsafeField = unsafeClass.getDeclaredField("theUnsafe"); + unsafeField.setAccessible(true); + Object unsafe = unsafeField.get(null); + stringValueOffset = (long) unsafeClass + .getMethod("objectFieldOffset", Field.class) + .invoke(unsafe, String.class.getDeclaredField("value")); + stringValueGetter = MethodHandles.lookup() + .unreflect(unsafeClass.getMethod("getObject", Object.class, long.class)) + .bindTo(unsafe) + .asType(MethodType.methodType(Object.class, Object.class, long.class)); + referenceBytes = (int) unsafeClass + .getMethod("arrayIndexScale", Class.class) + .invoke(unsafe, Object[].class); + objectArrayBaseBytes = (int) unsafeClass + .getMethod("arrayBaseOffset", Class.class) + .invoke(unsafe, Object[].class); + byteArrayBaseBytes = (int) unsafeClass + .getMethod("arrayBaseOffset", Class.class) + .invoke(unsafe, byte[].class); + charArrayBaseBytes = (int) unsafeClass + .getMethod("arrayBaseOffset", Class.class) + .invoke(unsafe, char[].class); + intArrayBaseBytes = (int) unsafeClass + .getMethod("arrayBaseOffset", Class.class) + .invoke(unsafe, int[].class); + } catch (ReflectiveOperationException | RuntimeException ignored) { + // A conservative UTF-16 fallback is used when the VM hides String storage. + } + STRING_VALUE_GETTER = stringValueGetter; + STRING_VALUE_OFFSET = stringValueOffset; + OBJECT_REFERENCE_BYTES = referenceBytes; + OBJECT_ARRAY_BASE_BYTES = objectArrayBaseBytes; + BYTE_ARRAY_BASE_BYTES = byteArrayBaseBytes; + CHAR_ARRAY_BASE_BYTES = charArrayBaseBytes; + INT_ARRAY_BASE_BYTES = intArrayBaseBytes; + String alignmentOption = readVmOption("ObjectAlignmentInBytes"); + int objectAlignmentBytes = parseObjectAlignment(alignmentOption); + OBJECT_ALIGNMENT_BYTES = objectAlignmentBytes; + SUPPORTED_OBJECT_LAYOUT = alignmentOption != null + && (objectAlignmentBytes == 8 || objectAlignmentBytes == 16); + boolean compressedClassPointers = readBooleanVmOption( + "UseCompressedClassPointers", false); + OBJECT_HEADER_BYTES = Long.BYTES + + (compressedClassPointers ? Integer.BYTES : Long.BYTES); + long referencePercent = referenceBytes <= Integer.BYTES ? 100L : 145L; + long classPointerPercent = compressedClassPointers ? 100L : 140L; + long alignmentPercent = objectAlignmentBytes <= 8 ? 100L : 120L; + OBJECT_LAYOUT_PERCENT = (referencePercent * classPointerPercent * alignmentPercent + + 9_999L) / 10_000L; + } private MetaCacheWeightUtils() { } - /** - * Estimate a String without inspecting its contents. Two bytes per character deliberately - * avoids depending on CompactStrings or VM-private layout details. - */ public static long estimatedStringBytes(String value) { - return estimatedCharSequenceBytes(value); + if (value == null) { + return 0L; + } + Object storage = stringStorage(value); + long backingArrayBytes; + if (storage instanceof byte[]) { + backingArrayBytes = estimatedByteArrayBytes(((byte[]) storage).length); + } else if (storage instanceof char[]) { + backingArrayBytes = alignedArrayBytes( + CHAR_ARRAY_BASE_BYTES, ((char[]) storage).length, Character.BYTES); + } else { + backingArrayBytes = alignedArrayBytes( + CHAR_ARRAY_BASE_BYTES, value.length(), Character.BYTES); + } + return saturatedAdd(estimatedObjectLayoutBytes(1L, 6L), backingArrayBytes); } /** Estimate retained character data without materializing a String copy. */ public static long estimatedCharSequenceBytes(CharSequence value) { - return value == null ? 0L : saturatedAdd( - STRING_BASE_BYTES, saturatedMultiply(value.length(), STRING_BYTES_PER_CHARACTER)); + if (value == null) { + return 0L; + } + if (value instanceof String) { + return estimatedStringBytes((String) value); + } + return saturatedAdd(estimatedObjectLayoutBytes(1L, 6L), + alignedArrayBytes(CHAR_ARRAY_BASE_BYTES, value.length(), Character.BYTES)); + } + + /** Whether calibrated formulas support the active VM object alignment. */ + public static boolean isSupportedJvmObjectLayout() { + return SUPPORTED_OBJECT_LAYOUT; + } + + /** Adjust a default compressed-reference object-graph constant to the active VM layout. */ + public static long estimatedObjectBytes(long compressedReferenceBytes) { + long product = saturatedMultiply(compressedReferenceBytes, OBJECT_LAYOUT_PERCENT); + if (product == Long.MAX_VALUE) { + return product; + } + long roundedProduct = saturatedAdd(product, 99L); + return roundedProduct == Long.MAX_VALUE ? roundedProduct : roundedProduct / 100L; + } + + /** Returns the actual backing-array payload in O(1), or a conservative UTF-16 fallback. */ + public static long estimatedStringPayloadBytes(String value) { + if (value == null) { + return 0L; + } + Object storage = stringStorage(value); + if (storage instanceof byte[]) { + return alignPayload(((byte[]) storage).length); + } + if (storage instanceof char[]) { + return alignPayload(saturatedMultiply( + ((char[]) storage).length, Character.BYTES)); + } + return alignPayload(saturatedMultiply(value.length(), Character.BYTES)); + } + + /** Estimate a generated String whose encoded width is derived from its source components. */ + public static long estimatedGeneratedStringBytes(long characterCount, boolean latin1) { + long payloadBytes = saturatedMultiply(characterCount, latin1 ? 1L : Character.BYTES); + return saturatedAdd(estimatedObjectLayoutBytes(1L, 6L), + estimatedByteArrayBytes(payloadBytes)); + } + + /** Whether this VM stores the String with one byte per character. */ + public static boolean isLatin1String(String value) { + if (value == null || value.isEmpty()) { + return true; + } + Object storage = stringStorage(value); + return storage instanceof byte[] && ((byte[]) storage).length == value.length(); + } + + /** VM-layout size of a retained byte array, conservatively if VM introspection is hidden. */ + public static long estimatedByteArrayBytes(long length) { + return alignedArrayBytes(BYTE_ARRAY_BASE_BYTES, length, Byte.BYTES); + } + + /** VM-layout size of an object-reference array, conservatively if introspection is hidden. */ + public static long estimatedObjectArrayBytes(long length) { + return alignedArrayBytes(OBJECT_ARRAY_BASE_BYTES, length, OBJECT_REFERENCE_BYTES); + } + + /** Size of an object with a known field layout on the active VM. */ + public static long estimatedObjectLayoutBytes(long referenceFields, long primitiveBytes) { + if (referenceFields < 0L || primitiveBytes < 0L) { + return Long.MAX_VALUE; + } + long bytes = saturatedAdd( + OBJECT_HEADER_BYTES, + saturatedMultiply(referenceFields, OBJECT_REFERENCE_BYTES)); + return alignPayload(saturatedAdd(bytes, primitiveBytes)); + } + + /** VM-layout size of a retained int array, conservatively if introspection is hidden. */ + public static long estimatedIntArrayBytes(long length) { + return alignedArrayBytes(INT_ARRAY_BASE_BYTES, length, Integer.BYTES); + } + + /** Incremental VM-layout payload of an int array whose header is accounted elsewhere. */ + public static long estimatedIntArrayPayloadBytes(long length) { + long populated = alignedArrayBytes(INT_ARRAY_BASE_BYTES, length, Integer.BYTES); + long empty = alignPayload(INT_ARRAY_BASE_BYTES); + return populated == Long.MAX_VALUE ? populated : populated - empty; } /** Estimate the fixed set of names retained by a cache key. */ @@ -47,13 +227,49 @@ public static long estimatedNameMappingBytes(NameMapping nameMapping) { if (nameMapping == null) { return 0L; } - long bytes = NAME_MAPPING_BASE_BYTES; + long bytes = estimatedObjectBytes(NAME_MAPPING_BASE_BYTES); bytes = saturatedAdd(bytes, estimatedStringBytes(nameMapping.getLocalDbName())); bytes = saturatedAdd(bytes, estimatedStringBytes(nameMapping.getLocalTblName())); bytes = saturatedAdd(bytes, estimatedStringBytes(nameMapping.getRemoteDbName())); return saturatedAdd(bytes, estimatedStringBytes(nameMapping.getRemoteTblName())); } + /** + * Whether {@code type} itself declares exactly the expected non-static instance fields, each + * written as {@code name:SimpleTypeName}. Estimator formulas are calibrated against pinned SDK + * layouts; callers fail closed when a library upgrade adds, removes or retypes a field so a + * new retained reference cannot be silently undercounted. Superclasses are pinned separately. + */ + public static boolean hasExpectedInstanceFields(Class type, String... expectedFields) { + if (type == null) { + return false; + } + Set expected = new HashSet<>(Arrays.asList(expectedFields)); + Set actual = new HashSet<>(); + try { + for (Field field : type.getDeclaredFields()) { + if (Modifier.isStatic(field.getModifiers()) || field.isSynthetic()) { + continue; + } + actual.add(field.getName() + ":" + field.getType().getSimpleName()); + } + } catch (RuntimeException | LinkageError e) { + return false; + } + return actual.equals(expected); + } + + /** Same as {@link #hasExpectedInstanceFields(Class, String...)} for a class resolved by name. */ + public static boolean hasExpectedInstanceFields( + String className, ClassLoader loader, String... expectedFields) { + try { + return hasExpectedInstanceFields( + Class.forName(className, false, loader), expectedFields); + } catch (ReflectiveOperationException | RuntimeException | LinkageError e) { + return false; + } + } + public static long saturatedAdd(long left, long right) { if (left < 0L || right < 0L || Long.MAX_VALUE - left < right) { return Long.MAX_VALUE; @@ -67,4 +283,66 @@ public static long saturatedMultiply(long left, long right) { } return left * right; } + + private static boolean readBooleanVmOption(String option, boolean fallback) { + String value = readVmOption(option); + return value == null ? fallback : Boolean.parseBoolean(value); + } + + private static String readVmOption(String option) { + try { + @SuppressWarnings("unchecked") + Class beanClass = + (Class) + Class.forName("com.sun.management.HotSpotDiagnosticMXBean"); + Object bean = ManagementFactory.getPlatformMXBean(beanClass); + Method getVmOption = beanClass.getMethod("getVMOption", String.class); + Object vmOption = getVmOption.invoke(bean, option); + Method getValue = vmOption.getClass().getMethod("getValue"); + return (String) getValue.invoke(vmOption); + } catch (ReflectiveOperationException | RuntimeException ignored) { + return null; + } + } + + private static long alignPayload(long bytes) { + if (bytes == Long.MAX_VALUE) { + return bytes; + } + long remainder = bytes % OBJECT_ALIGNMENT_BYTES; + return remainder == 0L ? bytes + : saturatedAdd(bytes, OBJECT_ALIGNMENT_BYTES - remainder); + } + + private static long alignedArrayBytes(long baseBytes, long length, long elementBytes) { + if (length < 0L) { + return Long.MAX_VALUE; + } + return alignPayload(saturatedAdd( + baseBytes, saturatedMultiply(length, elementBytes))); + } + + private static Object stringStorage(String value) { + if (STRING_VALUE_GETTER != null && STRING_VALUE_OFFSET >= 0L) { + try { + return (Object) STRING_VALUE_GETTER.invokeExact( + (Object) value, STRING_VALUE_OFFSET); + } catch (Throwable ignored) { + // Return null so callers use the conservative UTF-16 fallback. + } + } + return null; + } + + private static int parseObjectAlignment(String value) { + if (value == null) { + return 16; + } + try { + int alignment = Integer.parseInt(value); + return alignment > 0 ? alignment : 16; + } catch (NumberFormatException ignored) { + return 16; + } + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java index b5a6178538a9c4..8484c4aa477354 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java @@ -20,39 +20,182 @@ import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; +import com.google.common.collect.ImmutableMap; import org.apache.paimon.privilege.PrivilegedFileStoreTable; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.FallbackReadFileStoreTable; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; import org.apache.paimon.types.ArrayType; +import org.apache.paimon.types.BigIntType; +import org.apache.paimon.types.BinaryType; +import org.apache.paimon.types.BlobType; +import org.apache.paimon.types.BooleanType; +import org.apache.paimon.types.CharType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DateType; +import org.apache.paimon.types.DecimalType; +import org.apache.paimon.types.DoubleType; +import org.apache.paimon.types.FloatType; +import org.apache.paimon.types.IntType; +import org.apache.paimon.types.LocalZonedTimestampType; import org.apache.paimon.types.MapType; import org.apache.paimon.types.MultisetType; import org.apache.paimon.types.RowType; +import org.apache.paimon.types.SmallIntType; +import org.apache.paimon.types.TimeType; +import org.apache.paimon.types.TimestampType; +import org.apache.paimon.types.TinyIntType; +import org.apache.paimon.types.VarBinaryType; +import org.apache.paimon.types.VarCharType; +import org.apache.paimon.types.VariantType; +import org.apache.paimon.types.VectorType; import java.util.List; import java.util.Map; -/** Constant-time retained-weight formula for Paimon snapshot projections. */ +/** Publication-time retained-weight formula for Paimon snapshot projections. */ final class PaimonCacheSizeEstimator { - private static final long KEY_BASE_BYTES = 128L; - private static final long SNAPSHOT_BASE_BYTES = 4L * 1024L; - private static final long TABLE_BASE_BYTES = 16L * 1024L; - private static final long TABLE_FIELD_BYTES = 3584L; - private static final long TABLE_OPTION_BYTES = 256L; - private static final long TABLE_KEY_BYTES = 128L; - private static final long NESTED_FIELD_BYTES = 512L; - private static final long PARTITION_BYTES = 1280L; - private static final long PARTITION_ITEM_BYTES = 1024L; - private static final long WRAPPER_BYTES = 512L; + // Calibrated against JOL retained-graph deltas in PaimonExternalMetaCacheTest. + private static final long MAX_TABLE_ACCOUNTING_ELEMENTS = 50_000L; + private static final int MAX_TYPE_ACCOUNTING_DEPTH = 128; + private static final long KEY_BASE_BYTES = objectBytes(128L); + private static final long SNAPSHOT_BASE_BYTES = objectBytes(4L * 1024L); + private static final long TABLE_BASE_BYTES = objectBytes(16L * 1024L); + // A top-level DataField, its list slot and shared per-field overhead; the DataType instance + // is accounted separately by addTypePayload. + private static final long TABLE_FIELD_BYTES = objectBytes(40L); + private static final long TABLE_OPTION_BYTES = objectBytes(44L); + private static final long TABLE_KEY_BYTES = objectBytes(128L); + // Exact Paimon 1.4.2 layouts, pinned by PAIMON_TYPE_LAYOUT_SUPPORTED. + private static final long DATA_FIELD_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(4L, 4L); + private static final long ARRAY_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 1L); + private static final long VECTOR_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 5L); + private static final long MAP_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 1L); + private static final long MULTISET_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 1L); + // RowType plus Collections.unmodifiableList(new ArrayList<>(fields)). + private static final long ROW_TYPE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(6L, 1L); + private static final long UNMODIFIABLE_LIST_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); + private static final long ARRAY_LIST_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 8L); + private static final long HASH_MAP_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(4L, 16L); + private static final long HASH_MAP_NODE_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L); + private static final long INTEGER_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L); + private static final int ROW_TYPE_LAZY_MAP_COUNT = 4; + // Accepted leaf DataType implementations and the int fields each adds to DataType's nullable + // flag and type root. Any other class, including a future or third-party type, rejects + // weighted admission instead of being counted as an arbitrary primitive. + private static final String[] NO_LEAF_FIELDS = {}; + private static final String[] LENGTH_LEAF_FIELDS = {"length:int"}; + private static final String[] PRECISION_LEAF_FIELDS = {"precision:int"}; + private static final Map, String[]> LEAF_TYPE_FIELDS = + ImmutableMap., String[]>builder() + .put(CharType.class, LENGTH_LEAF_FIELDS) + .put(VarCharType.class, LENGTH_LEAF_FIELDS) + .put(BooleanType.class, NO_LEAF_FIELDS) + .put(BinaryType.class, LENGTH_LEAF_FIELDS) + .put(VarBinaryType.class, LENGTH_LEAF_FIELDS) + .put(DecimalType.class, new String[] {"precision:int", "scale:int"}) + .put(TinyIntType.class, NO_LEAF_FIELDS) + .put(SmallIntType.class, NO_LEAF_FIELDS) + .put(IntType.class, NO_LEAF_FIELDS) + .put(BigIntType.class, NO_LEAF_FIELDS) + .put(FloatType.class, NO_LEAF_FIELDS) + .put(DoubleType.class, NO_LEAF_FIELDS) + .put(DateType.class, NO_LEAF_FIELDS) + .put(TimeType.class, PRECISION_LEAF_FIELDS) + .put(TimestampType.class, PRECISION_LEAF_FIELDS) + .put(LocalZonedTimestampType.class, PRECISION_LEAF_FIELDS) + .put(VariantType.class, NO_LEAF_FIELDS) + .put(BlobType.class, NO_LEAF_FIELDS) + .build(); + private static final boolean PAIMON_TYPE_LAYOUT_SUPPORTED = checkPaimonTypeLayout(); + private static final boolean PAIMON_TABLE_LAYOUT_SUPPORTED = checkPaimonTableLayout(); + private static final long PARTITION_BYTES = objectBytes(160L); + private static final long PARTITION_ITEM_BYTES = objectBytes(640L); + private static final long WRAPPER_BYTES = objectBytes(512L); private PaimonCacheSizeEstimator() { } + private static long objectBytes(long bytes) { + return MetaCacheWeightUtils.estimatedObjectBytes(bytes); + } + + /** DataType: typeRoot reference plus the isNullable flag, then the subclass int fields. */ + private static long leafTypeBytes(String[] intFields) { + return MetaCacheWeightUtils.estimatedObjectLayoutBytes( + 1L, 1L + (long) Integer.BYTES * intFields.length); + } + + /** Pin the Paimon 1.4.2 DataType/DataField/RowType layouts the formulas above are built on. */ + private static boolean checkPaimonTypeLayout() { + boolean supported = MetaCacheWeightUtils.hasExpectedInstanceFields( + DataType.class, "isNullable:boolean", "typeRoot:DataTypeRoot") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + DataField.class, "id:int", "name:String", "type:DataType", + "description:String", "defaultValue:String") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + RowType.class, "fields:List", "laziedNameToField:Map", + "laziedNameToIndex:Map", "laziedFieldIdToField:Map", + "laziedFieldIdToIndex:Map") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + ArrayType.class, "elementType:DataType") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + VectorType.class, "elementType:DataType", "length:int") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + MapType.class, "keyType:DataType", "valueType:DataType") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + MultisetType.class, "elementType:DataType"); + for (Map.Entry, String[]> leaf : LEAF_TYPE_FIELDS.entrySet()) { + supported &= MetaCacheWeightUtils.hasExpectedInstanceFields( + leaf.getKey(), leaf.getValue()); + } + return supported; + } + + /** Pin TableSchema and the two accepted FileStoreTable implementations. */ + private static boolean checkPaimonTableLayout() { + ClassLoader loader = FileStoreTable.class.getClassLoader(); + String[] abstractTableFields = { + "fileIO:FileIO", "path:Path", "tableSchema:TableSchema", + "catalogEnvironment:CatalogEnvironment", "manifestCache:SegmentsCache", + "snapshotCache:Cache", "statsCache:Cache", "dvmetaCache:DVMetaCache"}; + return MetaCacheWeightUtils.hasExpectedInstanceFields( + TableSchema.class, "version:int", "id:long", "fields:List", + "highestFieldId:int", "partitionKeys:List", "primaryKeys:List", + "bucketKeys:List", "numBucket:int", "options:Map", "comment:String", + "timeMillis:long") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + "org.apache.paimon.table.AbstractFileStoreTable", loader, + abstractTableFields) + && MetaCacheWeightUtils.hasExpectedInstanceFields( + "org.apache.paimon.table.AppendOnlyFileStoreTable", loader, + "lazyStore:AppendOnlyFileStore") + && MetaCacheWeightUtils.hasExpectedInstanceFields( + "org.apache.paimon.table.PrimaryKeyFileStoreTable", loader, + "lazyStore:KeyValueFileStore"); + } + static MetaCacheSizeEstimate estimateSnapshotEntry( PaimonSnapshotEntryKey key, PaimonSnapshotCacheValue value) { + if (!MetaCacheWeightUtils.isSupportedJvmObjectLayout()) { + return MetaCacheSizeEstimate.incomplete("unsupported_jvm_object_alignment"); + } + if (!PAIMON_TYPE_LAYOUT_SUPPORTED || !PAIMON_TABLE_LAYOUT_SUPPORTED) { + return MetaCacheSizeEstimate.incomplete("unsupported_paimon_layout"); + } Table table = value.getSnapshot().getTable(); if (!isSupportedTable(table)) { return MetaCacheSizeEstimate.incomplete("unsupported_paimon_table:" @@ -118,14 +261,21 @@ private static long estimateTable(Table table) { * All collections are already materialized in TableSchema; this never opens the table store. */ static long retainedTablePayloadBytes(Table table) { + return retainedTablePayloadBytes( + table, new AccountingBudget(MAX_TABLE_ACCOUNTING_ELEMENTS)); + } + + private static long retainedTablePayloadBytes(Table table, AccountingBudget budget) { + budget.charge(1L); if (table instanceof PrivilegedFileStoreTable) { - return retainedTablePayloadBytes(((PrivilegedFileStoreTable) table).wrapped()); + return retainedTablePayloadBytes( + ((PrivilegedFileStoreTable) table).wrapped(), budget); } if (table instanceof FallbackReadFileStoreTable) { FallbackReadFileStoreTable fallback = (FallbackReadFileStoreTable) table; return MetaCacheWeightUtils.saturatedAdd( - retainedTablePayloadBytes(fallback.wrapped()), - retainedTablePayloadBytes(fallback.other())); + retainedTablePayloadBytes(fallback.wrapped(), budget), + retainedTablePayloadBytes(fallback.other(), budget)); } if (!(table instanceof FileStoreTable)) { return 0L; @@ -137,48 +287,137 @@ static long retainedTablePayloadBytes(Table table) { } long bytes = addString(0L, schema.comment()); for (DataField field : schema.fields()) { - bytes = addFieldPayload(bytes, field, false); + bytes = addFieldPayload(bytes, field, false, budget, 0); } + budget.charge(schema.options().size()); for (Map.Entry option : schema.options().entrySet()) { bytes = addString(bytes, option.getKey()); bytes = addString(bytes, option.getValue()); } - bytes = addStrings(bytes, schema.partitionKeys()); - bytes = addStrings(bytes, schema.primaryKeys()); - return addStrings(bytes, schema.bucketKeys()); + bytes = addStrings(bytes, schema.partitionKeys(), budget); + bytes = addStrings(bytes, schema.primaryKeys(), budget); + return addStrings(bytes, schema.bucketKeys(), budget); } - private static long addStrings(long bytes, List values) { + private static long addStrings( + long bytes, List values, AccountingBudget budget) { + budget.charge(values.size()); for (String value : values) { bytes = addString(bytes, value); } return bytes; } - private static long addFieldPayload(long bytes, DataField field, boolean nested) { + private static long addFieldPayload( + long bytes, DataField field, boolean nested, AccountingBudget budget, + int typeDepth) { + budget.charge(1L); if (nested) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, NESTED_FIELD_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, DATA_FIELD_BYTES); } bytes = addString(bytes, field.name()); bytes = addString(bytes, field.description()); bytes = addString(bytes, field.defaultValue()); - return addTypePayload(bytes, field.type()); + return addTypePayload(bytes, field.type(), budget, typeDepth); } - private static long addTypePayload(long bytes, DataType type) { - if (type instanceof RowType) { - for (DataField field : ((RowType) type).getFields()) { - bytes = addFieldPayload(bytes, field, true); + /** + * Account one DataType instance and its owned children. Every accepted implementation is + * matched explicitly; an unknown class throws so estimateSafely rejects weighted admission + * instead of counting a future composite type as a small primitive. + */ + private static long addTypePayload( + long bytes, DataType type, AccountingBudget budget, int typeDepth) { + if (typeDepth > MAX_TYPE_ACCOUNTING_DEPTH) { + throw new IllegalStateException( + "Paimon cache accounting type depth exceeded"); + } + budget.charge(1L); + if (type == null) { + throw new IllegalStateException("Paimon field type is missing"); + } + Class typeClass = type.getClass(); + if (typeClass == RowType.class) { + RowType rowType = (RowType) type; + List fields = rowType.getFields(); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, rowTypeBytes(fields)); + for (DataField field : fields) { + bytes = addFieldPayload(bytes, field, true, budget, typeDepth + 1); } - } else if (type instanceof ArrayType) { - bytes = addTypePayload(bytes, ((ArrayType) type).getElementType()); - } else if (type instanceof MapType) { - bytes = addTypePayload(bytes, ((MapType) type).getKeyType()); - bytes = addTypePayload(bytes, ((MapType) type).getValueType()); - } else if (type instanceof MultisetType) { - bytes = addTypePayload(bytes, ((MultisetType) type).getElementType()); + return bytes; } - return bytes; + if (typeClass == ArrayType.class) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ARRAY_TYPE_BYTES); + return addTypePayload( + bytes, ((ArrayType) type).getElementType(), budget, typeDepth + 1); + } + if (typeClass == VectorType.class) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, VECTOR_TYPE_BYTES); + return addTypePayload( + bytes, ((VectorType) type).getElementType(), budget, typeDepth + 1); + } + if (typeClass == MapType.class) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MAP_TYPE_BYTES); + bytes = addTypePayload( + bytes, ((MapType) type).getKeyType(), budget, typeDepth + 1); + return addTypePayload( + bytes, ((MapType) type).getValueType(), budget, typeDepth + 1); + } + if (typeClass == MultisetType.class) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MULTISET_TYPE_BYTES); + return addTypePayload( + bytes, ((MultisetType) type).getElementType(), budget, typeDepth + 1); + } + String[] leafFields = LEAF_TYPE_FIELDS.get(typeClass); + if (leafFields == null) { + throw new IllegalStateException( + "Unsupported Paimon data type: " + typeClass.getName()); + } + return MetaCacheWeightUtils.saturatedAdd(bytes, leafTypeBytes(leafFields)); + } + + /** + * RowType, its unmodifiable ArrayList copy of the fields, and the four lazy lookup maps that + * getField/getFieldIndex materialize after admission. The maps are reserved up front in O(N) + * so a query cannot grow the retained graph past the admitted weight; nothing is materialized. + */ + private static long rowTypeBytes(List fields) { + long fieldCount = fields.size(); + long bytes = MetaCacheWeightUtils.saturatedAdd(ROW_TYPE_BYTES, UNMODIFIABLE_LIST_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ARRAY_LIST_BYTES); + if (fieldCount == 0L) { + return bytes; + } + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedObjectArrayBytes(fieldCount)); + long uncachedFieldIds = 0L; + for (DataField field : fields) { + if (field.id() < -128 || field.id() > 127) { + uncachedFieldIds++; + } + } + long uncachedIndexes = fieldCount > 128L ? fieldCount - 128L : 0L; + long mapBytes = MetaCacheWeightUtils.saturatedAdd(HASH_MAP_BYTES, + MetaCacheWeightUtils.estimatedObjectArrayBytes(hashMapCapacity(fieldCount))); + mapBytes = addCount(mapBytes, fieldCount, HASH_MAP_NODE_BYTES); + bytes = addCount(bytes, ROW_TYPE_LAZY_MAP_COUNT, mapBytes); + // Boxed keys/values outside the Integer cache: nameToIndex values, fieldIdToField keys, + // and fieldIdToIndex boxes both again. + bytes = addCount(bytes, uncachedIndexes, INTEGER_BYTES); + bytes = addCount(bytes, uncachedFieldIds, INTEGER_BYTES); + bytes = addCount(bytes, uncachedIndexes, INTEGER_BYTES); + return addCount(bytes, uncachedFieldIds, INTEGER_BYTES); + } + + private static long hashMapCapacity(long size) { + long capacity = 16L; + while (size > capacity - capacity / 4L) { + capacity = MetaCacheWeightUtils.saturatedMultiply(capacity, 2L); + if (capacity == Long.MAX_VALUE) { + return capacity; + } + } + return capacity; } private static long addString(long bytes, String value) { @@ -190,4 +429,20 @@ private static long addCount(long bytes, long count, long bytesPerItem) { return MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.saturatedMultiply(count, bytesPerItem)); } + + private static final class AccountingBudget { + private long remaining; + + private AccountingBudget(long remaining) { + this.remaining = remaining; + } + + private void charge(long elements) { + if (elements < 0L || elements > remaining) { + throw new IllegalStateException( + "Paimon cache accounting work budget exceeded"); + } + remaining -= elements; + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 5d98c753b82463..180d2b44adf6cc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -28,6 +28,7 @@ import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryStats; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.hadoop.conf.Configuration; import org.apache.iceberg.BaseTable; @@ -41,6 +42,7 @@ import org.apache.iceberg.ManifestContent; import org.apache.iceberg.ManifestFile; import org.apache.iceberg.Metrics; +import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; @@ -48,6 +50,7 @@ import org.apache.iceberg.SnapshotRef; import org.apache.iceberg.SortOrder; import org.apache.iceberg.StaticTableOperations; +import org.apache.iceberg.StatisticsFile; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableMetadataParser; @@ -56,6 +59,7 @@ import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; +import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import org.junit.Assert; import org.junit.Rule; @@ -63,21 +67,27 @@ import org.junit.rules.TemporaryFolder; import org.mockito.Mockito; +import java.lang.reflect.Field; import java.lang.reflect.Proxy; import java.nio.ByteBuffer; +import java.util.ArrayList; import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.BiFunction; import java.util.stream.Collectors; import java.util.stream.IntStream; public class IcebergExternalMetaCacheTest { + // U+0130 (LATIN CAPITAL LETTER I WITH DOT ABOVE) lower-cases to two characters in Locale.ROOT. + private static final String DOTTED_CAPITAL_I = String.valueOf((char) 0x0130); @Rule public TemporaryFolder temporaryFolder = new TemporaryFolder(); @@ -359,9 +369,45 @@ public void testIcebergPreparationFailureIsFailClosed() { } @Test + public void testManifestAccountingAcceptsOnlyGenericContentFileCopies() { + DataFile copied = DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/copied.parquet").withFileSizeInBytes(10L).withRecordCount(1L) + .build().copy(); + ManifestCacheValue supported = ManifestCacheValue.forDataFiles( + Collections.singletonList(copied)); + Assert.assertTrue(supported.isAccountingComplete()); + Assert.assertTrue(IcebergCacheSizeEstimator.estimateManifestEntry( + new IcebergManifestEntryKey("/manifest/copied.avro", ManifestContent.DATA), + supported).isComplete()); + + // A proxy, mock or third-party ContentFile has an unknown retained layout: keep the file + // for the current query but reject weighted admission. + DataFile proxy = newInterfaceProxy(DataFile.class); + ManifestCacheValue unsupported = ManifestCacheValue.forDataFiles( + Collections.singletonList(proxy)); + Assert.assertEquals(Collections.singletonList(proxy), unsupported.getDataFiles()); + Assert.assertFalse(unsupported.isAccountingComplete()); + Assert.assertEquals("iceberg_manifest_accounting_incomplete", + IcebergCacheSizeEstimator.estimateManifestEntry( + new IcebergManifestEntryKey("/manifest/proxy.avro", ManifestContent.DATA), + unsupported).getIncompleteReason()); + + // A data-file implementation inside a delete manifest is equally unsupported. + ManifestCacheValue.Builder deleteBuilder = ManifestCacheValue.deleteFilesBuilder(); + deleteBuilder.addDeleteFile(newInterfaceProxy(DeleteFile.class)); + Assert.assertFalse(deleteBuilder.build().isAccountingComplete()); + } + + @Test + @SuppressWarnings("unchecked") public void testManifestAccountingFailureKeepsFilesAndRejectsWeightedAdmission() { - DataFile file = Mockito.mock(DataFile.class); - Mockito.when(file.columnSizes()).thenThrow(new IllegalStateException("new metrics representation")); + Map brokenColumnSizes = Mockito.mock(Map.class); + Mockito.when(brokenColumnSizes.size()) + .thenThrow(new IllegalStateException("new metrics representation")); + DataFile file = DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/broken-metrics.parquet").withFileSizeInBytes(10L) + .withMetrics(new Metrics(1L, brokenColumnSizes, null, null, null)) + .build(); ManifestCacheValue value = ManifestCacheValue.forDataFiles(Collections.singletonList(file)); MetaCacheSizeEstimate estimate = IcebergCacheSizeEstimator.estimateManifestEntry( @@ -385,7 +431,8 @@ public void testTableEstimateAccountsForNestedSchemaAndPropertyPayload() { smallValue.prepareForCachePublication(mapping); largeValue.prepareForCachePublication(mapping); - long expectedPayloadDelta = (largePayload.length() - 1L) * 4L; + long expectedPayloadDelta = (MetaCacheWeightUtils.estimatedStringBytes(largePayload) + - MetaCacheWeightUtils.estimatedStringBytes("x")) * 2L; Assert.assertTrue(largeValue.getSizeEstimate().getBytes() - smallValue.getSizeEstimate().getBytes() >= expectedPayloadDelta); } @@ -416,8 +463,10 @@ public void testTablePayloadCountsHistoricalSchemaSpecAndSortFields() { specBuilder.identity(field.name()); sortBuilder.asc(field.name()); } + PartitionSpec populatedSpec = specBuilder.build(); + SortOrder populatedSortOrder = sortBuilder.build(); TableMetadata fieldHistory = TableMetadata.newTableMetadata( - largeSchema, specBuilder.build(), sortBuilder.build(), + largeSchema, populatedSpec, populatedSortOrder, "file:/warehouse/field-history", Collections.emptyMap()); fieldHistory = TableMetadata.buildFrom(fieldHistory) .setDefaultPartitionSpec( @@ -428,6 +477,23 @@ public void testTablePayloadCountsHistoricalSchemaSpecAndSortFields() { TableMetadata emptyFields = TableMetadata.newTableMetadata( largeSchema, PartitionSpec.unpartitioned(), SortOrder.unsorted(), "file:/warehouse/field-history", Collections.emptyMap()); + TableMetadata partitionFields = TableMetadata.newTableMetadata( + largeSchema, populatedSpec, SortOrder.unsorted(), + "file:/warehouse/field-history", Collections.emptyMap()); + TableMetadata sortFields = TableMetadata.newTableMetadata( + largeSchema, PartitionSpec.unpartitioned(), populatedSortOrder, + "file:/warehouse/field-history", Collections.emptyMap()); + + // Cache values come from Iceberg's parser. Round-trip builder fixtures so JOL measures + // the same canonical ownership graph used in production instead of write-side builders. + // Metadata locations of compared fixtures have equal length: that String is not part of + // retainedTablePayloadBytes and must not leak into the JOL delta. + schemaHistory = roundTripMetadata(schemaHistory, "/metadata/jol-schema-large.json"); + smallSchemaOnly = roundTripMetadata(smallSchemaOnly, "/metadata/jol-schema-small.json"); + fieldHistory = roundTripMetadata(fieldHistory, "/metadata/jol-fields-both.json"); + emptyFields = roundTripMetadata(emptyFields, "/metadata/jol-fields-none.json"); + partitionFields = roundTripMetadata(partitionFields, "/metadata/jol-fields-spec.json"); + sortFields = roundTripMetadata(sortFields, "/metadata/jol-fields-sort.json"); long schemaDelta = IcebergCacheSizeEstimator.retainedTablePayloadBytes( tableWithMetadata(schemaHistory)) @@ -437,9 +503,296 @@ public void testTablePayloadCountsHistoricalSchemaSpecAndSortFields() { tableWithMetadata(fieldHistory)) - IcebergCacheSizeEstimator.retainedTablePayloadBytes( tableWithMetadata(emptyFields)); + long partitionFieldDelta = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(partitionFields)) + - IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(emptyFields)); + long sortFieldDelta = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(sortFields)) + - IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(emptyFields)); + + materializeAllLazyState(schemaHistory); + materializeAllLazyState(smallSchemaOnly); + materializeAllLazyState(fieldHistory); + materializeAllLazyState(emptyFields); + materializeAllLazyState(partitionFields); + materializeAllLazyState(sortFields); + + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg schema history", 0L, schemaDelta, + smallSchemaOnly, schemaHistory); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg partition fields", 0L, partitionFieldDelta, + emptyFields, partitionFields); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg sort fields", 0L, sortFieldDelta, + emptyFields, sortFields); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg spec and sort fields", 0L, specAndSortDelta, + emptyFields, fieldHistory); + } + + @Test + public void testSchemaLookupFormulaScalesWithSchemaWidth() { + for (int fieldCount : new int[] {3, 32, 100, 1000}) { + List fields = IntStream.range(0, fieldCount) + .mapToObj(index -> Types.NestedField.optional( + index + 1, "field_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + assertSchemaLookupFormula( + new Schema(0, fields), "schema lookup width " + fieldCount); + } + // Single-column schemas cannot grow; check their partition-spec graph instead. + assertPartitionSpecFormula(new Schema(0, Types.NestedField.optional( + 1, "field_0", Types.StringType.get())), "partition spec width 1"); + } + + @Test + public void testSchemaLookupFormulaCountsListAndMapSyntheticFields() { + List listFields = new ArrayList<>(); + listFields.add(Types.NestedField.optional( + 1, "identity_field", Types.IntegerType.get())); + for (int index = 0; index < 100; index++) { + listFields.add(Types.NestedField.optional( + index + 2, "list_" + index, + Types.ListType.ofOptional(10_000 + index, Types.StringType.get()))); + } + assertSchemaLookupFormula(new Schema(0, listFields), "schema lookup list synthetic fields"); + + List mapFields = new ArrayList<>(); + mapFields.add(Types.NestedField.optional( + 1, "identity_field", Types.IntegerType.get())); + for (int index = 0; index < 100; index++) { + mapFields.add(Types.NestedField.optional( + index + 2, "map_" + index, + Types.MapType.ofOptional( + 10_000 + index * 2, 10_001 + index * 2, + Types.StringType.get(), Types.LongType.get()))); + } + assertSchemaLookupFormula(new Schema(0, mapFields), "schema lookup map synthetic fields"); + } + + @Test + public void testSchemaLookupFormulaCountsNestedStructShortAliases() { + List listFields = new ArrayList<>(); + listFields.add(Types.NestedField.optional( + 1, "identity_field", Types.IntegerType.get())); + for (int index = 0; index < 100; index++) { + Types.StructType elementType = Types.StructType.of(Types.NestedField.optional( + 20_000 + index, "leaf", Types.StringType.get())); + listFields.add(Types.NestedField.optional( + index + 2, "list_" + index, + Types.ListType.ofOptional(10_000 + index, elementType))); + } + assertSchemaLookupFormula( + new Schema(0, listFields), "list struct short aliases"); + + List mapFields = new ArrayList<>(); + mapFields.add(Types.NestedField.optional( + 1, "identity_field", Types.IntegerType.get())); + for (int index = 0; index < 100; index++) { + Types.StructType valueType = Types.StructType.of(Types.NestedField.optional( + 30_000 + index, "leaf", Types.LongType.get())); + mapFields.add(Types.NestedField.optional( + index + 2, "map_" + index, + Types.MapType.ofOptional( + 10_000 + index * 2, 10_001 + index * 2, + Types.StringType.get(), valueType))); + } + assertSchemaLookupFormula( + new Schema(0, mapFields), "map struct short aliases"); + } + + @Test + public void testSchemaIdentifierFieldFormulaScalesWithWidth() { + for (int fieldCount : new int[] {1, 32, 100, 1000}) { + List fields = IntStream.range(0, fieldCount) + .mapToObj(index -> Types.NestedField.required( + index + 1, "identifier_" + index, Types.LongType.get())) + .collect(Collectors.toList()); + Set identifierIds = fields.stream() + .map(Types.NestedField::fieldId) + .collect(Collectors.toSet()); + Schema withoutIdentifiers = new Schema(0, fields); + Schema withIdentifiers = new Schema(0, fields, identifierIds); + TableMetadata empty = roundTripMetadata(TableMetadata.newTableMetadata( + withoutIdentifiers, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-identifiers", Collections.emptyMap()), + "/metadata/jol-schema-identifiers-none-" + fieldCount + ".json"); + TableMetadata populated = roundTripMetadata(TableMetadata.newTableMetadata( + withIdentifiers, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-identifiers", Collections.emptyMap()), + "/metadata/jol-schema-identifiers-with-" + fieldCount + ".json"); + + long emptyEstimate = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(empty)); + long populatedEstimate = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(populated)); + materializeAllLazyState(empty); + materializeAllLazyState(populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg identifier fields " + fieldCount, + emptyEstimate, populatedEstimate, empty, populated); + } + } + + @Test + public void testUnicodeLowerCaseSchemaFormulaAgainstJolOwnedGraph() { + // U+0130 lower-cases to "i" plus U+0307 in Locale.ROOT: the generated lower-case index + // keys are longer than their sources and switch the String coder to UTF-16. + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergTableCacheValue small = tableValueWithSchemaAndProperties( + unicodeNestedSchema(1), Collections.emptyMap()); + IcebergTableCacheValue populated = tableValueWithSchemaAndProperties( + unicodeNestedSchema(33), Collections.emptyMap()); + + long smallEstimate = small.prepareForCachePublication(mapping).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(mapping).getBytes(); + materializeAllLazyState(small); + materializeAllLazyState(populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg unicode lower-case nested fields", + smallEstimate, populatedEstimate, small, populated); + } + + @Test + public void testUnicodeLowerCasePartitionNameFormulaAgainstJolOwnedGraph() { + List fields = new ArrayList<>(); + fields.add(Types.NestedField.optional(1, DOTTED_CAPITAL_I + "dentity_Key", Types.StringType.get())); + for (int index = 0; index < 8; index++) { + fields.add(Types.NestedField.optional(index + 2, DOTTED_CAPITAL_I + "_field_" + index, + Types.StringType.get())); + } + // The identity partition name is lower-cased three times: by the partition StructType, + // by the secondary Schema and by the secondary StructType. + assertPartitionSpecFormula(new Schema(0, fields), "unicode partition name"); + } + + @Test + public void testTableAccountingCharacterBudgetFailsClosedWithoutFailingLoad() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + String hugeName = repeatedCharacter('x', 1 << 20); + List fields = IntStream.range(0, 5) + .mapToObj(index -> Types.NestedField.optional( + index + 1, hugeName + index, Types.StringType.get())) + .collect(Collectors.toList()); + IcebergTableCacheValue value = tableValueWithSchemaAndProperties( + new Schema(0, fields), Collections.emptyMap()); + + IllegalStateException budgetFailure = Assert.assertThrows(IllegalStateException.class, + () -> IcebergCacheSizeEstimator.retainedTablePayloadBytes( + value.getRetainedIcebergTable())); + Assert.assertTrue(budgetFailure.getMessage(), + budgetFailure.getMessage().contains("character budget")); + + MetaCacheSizeEstimate estimate = value.prepareForCachePublication(mapping); + + Assert.assertFalse(estimate.isComplete()); + Assert.assertTrue(estimate.getIncompleteReason(), + estimate.getIncompleteReason().startsWith("iceberg_table_preparation_failed:")); + Assert.assertNotNull(value.getRetainedIcebergTable()); + Assert.assertEquals(5, value.getRetainedIcebergTable().schema().columns().size()); + Assert.assertSame(value.getRetainedIcebergTable(), value.newQueryScopedTable()); + } + + @Test + public void testTableAccountingElementBudgetFailsClosedWithoutFailingLoad() { + TableMetadata metadata = Mockito.mock(TableMetadata.class); + @SuppressWarnings("unchecked") + Map oversizedProperties = Mockito.mock(Map.class); + Mockito.when(oversizedProperties.size()).thenReturn(2_000_001); + Mockito.when(metadata.schemas()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.specs()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.sortOrders()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.properties()).thenReturn(oversizedProperties); + Table table = tableWithMetadata(metadata); + + IllegalStateException budgetFailure = Assert.assertThrows(IllegalStateException.class, + () -> IcebergCacheSizeEstimator.retainedTablePayloadBytes(table)); + Assert.assertTrue(budgetFailure.getMessage(), + budgetFailure.getMessage().contains("work budget")); + Mockito.verify(oversizedProperties, Mockito.never()).entrySet(); + + IcebergTableCacheValue value = new IcebergTableCacheValue(table); + MetaCacheSizeEstimate estimate = value.prepareForCachePublication( + NameMapping.createForTest(1L, "db", "tbl")); + Assert.assertFalse(estimate.isComplete()); + Assert.assertTrue(estimate.getIncompleteReason(), + estimate.getIncompleteReason().startsWith("iceberg_table_preparation_failed:")); + Assert.assertSame(value.getRetainedIcebergTable(), value.newQueryScopedTable()); + } + + @Test + public void testAdmittedTableEstimateCoversFullyMaterializedRetainedGraph() { + // Whole-entry oracle: the admitted weight must cover the complete retained graph after + // every lazy Schema/StructType/PartitionSpec index a scan can create has materialized, + // including the O(distinctSources * fields) fieldsBySourceId graph. Component deltas can + // miss a shared baseline; this compares absolute sizes. + // The tight bound applies once the variable payload dominates the fixed per-table base + // (TABLE_BASE_BYTES); small tables are deliberately covered by that base. + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + for (int width : new int[] {1, 100, 1000}) { + assertAdmittedEstimateCoversRetainedGraph( + "flat identity-partitioned " + width, mapping, + tableValueWithIdentityPartitionedFields(width), width >= 100); + assertAdmittedEstimateCoversRetainedGraph( + "nested mixed-case " + width, mapping, + tableValueWithNestedMixedCaseFields(width), width >= 1000); + } + } + + private void assertAdmittedEstimateCoversRetainedGraph( + String fixture, NameMapping mapping, IcebergTableCacheValue value, + boolean requireTightBound) { + long estimate = value.prepareForCachePublication(mapping).getBytes(); + long before = EstimatorCalibrationAssertions.graphSize(value); + materializeAllLazyState(value); + long after = EstimatorCalibrationAssertions.graphSize(value); + Assert.assertTrue(fixture + " lazy state must grow the retained graph", after > before); + Assert.assertTrue(fixture + " underestimates the materialized entry: estimate=" + + estimate + ", retained=" + after, estimate >= after); + if (requireTightBound) { + Assert.assertTrue(fixture + " is excessively conservative: estimate=" + estimate + + ", retained=" + after, estimate <= Math.ceil(after * 1.10D)); + } + } + + @Test + public void testSchemaFormulaCountsBoxedIdsOfUncachedFieldIds() { + // TableMetadata.newTableMetadata() reassigns fresh ids from 1, which the JVM Integer + // cache serves for free. Add the schema to existing metadata instead so ids above 127 + // survive and every lookup map really boxes its keys and values. + List oneFlat = Collections.singletonList( + Types.NestedField.optional(10_000, "Field_0", Types.StringType.get())); + List manyFlat = IntStream.range(0, 32) + .mapToObj(index -> Types.NestedField.optional( + 10_000 + index, "Field_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + assertRetainedPayloadDelta("uncached flat field ids", + metadataWithAddedSchema(new Schema(1, oneFlat)), + metadataWithAddedSchema(new Schema(1, manyFlat)), "jol-uncached-ids"); + + List nestedFields = IntStream.range(0, 32) + .mapToObj(index -> Types.NestedField.optional( + 20_000 + index, "Nested_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + Schema oneNested = new Schema(1, Types.NestedField.optional(10_000, "payload", + Types.StructType.of(nestedFields.get(0)))); + Schema manyNested = new Schema(1, Types.NestedField.optional(10_000, "payload", + Types.StructType.of(nestedFields))); + assertRetainedPayloadDelta("uncached nested field ids", + metadataWithAddedSchema(oneNested), metadataWithAddedSchema(manyNested), + "jol-uncached-ids"); + } - Assert.assertTrue(schemaDelta >= 99L * 512L); - Assert.assertTrue(specAndSortDelta >= 100L * (384L + 256L)); + private TableMetadata metadataWithAddedSchema(Schema schema) { + Schema base = new Schema(0, Types.NestedField.optional(1, "base", Types.IntegerType.get())); + TableMetadata metadata = TableMetadata.newTableMetadata( + base, PartitionSpec.unpartitioned(), "file:/warehouse/uncached-ids", + Collections.emptyMap()); + return TableMetadata.buildFrom(metadata).addSchema(schema) + .setCurrentSchema(schema.schemaId()).discardChanges().build(); } @Test @@ -453,6 +806,33 @@ public void testTablePayloadAccountsForRetainedHistoricalMetadata() { Assert.assertTrue(largeBytes - smallBytes >= 64L * 1024L - 32L); } + @Test + public void testStatisticsBlobFormulaAgainstJolOwnedGraph() { + GenericStatisticsFile empty = new GenericStatisticsFile( + 1L, "/stats/file.puffin", 1L, 1L, Collections.emptyList()); + List blobs = IntStream.range(0, 32) + .mapToObj(index -> new GenericBlobMetadata( + "blob-type-" + index, + 1L, + 1L, + java.util.Arrays.asList(10_000 + index, 20_000 + index), + Collections.singletonMap( + "property-" + index, "value-" + index))) + .collect(Collectors.toList()); + GenericStatisticsFile populated = new GenericStatisticsFile( + 1L, "/stats/file.puffin", 1L, 1L, blobs); + TableMetadata emptyMetadata = metadataWithStatisticsFile(empty); + TableMetadata populatedMetadata = metadataWithStatisticsFile(populated); + + long emptyEstimate = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(emptyMetadata)); + long populatedEstimate = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(populatedMetadata)); + + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg statistics blobs", emptyEstimate, populatedEstimate, empty, populated); + } + @Test public void testTableEstimateAccountsForRetainedBranchHistory() { TableMetadata oneCommit = metadataWithSnapshotSequence(1L); @@ -688,7 +1068,7 @@ protected CatalogIf getCatalog(long catalogId) { } @Test - public void testWeightedTablePublicationRetainsNonGrowingGeneration() { + public void testWeightedV1TablePublicationFailsClosedWithoutIo() { NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), @@ -711,29 +1091,13 @@ public void testWeightedTablePublicationRetainsNonGrowingGeneration() { value.prepareForCachePublication(mapping); - Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), value.getSizeEstimate().isComplete()); + Assert.assertFalse(value.getSizeEstimate().isComplete()); Table retained = value.getRetainedIcebergTable(); Assert.assertTrue(IcebergSnapshotCacheValue.isFrozenGeneration(retained)); - Table firstUse = value.getIcebergTable(); - Table secondUse = value.getIcebergTable(); - Assert.assertNotSame(retained, firstUse); - Assert.assertNotSame(firstUse, secondUse); - Assert.assertNotSame(retained.currentSnapshot(), firstUse.currentSnapshot()); - Assert.assertNotSame(firstUse.currentSnapshot(), secondUse.currentSnapshot()); - Assert.assertEquals(2, firstUse.snapshot(7L).dataManifests(firstUse.io()).size()); - Assert.assertEquals(2, secondUse.snapshot(7L).dataManifests(secondUse.io()).size()); - - IcebergSnapshotEntryKey snapshotKey = - IcebergSnapshotEntryKey.tryCreate(mapping, retained).get(); - IcebergSnapshotCacheValue snapshotValue = new IcebergSnapshotCacheValue( - IcebergPartitionInfo.empty(), new IcebergSnapshot(7L, 0L), - Optional.empty(), retained, value.getRetainedCurrentSnapshotJson()); - snapshotValue.prepareForCachePublication(snapshotKey); - Assert.assertTrue(snapshotValue.getSizeEstimate().getIncompleteReason(), - snapshotValue.getSizeEstimate().isComplete()); - Table snapshotQuery = snapshotValue.getIcebergTable().get(); - Assert.assertEquals(2, - snapshotQuery.currentSnapshot().dataManifests(snapshotQuery.io()).size()); + Mockito.verifyNoInteractions(fileIO); + retained.currentSnapshot().allManifests(fileIO); + Mockito.verify(fileIO, Mockito.atLeastOnce()).newInputFile("/manifest/current-a.avro"); + Mockito.verify(fileIO, Mockito.atLeastOnce()).newInputFile("/manifest/current-b.avro"); } @Test @@ -778,7 +1142,8 @@ public void testTablePublicationDoesNotReadHistoricalManifestLists() { + "\"summary\":{\"operation\":\"append\"}," + "\"manifest-list\":\"/manifest-list/history.avro\",\"schema-id\":0}"); Snapshot current = SnapshotParser.fromJson("{\"snapshot-id\":7,\"timestamp-ms\":2," - + "\"summary\":{\"operation\":\"append\"},\"manifests\":[],\"schema-id\":0}"); + + "\"summary\":{\"operation\":\"append\"}," + + "\"manifest-list\":\"/manifest-list/current.avro\",\"schema-id\":0}"); metadata = TableMetadata.buildFrom(metadata) .addSnapshot(historical) .setBranchSnapshot(current, SnapshotRef.MAIN_BRANCH) @@ -820,16 +1185,288 @@ public void testManifestFormulaAgainstJolOwnedGraph() { Collections.singletonList(dataFileWithPathPayload(16))); ManifestCacheValue longTail = ManifestCacheValue.forDataFiles( Collections.singletonList(dataFileWithPathPayload(4096))); + ManifestCacheValue emptyDeletes = ManifestCacheValue.forDeleteFiles(Collections.emptyList()); + ManifestCacheValue populatedDeletes = ManifestCacheValue.forDeleteFiles( + IntStream.range(0, 32).mapToObj(this::deleteFileWithMetrics) + .collect(Collectors.toList())); long emptyEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, empty).getBytes(); long populatedEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, populated).getBytes(); long shortTailEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, shortTail).getBytes(); long longTailEstimate = IcebergCacheSizeEstimator.estimateManifestEntry(key, longTail).getBytes(); + IcebergManifestEntryKey deleteKey = new IcebergManifestEntryKey( + "/manifest/jol-delete.avro", ManifestContent.DELETES); + long emptyDeleteEstimate = IcebergCacheSizeEstimator.estimateManifestEntry( + deleteKey, emptyDeletes).getBytes(); + long populatedDeleteEstimate = IcebergCacheSizeEstimator.estimateManifestEntry( + deleteKey, populatedDeletes).getBytes(); EstimatorCalibrationAssertions.assertConservativeDelta( "iceberg manifest files", emptyEstimate, populatedEstimate, empty, populated); EstimatorCalibrationAssertions.assertConservativeDelta( "iceberg long-tail path", shortTailEstimate, longTailEstimate, shortTail, longTail); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg manifest delete files", emptyDeleteEstimate, populatedDeleteEstimate, + emptyDeletes, populatedDeletes); + } + + @Test + public void testManifestPartitionDataFormulaAgainstJolOwnedGraph() { + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/jol-partitioned.avro", ManifestContent.DATA); + for (int fieldCount : new int[] {1, 8, 32, 100}) { + ManifestCacheValue unpartitioned = ManifestCacheValue.forDataFiles( + manifestFilesWithIntegerPartitions(32, 0)); + ManifestCacheValue partitioned = ManifestCacheValue.forDataFiles( + manifestFilesWithIntegerPartitions(32, fieldCount)); + long unpartitionedEstimate = IcebergCacheSizeEstimator.estimateManifestEntry( + key, unpartitioned).getBytes(); + long partitionedEstimate = IcebergCacheSizeEstimator.estimateManifestEntry( + key, partitioned).getBytes(); + + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg manifest partition fields " + fieldCount, + unpartitionedEstimate, partitionedEstimate, + unpartitioned, partitioned); + } + } + + @Test + public void testManifestVariablePartitionValuesAgainstJolOwnedGraph() { + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/jol-variable-partition.avro", ManifestContent.DATA); + ManifestCacheValue unpartitioned = ManifestCacheValue.forDataFiles( + manifestFilesWithIntegerPartitions(32, 0)); + long unpartitionedEstimate = IcebergCacheSizeEstimator.estimateManifestEntry( + key, unpartitioned).getBytes(); + + ManifestCacheValue strings = ManifestCacheValue.forDataFiles( + manifestFilesWithPartitions(32, 8, Types.StringType.get(), + (fileIndex, fieldIndex) -> repeatedCharacter('s', 64) + + fileIndex + "_" + fieldIndex)); + long stringEstimate = IcebergCacheSizeEstimator.estimateManifestEntry( + key, strings).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg manifest string partitions", + unpartitionedEstimate, stringEstimate, unpartitioned, strings); + + ManifestCacheValue binary = ManifestCacheValue.forDataFiles( + manifestFilesWithPartitions(32, 8, Types.BinaryType.get(), + (fileIndex, fieldIndex) -> ByteBuffer.allocate(64))); + long binaryEstimate = IcebergCacheSizeEstimator.estimateManifestEntry( + key, binary).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg manifest binary partitions", + unpartitionedEstimate, binaryEstimate, unpartitioned, binary); + } + + @Test + public void testManifestAccountingFindsPathTailAtAnyPosition() { + List baselineFiles = new ArrayList<>(); + List tailFiles = new ArrayList<>(); + String largePath = "/data/" + repeatedCharacter('x', 64 * 1024) + ".parquet"; + for (int index = 0; index < 101; index++) { + baselineFiles.add(DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/file-" + index + ".parquet") + .withFileSizeInBytes(10L).withRecordCount(1L).build()); + tailFiles.add(DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath(index == 57 ? largePath : "/data/file-" + index + ".parquet") + .withFileSizeInBytes(10L).withRecordCount(1L).build()); + } + ManifestCacheValue baseline = ManifestCacheValue.forDataFiles(baselineFiles); + ManifestCacheValue withTail = ManifestCacheValue.forDataFiles(tailFiles); + IcebergManifestEntryKey key = new IcebergManifestEntryKey( + "/manifest/path-tail.avro", ManifestContent.DATA); + + long baselineBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, baseline).getBytes(); + long tailBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, withTail).getBytes(); + + Assert.assertTrue(tailBytes - baselineBytes >= largePath.length() - 32L); + Assert.assertTrue(tailBytes - baselineBytes < largePath.length() * 2L); + } + + @Test + @SuppressWarnings("unchecked") + public void testManifestAccountingFailsClosedBeyondWorkBudget() { + Map oversizedBounds = Mockito.mock(Map.class); + Mockito.when(oversizedBounds.size()).thenReturn(8_000_001); + // GenericDataFile wraps the bounds map without copying it, so the oversized size is + // observed by accounting without allocating the entries. + DataFile file = DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/data/oversized-bounds.parquet").withFileSizeInBytes(10L) + .withMetrics(new Metrics(1L, null, null, null, null, oversizedBounds, null)) + .build(); + + ManifestCacheValue value = ManifestCacheValue.forDataFiles( + Collections.singletonList(file)); + + Assert.assertFalse(value.isAccountingComplete()); + Assert.assertFalse(IcebergCacheSizeEstimator.estimateManifestEntry( + new IcebergManifestEntryKey("/manifest/oversized.avro", ManifestContent.DATA), + value).isComplete()); + } + + @Test + public void testManifestAccountingFailsClosedForUnreadablePartition() { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + PartitionSpec spec = PartitionSpec.builderFor(schema).identity("id").build(); + DataFile file = DataFiles.builder(spec) + .withPath("/data/unreadable-partition.parquet").withFileSizeInBytes(10L) + .withRecordCount(1L).withPartitionPath("id=1").build(); + // A partition value of a class the accounting does not know cannot be sized. + ((PartitionData) file.partition()).set(0, new Object()); + + ManifestCacheValue value = ManifestCacheValue.forDataFiles( + Collections.singletonList(file)); + + Assert.assertFalse(value.isAccountingComplete()); + } + + @Test + public void testTableAndSnapshotFormulasAgainstJolOwnedGraphs() throws Exception { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + // Compare two non-empty schemas: an empty schema never materializes any lookup index, + // so it is not a fair baseline for the per-field formula. + IcebergTableCacheValue emptyTable = tableValueWithFields(1); + IcebergTableCacheValue populatedTable = tableValueWithFields(32); + long emptyTableEstimate = emptyTable.prepareForCachePublication(mapping).getBytes(); + long populatedTableEstimate = populatedTable.prepareForCachePublication(mapping).getBytes(); + materializeAllLazyState(emptyTable); + materializeAllLazyState(populatedTable); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg table fields", emptyTableEstimate, populatedTableEstimate, + emptyTable, populatedTable); + + Table keyTable = tableWithMetadataLocation("/metadata/jol-snapshot-v1.json"); + IcebergSnapshotEntryKey key = IcebergSnapshotEntryKey.tryCreate(mapping, keyTable).get(); + IcebergSnapshotCacheValue emptySnapshot = new IcebergSnapshotCacheValue( + realPartitionInfo(0), new IcebergSnapshot(-1L, 0L)); + IcebergSnapshotCacheValue populatedSnapshot = new IcebergSnapshotCacheValue( + realPartitionInfo(32), new IcebergSnapshot(-1L, 0L)); + long emptySnapshotEstimate = IcebergCacheSizeEstimator.estimateSnapshotEntry( + key, emptySnapshot).getBytes(); + long populatedSnapshotEstimate = IcebergCacheSizeEstimator.estimateSnapshotEntry( + key, populatedSnapshot).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg snapshot partitions", emptySnapshotEstimate, populatedSnapshotEstimate, + emptySnapshot, populatedSnapshot); + } + + @Test + public void testNestedSchemaFormulaAgainstJolOwnedGraph() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergTableCacheValue small = tableValueWithNestedFields(1); + IcebergTableCacheValue populated = tableValueWithNestedFields(33); + + long smallEstimate = small.prepareForCachePublication(mapping).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(mapping).getBytes(); + materializeAllLazyState(small); + materializeAllLazyState(populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg nested fields", smallEstimate, populatedEstimate, small, populated); + } + + @Test + public void testTablePropertyFormulaAgainstJolOwnedGraph() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergTableCacheValue empty = tableValueWithProperties(0); + IcebergTableCacheValue populated = tableValueWithProperties(32); + + long emptyEstimate = empty.prepareForCachePublication(mapping).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(mapping).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg table properties", emptyEstimate, populatedEstimate, empty, populated); + } + + @Test + public void testSnapshotHistoryFormulaAgainstJolOwnedGraph() { + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergTableCacheValue small = tableValueWithSnapshotHistory(1, false); + IcebergTableCacheValue populated = tableValueWithSnapshotHistory(33, false); + long smallEstimate = small.prepareForCachePublication(mapping).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(mapping).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg snapshot history", smallEstimate, populatedEstimate, small, populated); + + IcebergTableCacheValue withoutLog = tableValueWithSnapshotHistory(33, false); + IcebergTableCacheValue withLog = tableValueWithSnapshotHistory(33, true); + long withoutLogEstimate = withoutLog.prepareForCachePublication(mapping).getBytes(); + long withLogEstimate = withLog.prepareForCachePublication(mapping).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg snapshot log", withoutLogEstimate, withLogEstimate, withoutLog, withLog); + } + + @Test + public void testV1SnapshotAccountingFailsClosedWithoutIo() { + Snapshot snapshot = SnapshotParser.fromJson( + "{\"snapshot-id\":1,\"timestamp-ms\":1," + + "\"manifests\":[\"/manifest/v1-a.avro\",\"/manifest/v1-b.avro\"]}"); + IcebergTableCacheValue value = new IcebergTableCacheValue( + tableWithMetadata(metadataWithSnapshots(snapshot))); + + MetaCacheSizeEstimate estimate = value.prepareForCachePublication( + NameMapping.createForTest(1L, "db", "tbl")); + + Assert.assertFalse(estimate.isComplete()); + } + + @Test + public void testSnapshotWithoutSummaryRemainsCacheable() { + Snapshot snapshot = SnapshotParser.fromJson( + "{\"snapshot-id\":1,\"timestamp-ms\":1," + + "\"manifest-list\":\"/manifest/list.avro\"}"); + IcebergTableCacheValue value = new IcebergTableCacheValue( + tableWithMetadata(metadataWithSnapshots(snapshot))); + + MetaCacheSizeEstimate estimate = value.prepareForCachePublication( + NameMapping.createForTest(1L, "db", "tbl")); + + Assert.assertTrue(estimate.getIncompleteReason(), estimate.isComplete()); + } + + @Test + public void testMaterializedV2SnapshotPayloadFailsClosed() throws Exception { + String snapshotJson = "{\"snapshot-id\":1,\"timestamp-ms\":1," + + "\"manifest-list\":\"/manifest/list.avro\"}"; + Snapshot unloaded = SnapshotParser.fromJson(snapshotJson); + MetaCacheSizeEstimate unloadedEstimate = new IcebergTableCacheValue( + tableWithMetadata(metadataWithSnapshots(unloaded))) + .prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + Assert.assertTrue(unloadedEstimate.getIncompleteReason(), unloadedEstimate.isComplete()); + + for (String fieldName : new String[] { + "allManifests", "dataManifests", "deleteManifests", + "addedDataFiles", "removedDataFiles", + "addedDeleteFiles", "removedDeleteFiles"}) { + Snapshot loaded = SnapshotParser.fromJson(snapshotJson); + Field retainedField = loaded.getClass().getDeclaredField(fieldName); + retainedField.setAccessible(true); + retainedField.set(loaded, Collections.emptyList()); + + MetaCacheSizeEstimate loadedEstimate = new IcebergTableCacheValue( + tableWithMetadata(metadataWithSnapshots(loaded))) + .prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + Assert.assertFalse(fieldName, loadedEstimate.isComplete()); + } + } + + @Test + public void testSnapshotKeyIdPayloadIsAccounted() { + String longKeyId = repeatedCharacter('k', 64 * 1024); + Snapshot shortSnapshot = SnapshotParser.fromJson( + "{\"snapshot-id\":1,\"timestamp-ms\":1," + + "\"manifest-list\":\"/manifest/list.avro\",\"key-id\":\"k\"}"); + Snapshot longSnapshot = SnapshotParser.fromJson( + "{\"snapshot-id\":1,\"timestamp-ms\":1," + + "\"manifest-list\":\"/manifest/list.avro\",\"key-id\":\"" + + longKeyId + "\"}"); + + long shortBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(metadataWithSnapshots(shortSnapshot))); + long longBytes = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(metadataWithSnapshots(longSnapshot))); + + Assert.assertTrue(longBytes - shortBytes >= longKeyId.length() - 8L); } @Test @@ -853,7 +1490,7 @@ public void testManifestEstimateAccountsForSkewedFilePaths() { long smallBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, smallValue).getBytes(); long largeBytes = IcebergCacheSizeEstimator.estimateManifestEntry(key, largeValue).getBytes(); - Assert.assertTrue(largeBytes - smallBytes >= (largePath.length() - "/data/x.parquet".length()) * 2L); + Assert.assertTrue(largeBytes - smallBytes >= largePath.length() - "/data/x.parquet".length()); } @Test @@ -940,7 +1577,7 @@ public void testManifestEstimateAccountsForDeleteFileAuxiliaryPayload() { } @Test - public void testSnapshotPublicationDoesNotMaterializeManifestLists() { + public void testV1SnapshotPublicationDoesNotPolluteManifestIo() { Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), "file:/warehouse/db/tbl", Collections.emptyMap()); @@ -950,6 +1587,11 @@ public void testSnapshotPublicationDoesNotMaterializeManifestLists() { metadata = TableMetadata.buildFrom(metadata).setBranchSnapshot(snapshot, SnapshotRef.MAIN_BRANCH) .discardChanges().withMetadataLocation("/metadata/v1.json").build(); FileIO fileIO = Mockito.mock(FileIO.class); + Mockito.when(fileIO.newInputFile(Mockito.anyString())).thenAnswer(invocation -> { + InputFile inputFile = Mockito.mock(InputFile.class); + Mockito.when(inputFile.location()).thenReturn(invocation.getArgument(0)); + return inputFile; + }); Table table = new BaseTable(new StaticTableOperations(metadata, fileIO), "db.tbl"); IcebergSnapshotCacheValue value = new IcebergSnapshotCacheValue( IcebergPartitionInfo.empty(), new IcebergSnapshot(7L, 0L), Optional.empty(), table); @@ -958,11 +1600,13 @@ public void testSnapshotPublicationDoesNotMaterializeManifestLists() { value.prepareForCachePublication(key); - Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), - value.getSizeEstimate().isComplete()); + Assert.assertFalse(value.getSizeEstimate().isComplete()); Table queryTable = value.getIcebergTable().get(); - Assert.assertNotSame(table.currentSnapshot(), queryTable.currentSnapshot()); + Assert.assertSame(table.currentSnapshot(), queryTable.currentSnapshot()); Mockito.verifyNoInteractions(fileIO); + queryTable.currentSnapshot().allManifests(fileIO); + Mockito.verify(fileIO, Mockito.atLeastOnce()).newInputFile("/manifest/a.avro"); + Mockito.verify(fileIO, Mockito.atLeastOnce()).newInputFile("/manifest/b.avro"); } @Test @@ -1220,23 +1864,190 @@ private Table tableWithMetadataLocation(String metadataLocation) { return new BaseTable(new StaticTableOperations(metadata, null), "db.tbl"); } + private IcebergTableCacheValue tableValueWithFields(int fieldCount) { + List fields = IntStream.range(0, fieldCount) + .mapToObj(index -> Types.NestedField.optional( + index + 1, "field_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + TableMetadata metadata = TableMetadata.newTableMetadata( + new Schema(fields), PartitionSpec.unpartitioned(), + "file:/warehouse/jol-table", Collections.emptyMap()); + metadata = TableMetadata.buildFrom(metadata).discardChanges() + .withMetadataLocation("/metadata/jol-table-v1.json").build(); + metadata = roundTripMetadata(metadata, "/metadata/jol-table-v1.json"); + return new IcebergTableCacheValue( + new BaseTable(new StaticTableOperations(metadata, null), "db.tbl")); + } + + private IcebergTableCacheValue tableValueWithNestedFields(int nestedFieldCount) { + List nestedFields = IntStream.range(0, nestedFieldCount) + .mapToObj(index -> Types.NestedField.optional( + index + 2, "nested_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + Schema schema = new Schema(Types.NestedField.optional( + 1, "payload", Types.StructType.of(nestedFields))); + return tableValueWithSchemaAndProperties(schema, Collections.emptyMap()); + } + + private Schema unicodeNestedSchema(int nestedFieldCount) { + List nestedFields = new ArrayList<>(); + for (int index = 0; index < nestedFieldCount; index++) { + nestedFields.add(Types.NestedField.optional( + index + 10, "Nested_" + DOTTED_CAPITAL_I + "_" + index, Types.StringType.get())); + } + Types.StructType element = Types.StructType.of(Types.NestedField.optional( + 3, "Leaf_" + DOTTED_CAPITAL_I, Types.StringType.get())); + return new Schema( + Types.NestedField.optional(1, "Payload_" + DOTTED_CAPITAL_I, Types.StructType.of(nestedFields)), + Types.NestedField.optional(2, "List_" + DOTTED_CAPITAL_I, + Types.ListType.ofOptional(4, element)), + Types.NestedField.optional(5, "Map_" + DOTTED_CAPITAL_I, Types.MapType.ofOptional( + 6, 7, Types.StringType.get(), Types.StringType.get()))); + } + + private IcebergTableCacheValue tableValueWithIdentityPartitionedFields(int fieldCount) { + List fields = IntStream.range(0, fieldCount) + .mapToObj(index -> Types.NestedField.optional( + index + 1, "field_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + Schema schema = new Schema(fields); + PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(schema); + for (Types.NestedField field : fields) { + specBuilder.identity(field.name()); + } + return tableValueWithSchemaAndSpec(schema, specBuilder.build()); + } + + private IcebergTableCacheValue tableValueWithNestedMixedCaseFields(int nestedFieldCount) { + // Uncached field ids, upper-case names and list/map synthetic fields exercise the boxed + // key, lower-case String and short-alias terms of the schema formula. + List nestedFields = IntStream.range(0, nestedFieldCount) + .mapToObj(index -> Types.NestedField.optional( + 1000 + index, "Nested_" + index, Types.StringType.get())) + .collect(Collectors.toList()); + Schema schema = new Schema( + Types.NestedField.optional(1, "payload", Types.StructType.of(nestedFields)), + Types.NestedField.optional(2, "list", Types.ListType.ofOptional(3, + Types.StructType.of(Types.NestedField.optional( + 4, "leaf", Types.StringType.get())))), + Types.NestedField.optional(5, "map", Types.MapType.ofOptional( + 6, 7, Types.StringType.get(), Types.LongType.get())), + Types.NestedField.optional(8, "id", Types.LongType.get())); + return tableValueWithSchemaAndSpec( + schema, PartitionSpec.builderFor(schema).identity("id").build()); + } + + private IcebergTableCacheValue tableValueWithSchemaAndSpec(Schema schema, PartitionSpec spec) { + TableMetadata metadata = TableMetadata.newTableMetadata( + schema, spec, "file:/warehouse/jol-table", Collections.emptyMap()); + metadata = TableMetadata.buildFrom(metadata).discardChanges() + .withMetadataLocation("/metadata/jol-table-v1.json").build(); + metadata = roundTripMetadata(metadata, "/metadata/jol-table-v1.json"); + return new IcebergTableCacheValue( + new BaseTable(new StaticTableOperations(metadata, null), "db.tbl")); + } + + private IcebergTableCacheValue tableValueWithProperties(int propertyCount) { + Map properties = IntStream.range(0, propertyCount).boxed() + .collect(Collectors.toMap(index -> "key_" + index, index -> "value_" + index)); + Schema schema = new Schema(Types.NestedField.required( + 1, "id", Types.IntegerType.get())); + return tableValueWithSchemaAndProperties(schema, properties); + } + + private IcebergTableCacheValue tableValueWithSchemaAndProperties( + Schema schema, Map properties) { + TableMetadata metadata = TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), + "file:/warehouse/jol-table", properties); + metadata = TableMetadata.buildFrom(metadata).discardChanges() + .withMetadataLocation("/metadata/jol-table-v1.json").build(); + metadata = roundTripMetadata(metadata, "/metadata/jol-table-v1.json"); + return new IcebergTableCacheValue( + new BaseTable(new StaticTableOperations(metadata, null), "db.tbl")); + } + + private IcebergTableCacheValue tableValueWithSnapshotHistory( + int snapshotCount, boolean includeSnapshotLog) { + long currentSnapshotId = 1000L + snapshotCount - 1L; + StringBuilder json = new StringBuilder() + .append("{\"format-version\":2,\"table-uuid\":\"jol-table\",") + .append("\"location\":\"file:/warehouse/jol-table\",\"last-sequence-number\":") + .append(snapshotCount).append(",\"last-updated-ms\":").append(snapshotCount) + .append(",\"last-column-id\":1,\"current-schema-id\":0,") + .append("\"schemas\":[{\"type\":\"struct\",\"schema-id\":0,\"fields\":[") + .append("{\"id\":1,\"name\":\"field\",\"required\":false,\"type\":\"string\"}]}],") + .append("\"default-spec-id\":0,\"partition-specs\":[{\"spec-id\":0,\"fields\":[]}],") + .append("\"last-partition-id\":999,\"default-sort-order-id\":0,") + .append("\"sort-orders\":[{\"order-id\":0,\"fields\":[]}],\"properties\":{},") + .append("\"current-snapshot-id\":").append(currentSnapshotId) + .append(",\"refs\":{\"main\":{\"snapshot-id\":").append(currentSnapshotId) + .append(",\"type\":\"branch\"}},\"snapshots\":["); + for (int index = 0; index < snapshotCount; index++) { + if (index > 0) { + json.append(','); + } + json.append("{\"sequence-number\":").append(index + 1L) + .append(",\"snapshot-id\":").append(1000L + index); + if (index > 0) { + json.append(",\"parent-snapshot-id\":").append(1000L + index - 1L); + } + json.append(",\"timestamp-ms\":").append(index + 1L) + .append(",\"summary\":{\"operation\":\"append\"},") + .append("\"manifest-list\":\"/jol/list-").append(index) + .append(".avro\",\"schema-id\":0}"); + } + json.append("],\"statistics\":[],\"partition-statistics\":[],\"snapshot-log\":["); + if (includeSnapshotLog) { + for (int index = 0; index < snapshotCount; index++) { + if (index > 0) { + json.append(','); + } + json.append("{\"timestamp-ms\":").append(index + 1L) + .append(",\"snapshot-id\":").append(1000L + index).append('}'); + } + } + json.append("],\"metadata-log\":[]}"); + TableMetadata metadata = TableMetadataParser.fromJson( + "/metadata/jol-history-v1.json", json.toString()); + return new IcebergTableCacheValue( + new BaseTable(new StaticTableOperations(metadata, null), "db.tbl")); + } + + private IcebergPartitionInfo realPartitionInfo(int partitionCount) throws Exception { + Map partitionItems = new java.util.HashMap<>(); + Map partitions = new java.util.HashMap<>(); + List partitionColumns = Collections.singletonList( + new org.apache.doris.catalog.Column( + "part", org.apache.doris.catalog.PrimitiveType.DATETIMEV2)); + for (int index = 0; index < partitionCount; index++) { + String value = Integer.toString(index); + String name = "part=" + value; + partitionItems.put(name, new org.apache.doris.catalog.RangePartitionItem( + IcebergUtils.getPartitionRange(value, "day", partitionColumns))); + partitions.put(name, new IcebergPartition(name, 0, 1L, 1L, 1L, + 1L, 1L, Collections.singletonList(value), + Collections.singletonList("day"))); + } + return new IcebergPartitionInfo( + partitionItems, partitions, Collections.emptyMap()); + } + private org.apache.iceberg.DataFile dataFileWithMetrics(int index) { - Map columnSizes = IntStream.range(0, 8).boxed() - .collect(Collectors.toMap(column -> column, column -> (long) index + column)); - Map valueCounts = new java.util.HashMap<>(columnSizes); - Map nullCounts = new java.util.HashMap<>(columnSizes); - Map nanCounts = new java.util.HashMap<>(columnSizes); - Map lowerBounds = IntStream.range(0, 8).boxed() - .collect(Collectors.toMap(column -> column, column -> ByteBuffer.allocate(32))); - Map upperBounds = IntStream.range(0, 8).boxed() - .collect(Collectors.toMap(column -> column, column -> ByteBuffer.allocate(32))); + Map columnSizes = metricLongMap(index, 0); + Map valueCounts = metricLongMap(index, 1); + Map nullCounts = metricLongMap(index, 2); + Map nanCounts = metricLongMap(index, 3); + Map lowerBounds = metricBufferMap(); + Map upperBounds = metricBufferMap(); Metrics metrics = new Metrics( 100L, columnSizes, valueCounts, nullCounts, nanCounts, lowerBounds, upperBounds); return DataFiles.builder(PartitionSpec.unpartitioned()) .withPath("/data/jol-" + index + ".parquet") .withFileSizeInBytes(1024L) .withMetrics(metrics) - .build(); + .build() + .copy(); } private org.apache.iceberg.DataFile dataFileWithPathPayload(int pathLength) { @@ -1247,12 +2058,198 @@ private org.apache.iceberg.DataFile dataFileWithPathPayload(int pathLength) { .build(); } + private DeleteFile deleteFileWithMetrics(int index) { + Map columnSizes = metricLongMap(index, 0); + Map valueCounts = metricLongMap(index, 1); + Map nullCounts = metricLongMap(index, 2); + Map nanCounts = metricLongMap(index, 3); + Map lowerBounds = metricBufferMap(); + Map upperBounds = metricBufferMap(); + Metrics metrics = new Metrics( + 100L, columnSizes, valueCounts, nullCounts, nanCounts, lowerBounds, upperBounds); + return FileMetadata.deleteFileBuilder(PartitionSpec.unpartitioned()) + .ofPositionDeletes() + .withPath("/delete/jol-" + index + ".parquet") + .withFileSizeInBytes(1024L) + .withRecordCount(1L) + .withReferencedDataFile("/data/jol-" + index + ".parquet") + .withMetrics(metrics) + .build() + .copy(); + } + + private Map metricLongMap(int fileIndex, int mapIndex) { + Map values = new java.util.HashMap<>(); + for (int column = 0; column < 8; column++) { + values.put(Integer.valueOf(10_000 + column), + Long.valueOf(10_000L + fileIndex * 100L + mapIndex * 10L + column)); + } + return values; + } + + private Map metricBufferMap() { + Map values = new java.util.HashMap<>(); + for (int column = 0; column < 8; column++) { + values.put(Integer.valueOf(10_000 + column), ByteBuffer.allocate(32)); + } + return values; + } + + private List manifestFilesWithIntegerPartitions( + int fileCount, int partitionFieldCount) { + return manifestFilesWithPartitions( + fileCount, partitionFieldCount, Types.IntegerType.get(), + (fileIndex, fieldIndex) -> Integer.valueOf( + 10_000 + fileIndex * partitionFieldCount + fieldIndex)); + } + + private List manifestFilesWithPartitions( + int fileCount, int partitionFieldCount, Type.PrimitiveType partitionType, + BiFunction valueFactory) { + if (partitionFieldCount == 0) { + return IntStream.range(0, fileCount) + .mapToObj(index -> DataFiles.builder(PartitionSpec.unpartitioned()) + .withPath("/partition-data/file-" + index + ".parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build()) + .collect(Collectors.toList()); + } + List fields = IntStream.range(0, partitionFieldCount) + .mapToObj(index -> Types.NestedField.optional( + index + 1, "partition_" + index, partitionType)) + .collect(Collectors.toList()); + Schema schema = new Schema(fields); + PartitionSpec.Builder specBuilder = PartitionSpec.builderFor(schema); + fields.forEach(field -> specBuilder.identity(field.name())); + PartitionSpec spec = specBuilder.build(); + DataFiles.Builder fileBuilder = DataFiles.builder(spec); + PartitionData partitionData = new PartitionData(spec.partitionType()); + List files = new ArrayList<>(fileCount); + for (int fileIndex = 0; fileIndex < fileCount; fileIndex++) { + for (int fieldIndex = 0; fieldIndex < partitionFieldCount; fieldIndex++) { + partitionData.set(fieldIndex, valueFactory.apply(fileIndex, fieldIndex)); + } + files.add(fileBuilder.withPartition(partitionData) + .withPath("/partition-data/file-" + fileIndex + ".parquet") + .withFileSizeInBytes(10L) + .withRecordCount(1L) + .build()); + } + return files; + } + private Table tableWithMetadata(TableMetadata metadata) { TableOperations operations = Mockito.mock(TableOperations.class); Mockito.when(operations.current()).thenReturn(metadata); return new BaseTable(operations, "db.tbl"); } + private TableMetadata roundTripMetadata(TableMetadata metadata, String metadataLocation) { + return TableMetadataParser.fromJson(metadataLocation, TableMetadataParser.toJson(metadata)); + } + + private void materializeAllLazyState(IcebergTableCacheValue value) { + Table table = value.getRetainedIcebergTable(); + materializeAllLazyState(((HasTableOperations) table).operations().current()); + } + + private void materializeAllLazyState(TableMetadata metadata) { + for (Schema schema : metadata.schemas()) { + materializeSchemaAndStruct(schema); + } + for (PartitionSpec spec : metadata.specs()) { + spec.fields(); + spec.javaClasses(); + Types.StructType partitionType = spec.partitionType(); + spec.rawPartitionType(); + if (!spec.fields().isEmpty()) { + spec.getFieldsBySourceId(spec.fields().get(0).sourceId()); + } + materializeStructAndSecondarySchema(partitionType); + } + } + + private void materializeSchemaAndStruct(Schema schema) { + if (schema.columns().isEmpty()) { + return; + } + Types.NestedField first = schema.columns().get(0); + materializeSchemaIndexes(schema, first); + materializeStructAndSecondarySchema(schema.asStruct()); + } + + private void materializeStructAndSecondarySchema(Types.StructType struct) { + if (struct.fields().isEmpty()) { + return; + } + Types.NestedField first = struct.fields().get(0); + materializeStructIndexes(struct, first); + Schema secondary = struct.asSchema(); + materializeSchemaIndexes(secondary, first); + materializeStructIndexes(secondary.asStruct(), first); + } + + private void materializeSchemaIndexes(Schema schema, Types.NestedField first) { + schema.findField(first.name()); + schema.findField(first.fieldId()); + schema.caseInsensitiveFindField(first.name().toUpperCase(java.util.Locale.ROOT)); + schema.idToName(); + schema.identifierFieldIds(); + schema.accessorForField(first.fieldId()); + } + + private void materializeStructIndexes(Types.StructType struct, Types.NestedField first) { + struct.fields(); + struct.field(first.name()); + struct.caseInsensitiveField(first.name().toUpperCase(java.util.Locale.ROOT)); + struct.field(first.fieldId()); + } + + /** + * Delta between the first two columns and the whole schema: measures the schema graph. Both + * sides reach the same shared type singletons (StringType, ListType element names, ...) so + * only per-column growth is compared. + */ + private void assertSchemaLookupFormula(Schema schema, String fixture) { + Schema firstColumns = new Schema(schema.schemaId(), schema.columns().subList(0, 2)); + TableMetadata empty = TableMetadata.newTableMetadata( + firstColumns, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-lookup", Collections.emptyMap()); + TableMetadata populated = TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-lookup", Collections.emptyMap()); + assertRetainedPayloadDelta(fixture, empty, populated, "jol-schema-lookup"); + } + + /** Delta between an unpartitioned spec and one identity field: measures the spec graph. */ + private void assertPartitionSpecFormula(Schema schema, String fixture) { + TableMetadata empty = TableMetadata.newTableMetadata( + schema, PartitionSpec.unpartitioned(), + "file:/warehouse/schema-lookup", Collections.emptyMap()); + TableMetadata populated = TableMetadata.newTableMetadata( + schema, PartitionSpec.builderFor(schema).identity(schema.columns().get(0).name()).build(), + "file:/warehouse/schema-lookup", Collections.emptyMap()); + assertRetainedPayloadDelta(fixture, empty, populated, "jol-partition-spec"); + } + + private void assertRetainedPayloadDelta( + String fixture, TableMetadata empty, TableMetadata populated, String locationPrefix) { + empty = roundTripMetadata(empty, + "/metadata/" + locationPrefix + "-none-" + fixture.hashCode() + ".json"); + populated = roundTripMetadata(populated, + "/metadata/" + locationPrefix + "-with-" + fixture.hashCode() + ".json"); + + long emptyEstimate = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(empty)); + long populatedEstimate = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(populated)); + materializeAllLazyState(empty); + materializeAllLazyState(populated); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg " + fixture, emptyEstimate, populatedEstimate, empty, populated); + } + private TableMetadata metadataWithMaterializedPayload(String payload, int bufferBytes) { TableMetadata metadata = Mockito.mock(TableMetadata.class); Mockito.when(metadata.schemas()).thenReturn(Collections.emptyList()); @@ -1289,6 +2286,42 @@ private TableMetadata metadataWithMaterializedPayload(String payload, int buffer return metadata; } + private TableMetadata metadataWithSnapshots(Snapshot snapshot) { + TableMetadata metadata = Mockito.mock(TableMetadata.class); + Mockito.when(metadata.schemas()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.specs()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.sortOrders()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.properties()).thenReturn(Collections.emptyMap()); + Mockito.when(metadata.snapshots()).thenReturn(Collections.singletonList(snapshot)); + Mockito.when(metadata.currentSnapshot()).thenReturn(snapshot); + Mockito.when(metadata.snapshotLog()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.previousFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.refs()).thenReturn(Collections.emptyMap()); + Mockito.when(metadata.statisticsFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.partitionStatisticsFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.encryptionKeys()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.metadataFileLocation()).thenReturn("/metadata/snapshot.json"); + return metadata; + } + + private TableMetadata metadataWithStatisticsFile(StatisticsFile statisticsFile) { + TableMetadata metadata = Mockito.mock(TableMetadata.class); + Mockito.when(metadata.schemas()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.specs()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.sortOrders()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.properties()).thenReturn(Collections.emptyMap()); + Mockito.when(metadata.snapshots()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.snapshotLog()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.previousFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.refs()).thenReturn(Collections.emptyMap()); + Mockito.when(metadata.statisticsFiles()).thenReturn( + Collections.singletonList(statisticsFile)); + Mockito.when(metadata.partitionStatisticsFiles()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.encryptionKeys()).thenReturn(Collections.emptyList()); + Mockito.when(metadata.metadataFileLocation()).thenReturn("/metadata/statistics.json"); + return metadata; + } + private TableMetadata metadataWithSnapshotSequence(long lastSequenceNumber) { TableMetadata metadata = Mockito.mock(TableMetadata.class); Mockito.when(metadata.schemas()).thenReturn(Collections.emptyList()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java index 84745f815f828c..c184debd3d70ac 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfoTest.java @@ -17,6 +17,8 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; + import com.google.common.collect.Maps; import com.google.common.collect.Sets; import org.junit.jupiter.api.Assertions; @@ -36,8 +38,12 @@ public void testRetainedPayloadCounterTracksSkewedPartitionValues() { IcebergPartition large = new IcebergPartition("p=" + largeValue, 0, 0, 0, 0, 1, 101, Collections.singletonList(largeValue), Collections.singletonList("identity")); + long expectedDelta = MetaCacheWeightUtils.estimatedStringBytes("p=" + largeValue) + - MetaCacheWeightUtils.estimatedStringBytes("p=x") + + MetaCacheWeightUtils.estimatedStringBytes(largeValue) + - MetaCacheWeightUtils.estimatedStringBytes("x"); Assertions.assertTrue(large.getRetainedPayloadBytes() - small.getRetainedPayloadBytes() - >= (largeValue.length() - 1L) * 4L); + >= expectedDelta); IcebergPartitionInfo info = new IcebergPartitionInfo( Collections.emptyMap(), Collections.singletonMap(large.getPartitionName(), large), Collections.emptyMap()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java index be4978d33c129d..d4215518c86356 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java @@ -19,12 +19,41 @@ import org.junit.Assert; import org.openjdk.jol.info.GraphLayout; +import org.openjdk.jol.info.GraphPathRecord; + +import java.lang.reflect.Field; +import java.util.stream.IntStream; +import java.util.stream.LongStream; /** JOL oracle used only by estimator calibration tests. */ public final class EstimatorCalibrationAssertions { - private static final long MAX_CONSERVATIVE_FACTOR = 8L; + private static final double MAX_CONSERVATIVE_FACTOR = 1.10D; private static final boolean PRINT_RESULT = Boolean.getBoolean( "metacache.estimator.calibration.print"); + // Integer.valueOf/Long.valueOf serve -128..127 from JVM-wide static caches. A populated + // fixture that reaches those shared instances (field ids, list indexes, small partition + // values) must not be charged for them as retained growth, so every graph is measured + // together with the same cache roots and the shared instances cancel out of the delta. + private static final Integer[] SHARED_INTEGER_CACHE = + IntStream.rangeClosed(-128, 127).boxed().toArray(Integer[]::new); + private static final Long[] SHARED_LONG_CACHE = + LongStream.rangeClosed(-128L, 127L).boxed().toArray(Long[]::new); + // Accessor objects reference java.lang.Class instances (String.class, StructLike.class, ...). + // JOL follows them into the JVM's per-class reflection and ClassValue caches, whose size + // depends on unrelated reflective use earlier in the same JVM (Mockito, layout fingerprints, + // JOL itself). Everything reached through a Class object is shared JVM state, not retained + // cache payload, and is excluded from every measurement. + private static final Field GRAPH_PATH_PARENT = graphPathParentField(); + + private static Field graphPathParentField() { + try { + Field field = GraphPathRecord.class.getDeclaredField("parent"); + field.setAccessible(true); + return field; + } catch (ReflectiveOperationException e) { + throw new IllegalStateException("JOL GraphPathRecord.parent is unavailable", e); + } + } static { // Doris expression graphs contain JVM hidden lambda classes. JOL cannot obtain their @@ -43,8 +72,7 @@ private EstimatorCalibrationAssertions() { public static void assertConservativeDelta( String fixture, long emptyEstimate, long populatedEstimate, Object emptyGraph, Object populatedGraph) { - long actualDelta = GraphLayout.parseInstance(populatedGraph).totalSize() - - GraphLayout.parseInstance(emptyGraph).totalSize(); + long actualDelta = graphSize(populatedGraph) - graphSize(emptyGraph); long estimatedDelta = populatedEstimate - emptyEstimate; if (PRINT_RESULT) { System.out.printf("%s: estimated=%d, jol=%d, ratio=%.3f%n", @@ -57,11 +85,36 @@ public static void assertConservativeDelta( estimatedDelta >= actualDelta); Assert.assertTrue(fixture + " estimate is excessively conservative: estimated=" + estimatedDelta + ", actual=" + actualDelta, - estimatedDelta <= MetaCacheWeightUtils.saturatedMultiply( - actualDelta, MAX_CONSERVATIVE_FACTOR)); + estimatedDelta <= Math.ceil(actualDelta * MAX_CONSERVATIVE_FACTOR)); } + /** Retained size of the graph excluding JVM-shared boxed-value caches and Class metadata. */ public static long graphSize(Object graph) { - return GraphLayout.parseInstance(graph).totalSize(); + long sharedCacheBytes = GraphLayout.parseInstance( + SHARED_INTEGER_CACHE, SHARED_LONG_CACHE).totalSize(); + GraphLayout layout = GraphLayout.parseInstance( + graph, SHARED_INTEGER_CACHE, SHARED_LONG_CACHE); + long bytes = 0L; + for (long address : layout.addresses()) { + GraphPathRecord record = layout.record(address); + if (!reachedThroughClassObject(record)) { + bytes += record.size(); + } + } + return bytes - sharedCacheBytes; + } + + private static boolean reachedThroughClassObject(GraphPathRecord record) { + try { + for (GraphPathRecord current = record; current != null; + current = (GraphPathRecord) GRAPH_PATH_PARENT.get(current)) { + if (current.klass() == Class.class) { + return true; + } + } + return false; + } catch (IllegalAccessException e) { + throw new IllegalStateException(e); + } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java index 41d575417654e5..e3052fc42f31fd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java @@ -46,6 +46,18 @@ public class MetaCacheEntryTest { + @Test + public void testCompactStringPayloadEstimate() { + // Three Latin-1 bytes and two UTF-16 characters both occupy one aligned slot; the exact + // slot size follows the JVM object alignment (8 by default, 16 with large heaps). + long latin1 = MetaCacheWeightUtils.estimatedStringPayloadBytes("abc"); + long utf16 = MetaCacheWeightUtils.estimatedStringPayloadBytes("中文"); + Assert.assertEquals(latin1, utf16); + Assert.assertTrue(latin1 >= 4L && latin1 <= 16L); + Assert.assertTrue(MetaCacheWeightUtils.estimatedStringPayloadBytes("abcdefghijklmnopq") + > latin1); + } + @Test public void testRefreshUsesConfiguredLoader() throws Exception { boolean originalManualMissLoad = Config.enable_external_meta_cache_manual_miss_load; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index fa016361c9ad3b..99bce8a3259c40 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -31,6 +31,7 @@ import org.apache.doris.datasource.metacache.EstimatorCalibrationAssertions; import org.apache.doris.datasource.metacache.MetaCacheEntryStats; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.doris.datasource.metacache.paimon.PaimonLatestSnapshotProjectionLoader; import org.apache.doris.datasource.metacache.paimon.PaimonPartitionInfoLoader; @@ -55,10 +56,19 @@ import org.apache.paimon.table.sink.StreamTableWrite; import org.apache.paimon.table.source.ReadBuilder; import org.apache.paimon.table.source.TableScan; +import org.apache.paimon.types.ArrayType; import org.apache.paimon.types.DataField; +import org.apache.paimon.types.DataType; +import org.apache.paimon.types.DataTypeRoot; +import org.apache.paimon.types.DataTypeVisitor; import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.DecimalType; +import org.apache.paimon.types.FloatType; import org.apache.paimon.types.IntType; +import org.apache.paimon.types.MapType; +import org.apache.paimon.types.MultisetType; import org.apache.paimon.types.RowType; +import org.apache.paimon.types.VectorType; import org.junit.Assert; import org.junit.Assume; import org.junit.Rule; @@ -115,7 +125,9 @@ public void testSnapshotWeightAccountsForSkewedTableOptions() throws Exception { long smallBytes = snapshotWeight(smallKey, smallTable, 0); long largeBytes = snapshotWeight(largeKey, largeTable, 0); - Assert.assertTrue(largeBytes - smallBytes >= (largePayload.length() - 1L) * 2L); + Assert.assertTrue(largeBytes - smallBytes + >= MetaCacheWeightUtils.estimatedStringBytes(largePayload) + - MetaCacheWeightUtils.estimatedStringBytes("x")); } @Test @@ -133,7 +145,9 @@ public void testSnapshotWeightAccountsForNestedSchemaPayload() throws Exception long smallBytes = snapshotWeight(smallKey, smallTable, 0); long largeBytes = snapshotWeight(largeKey, largeTable, 0); - Assert.assertTrue(largeBytes - smallBytes >= (largeFieldName.length() - 1L) * 2L); + Assert.assertTrue(largeBytes - smallBytes + >= MetaCacheWeightUtils.estimatedStringBytes(largeFieldName) + - MetaCacheWeightUtils.estimatedStringBytes("x")); } @Test @@ -150,19 +164,25 @@ public void testSnapshotWeightAccountsForTableComment() throws Exception { long largeBytes = snapshotWeight(new PaimonSnapshotEntryKey( mapping, 1L, largeTable.schema().id(), 1L), largeTable, 0); - Assert.assertTrue(largeBytes - smallBytes >= (largeComment.length() - 1L) * 2L); + Assert.assertTrue(largeBytes - smallBytes + >= MetaCacheWeightUtils.estimatedStringBytes(largeComment) + - MetaCacheWeightUtils.estimatedStringBytes("x")); } @Test public void testSnapshotFormulaAgainstJolOwnedGraph() throws Exception { - FileStoreTable table = newPartitionedTable("jol_snapshot", Collections.emptyMap()); + FileStoreTable table = newStringPartitionedTable("jol_snapshot"); + FileStoreTable intTable = newPartitionedTable("jol_int_snapshot", Collections.emptyMap()); NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey( mapping, 1L, table.schema().id(), 1L); - PaimonSnapshotCacheValue empty = snapshotValueWithRealPartitions(table, 0, 16); - PaimonSnapshotCacheValue populated = snapshotValueWithRealPartitions(table, 32, 16); - PaimonSnapshotCacheValue shortTail = snapshotValueWithRealPartitions(table, 1, 16); - PaimonSnapshotCacheValue longTail = snapshotValueWithRealPartitions(table, 1, 4096); + PaimonSnapshotCacheValue empty = snapshotValueWithRealPartitions(table, 0, 16, Type.STRING); + PaimonSnapshotCacheValue populated = snapshotValueWithRealPartitions( + table, 32, 16, Type.STRING); + PaimonSnapshotCacheValue shortTail = snapshotValueWithRealPartitions( + table, 1, 16, Type.STRING); + PaimonSnapshotCacheValue longTail = snapshotValueWithRealPartitions( + table, 1, 4096, Type.STRING); long emptyEstimate = empty.prepareForCachePublication(key).getBytes(); long populatedEstimate = populated.prepareForCachePublication(key).getBytes(); @@ -173,6 +193,113 @@ public void testSnapshotFormulaAgainstJolOwnedGraph() throws Exception { "paimon snapshot partitions", emptyEstimate, populatedEstimate, empty, populated); EstimatorCalibrationAssertions.assertConservativeDelta( "paimon long-tail partition", shortTailEstimate, longTailEstimate, shortTail, longTail); + + PaimonSnapshotEntryKey intKey = new PaimonSnapshotEntryKey( + mapping, 1L, intTable.schema().id(), 1L); + PaimonSnapshotCacheValue emptyInts = snapshotValueWithRealPartitions( + intTable, 0, 0, Type.INT); + PaimonSnapshotCacheValue populatedInts = snapshotValueWithRealPartitions( + intTable, 32, 0, Type.INT); + EstimatorCalibrationAssertions.assertConservativeDelta( + "paimon int snapshot partitions", + emptyInts.prepareForCachePublication(intKey).getBytes(), + populatedInts.prepareForCachePublication(intKey).getBytes(), + emptyInts, populatedInts); + } + + @Test + public void testTableSchemaFormulaAgainstJolOwnedGraph() throws Exception { + FileStoreTable emptyTable = newTableWithExtraFields("jol_empty_schema", 0); + FileStoreTable populatedTable = newTableWithExtraFields("jol_populated_schema", 32); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey emptyKey = new PaimonSnapshotEntryKey( + mapping, 1L, emptyTable.schema().id(), 1L); + PaimonSnapshotEntryKey populatedKey = new PaimonSnapshotEntryKey( + mapping, 1L, populatedTable.schema().id(), 1L); + PaimonSnapshotCacheValue empty = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, + new PaimonSnapshot(1L, emptyTable.schema().id(), emptyTable)); + PaimonSnapshotCacheValue populated = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, + new PaimonSnapshot(1L, populatedTable.schema().id(), populatedTable)); + + long emptyEstimate = empty.prepareForCachePublication(emptyKey).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(populatedKey).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "paimon table fields", emptyEstimate, populatedEstimate, empty, populated); + } + + @Test + public void testNestedTableSchemaFormulaAgainstJolOwnedGraph() throws Exception { + FileStoreTable smallTable = newTableWithNestedFields("jol_nested_small", 1); + FileStoreTable populatedTable = newTableWithNestedFields("jol_nested_large", 33); + assertTableDeltaAgainstJol("paimon nested fields", smallTable, populatedTable); + } + + @Test + public void testTableOptionFormulaAgainstJolOwnedGraph() throws Exception { + FileStoreTable emptyTable = newTableWithOptions("jol_options_empty", 0); + FileStoreTable populatedTable = newTableWithOptions("jol_options_large", 32); + assertTableDeltaAgainstJol("paimon table options", emptyTable, populatedTable); + } + + @Test + public void testCompositeTypeFormulaAgainstJolOwnedGraph() { + assertTableDeltaAgainstJol("paimon array type", + newTableWithPayloadType("array", nestedArrayType(1)), + newTableWithPayloadType("array", nestedArrayType(100))); + assertTableDeltaAgainstJol("paimon map type", + newTableWithPayloadType("map", nestedMapType(1)), + newTableWithPayloadType("map", nestedMapType(100))); + assertTableDeltaAgainstJol("paimon multiset type", + newTableWithPayloadType("multiset", nestedMultisetType(1)), + newTableWithPayloadType("multiset", nestedMultisetType(100))); + assertTableDeltaAgainstJol("paimon row type", + newTableWithPayloadType("row", nestedRowType(1)), + newTableWithPayloadType("row", nestedRowType(100))); + assertTableDeltaAgainstJol("paimon vector type", + newTableWithPayloadType("vector", rowOfLeafTypes(1, VectorType.class)), + newTableWithPayloadType("vector", rowOfLeafTypes(100, VectorType.class))); + assertTableDeltaAgainstJol("paimon decimal type", + newTableWithPayloadType("decimal", rowOfLeafTypes(1, DecimalType.class)), + newTableWithPayloadType("decimal", rowOfLeafTypes(100, DecimalType.class))); + } + + @Test + public void testUnknownDataTypeFailsClosedWithoutFailingLoad() { + DataType unknownType = new DataType(true, DataTypeRoot.INTEGER) { + @Override + public int defaultSize() { + return Integer.BYTES; + } + + @Override + public DataType copy(boolean isNullable) { + return this; + } + + @Override + public String asSQLString() { + return "UNKNOWN"; + } + + @Override + public R accept(DataTypeVisitor visitor) { + return new IntType().accept(visitor); + } + }; + FileStoreTable table = newTableWithPayloadType("unknown-type", unknownType); + Assert.assertThrows(IllegalStateException.class, + () -> PaimonCacheSizeEstimator.retainedTablePayloadBytes(table)); + + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey(mapping, 1L, table.schema().id(), 1L); + PaimonSnapshotCacheValue value = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, table.schema().id(), table)); + MetaCacheSizeEstimate estimate = value.prepareForCachePublication(key); + + Assert.assertFalse(estimate.isComplete()); + Assert.assertSame(table, value.getSnapshot().getTable()); } @Test @@ -209,6 +336,48 @@ public void testSnapshotWeightEntryAndPrecomputedEstimate() throws Exception { } } + @Test + public void testTablePayloadAccountingWorkIsBounded() { + FileStoreTable table = Mockito.mock(FileStoreTable.class); + TableSchema schema = Mockito.mock(TableSchema.class); + @SuppressWarnings("unchecked") + Map oversizedOptions = Mockito.mock(Map.class); + Mockito.when(table.schema()).thenReturn(schema); + Mockito.when(schema.fields()).thenReturn(Collections.emptyList()); + Mockito.when(schema.options()).thenReturn(oversizedOptions); + Mockito.when(oversizedOptions.size()).thenReturn(50_001); + + Assert.assertThrows(IllegalStateException.class, + () -> PaimonCacheSizeEstimator.retainedTablePayloadBytes(table)); + } + + @Test + public void testRowTypeLazyLookupReservationCoversPostAdmissionGrowth() throws Exception { + // Field ids above the Integer cache make every lazy map box its keys and values. + RowType small = wideRowType(1); + RowType populated = wideRowType(200); + FileStoreTable smallTable = newTableWithPayloadType("row-lazy-small", small); + FileStoreTable populatedTable = newTableWithPayloadType("row-lazy-large", populated); + for (String fieldName : ROW_TYPE_LAZY_FIELDS) { + Assert.assertNull(fieldName, readField(populated, fieldName)); + } + + // The oracle inside assertTableDeltaAgainstJol materializes the four maps after the + // estimate is taken; the estimate reserved at admission must already cover them. + assertTableDeltaAgainstJol("paimon row lazy lookup maps", smallTable, populatedTable); + for (String fieldName : ROW_TYPE_LAZY_FIELDS) { + Assert.assertNotNull(fieldName, readField(populated, fieldName)); + } + + // A RowType whose maps were materialized before admission is estimated identically. + RowType preloaded = wideRowType(200); + long unloadedEstimate = PaimonCacheSizeEstimator.retainedTablePayloadBytes( + newTableWithPayloadType("row-unloaded", wideRowType(200))); + materializeRowTypeIndexes(preloaded); + Assert.assertEquals(unloadedEstimate, PaimonCacheSizeEstimator.retainedTablePayloadBytes( + newTableWithPayloadType("row-loaded", preloaded))); + } + @Test public void testSnapshotKeySeparatesReloadedTableGenerations() { NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); @@ -786,6 +955,24 @@ private FileStoreTable newPartitionedTable( CatalogEnvironment.empty()); } + private FileStoreTable newStringPartitionedTable(String name) throws Exception { + TableSchema schema = new TableSchema( + 0, + java.util.Arrays.asList( + new DataField(0, "id", new IntType()), + new DataField(1, "part", DataTypes.STRING())), + 1, + Collections.singletonList("part"), + Collections.emptyList(), + Collections.emptyMap(), + null); + return new AppendOnlyFileStoreTable( + LocalFileIO.create(), + new Path(temporaryFolder.newFolder(name).toURI()), + schema, + CatalogEnvironment.empty()); + } + private FileStoreTable newPartitionedTableWithNestedField( String name, String nestedFieldName) throws Exception { RowType nestedType = new RowType(Collections.singletonList( @@ -807,6 +994,173 @@ private FileStoreTable newPartitionedTableWithNestedField( CatalogEnvironment.empty()); } + private FileStoreTable newTableWithExtraFields(String name, int fieldCount) throws Exception { + ArrayList fields = new ArrayList<>(); + fields.add(new DataField(0, "part", new IntType())); + for (int index = 0; index < fieldCount; index++) { + fields.add(new DataField(index + 1, "field_" + index, new IntType())); + } + TableSchema schema = new TableSchema( + 0, + fields, + 1, + Collections.singletonList("part"), + Collections.emptyList(), + Collections.emptyMap(), + null); + return new AppendOnlyFileStoreTable( + LocalFileIO.create(), + new Path(temporaryFolder.newFolder(name).toURI()), + schema, + CatalogEnvironment.empty()); + } + + private FileStoreTable newTableWithNestedFields(String name, int nestedFieldCount) throws Exception { + ArrayList nestedFields = new ArrayList<>(); + for (int index = 0; index < nestedFieldCount; index++) { + nestedFields.add(new DataField(index + 2, "nested_" + index, new IntType())); + } + RowType nestedType = new RowType(nestedFields); + TableSchema schema = new TableSchema( + 0, + java.util.Arrays.asList( + new DataField(0, "payload", nestedType), + new DataField(1, "part", new IntType())), + 1, + Collections.singletonList("part"), + Collections.emptyList(), + Collections.emptyMap(), + null); + return new AppendOnlyFileStoreTable( + LocalFileIO.create(), new Path(temporaryFolder.newFolder(name).toURI()), + schema, CatalogEnvironment.empty()); + } + + private FileStoreTable newTableWithOptions(String name, int optionCount) throws Exception { + Map options = new HashMap<>(); + for (int index = 0; index < optionCount; index++) { + options.put("key_" + index, "value_" + index); + } + return newPartitionedTable(name, options); + } + + private FileStoreTable newTableWithPayloadType(String name, DataType type) { + TableSchema schema = new TableSchema( + 0, Collections.singletonList(new DataField(0, "payload", type)), 1, + Collections.emptyList(), Collections.emptyList(), Collections.emptyMap(), null); + return new AppendOnlyFileStoreTable( + LocalFileIO.create(), new Path("file:/tmp/paimon-composite-" + name), + schema, CatalogEnvironment.empty()); + } + + private DataType nestedArrayType(int depth) { + DataType type = new IntType(); + for (int index = 0; index < depth; index++) { + type = new ArrayType(type); + } + return type; + } + + private DataType nestedMapType(int depth) { + DataType type = new IntType(); + for (int index = 0; index < depth; index++) { + type = new MapType(new IntType(), type); + } + return type; + } + + private DataType nestedMultisetType(int depth) { + DataType type = new IntType(); + for (int index = 0; index < depth; index++) { + type = new MultisetType(type); + } + return type; + } + + private DataType nestedRowType(int depth) { + DataType type = new IntType(); + for (int index = 0; index < depth; index++) { + type = new RowType(Collections.singletonList( + new DataField(index + 1, "nested_" + index, type))); + } + return type; + } + + private void assertTableDeltaAgainstJol( + String fixture, FileStoreTable smallTable, FileStoreTable populatedTable) { + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotEntryKey smallKey = new PaimonSnapshotEntryKey( + mapping, 1L, smallTable.schema().id(), 1L); + PaimonSnapshotEntryKey populatedKey = new PaimonSnapshotEntryKey( + mapping, 1L, populatedTable.schema().id(), 1L); + PaimonSnapshotCacheValue small = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, + new PaimonSnapshot(1L, smallTable.schema().id(), smallTable)); + PaimonSnapshotCacheValue populated = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, + new PaimonSnapshot(1L, populatedTable.schema().id(), populatedTable)); + long smallEstimate = small.prepareForCachePublication(smallKey).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(populatedKey).getBytes(); + // The estimate reserves the lookup maps every nested RowType can materialize after + // admission, so the JOL oracle measures the fully grown graph. + materializeRowTypeIndexes(smallTable.schema()); + materializeRowTypeIndexes(populatedTable.schema()); + EstimatorCalibrationAssertions.assertConservativeDelta( + fixture, smallEstimate, populatedEstimate, small, populated); + } + + private static final String[] ROW_TYPE_LAZY_FIELDS = { + "laziedNameToField", "laziedNameToIndex", "laziedFieldIdToField", "laziedFieldIdToIndex"}; + + private void materializeRowTypeIndexes(TableSchema schema) { + for (DataField field : schema.fields()) { + materializeRowTypeIndexes(field.type()); + } + } + + private void materializeRowTypeIndexes(DataType type) { + if (type instanceof RowType) { + RowType rowType = (RowType) type; + if (!rowType.getFields().isEmpty()) { + DataField first = rowType.getFields().get(0); + rowType.getField(first.name()); + rowType.getFieldIndex(first.name()); + rowType.getField(first.id()); + rowType.getFieldIndexByFieldId(first.id()); + } + for (DataField field : rowType.getFields()) { + materializeRowTypeIndexes(field.type()); + } + } else if (type instanceof ArrayType) { + materializeRowTypeIndexes(((ArrayType) type).getElementType()); + } else if (type instanceof MultisetType) { + materializeRowTypeIndexes(((MultisetType) type).getElementType()); + } else if (type instanceof MapType) { + materializeRowTypeIndexes(((MapType) type).getKeyType()); + materializeRowTypeIndexes(((MapType) type).getValueType()); + } else if (type instanceof VectorType) { + materializeRowTypeIndexes(((VectorType) type).getElementType()); + } + } + + private RowType wideRowType(int fieldCount) { + ArrayList fields = new ArrayList<>(); + for (int index = 0; index < fieldCount; index++) { + fields.add(new DataField(1000 + index, "wide_" + index, new IntType())); + } + return new RowType(fields); + } + + private DataType rowOfLeafTypes(int fieldCount, Class leafType) { + ArrayList fields = new ArrayList<>(); + for (int index = 0; index < fieldCount; index++) { + DataType type = leafType == VectorType.class + ? new VectorType(4, new FloatType()) : new DecimalType(10, 2); + fields.add(new DataField(index + 1, "leaf_" + index, type)); + } + return new RowType(fields); + } + private long snapshotWeight(PaimonSnapshotEntryKey key, FileStoreTable table, int partitionCount) { PaimonPartitionInfo partitionInfo = Mockito.mock(PaimonPartitionInfo.class); Map partitionItems = sizeOnlyMap(partitionCount); @@ -821,14 +1175,17 @@ private long snapshotWeight(PaimonSnapshotEntryKey key, FileStoreTable table, in } private PaimonSnapshotCacheValue snapshotValueWithRealPartitions( - FileStoreTable table, int partitionCount, int valueLength) { + FileStoreTable table, int partitionCount, int valueLength, Type partitionType) + throws AnalysisException { Map partitionItems = new HashMap<>(); Map partitions = new HashMap<>(); for (int index = 0; index < partitionCount; index++) { - String value = "p" + index + repeatedCharacter('x', valueLength); + String value = partitionType == Type.INT + ? Integer.toString(index) + : "p" + index + repeatedCharacter('x', valueLength); String name = "part=" + value; - partitionItems.put(name, new org.apache.doris.catalog.ListPartitionItem( - new ArrayList<>())); + partitionItems.put(name, PaimonUtil.toListPartitionItem( + Collections.singletonList(value), Collections.singletonList(partitionType))); partitions.put(name, new org.apache.paimon.partition.Partition( Collections.singletonMap("part", value), 100L, 1024L, 1L, 1L, 1, true)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java index 17fa76bb5593bc..97597d774d85ed 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java @@ -26,6 +26,7 @@ import org.apache.doris.catalog.Type; import org.apache.doris.catalog.VariantType; import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.doris.datasource.metacache.paimon.PaimonPartitionInfoLoader; import org.apache.doris.thrift.TPrimitiveType; import org.apache.doris.thrift.schema.external.TFieldPtr; @@ -98,7 +99,7 @@ public void testCompatibilityConstructorDerivesRetainedPartitionPayload() { PaimonPartitionInfo info = new PaimonPartitionInfo( Collections.emptyMap(), Collections.singletonMap("part=" + largeValue, partition)); - Assert.assertTrue(info.getRetainedPayloadBytes() >= largeValue.length() * 4L); + Assert.assertTrue(info.getRetainedPayloadBytes() >= largeValue.length() * 2L); } @Test @@ -261,7 +262,7 @@ public void testGeneratePartitionInfoWithSpecialCharacters() { Assert.assertEquals(1, partitionInfo.getNameToPartitionItem().size()); String partitionName = "source=dataset%2Fteam-a%2Fsegment-01" + "/part_str=%2Fymd%3D20260701%2Fhour%3D%5B0-9%5D%5B0-9%5D%2F%2A.jsonl/pass=s1"; - Assert.assertTrue(partitionInfo.getRetainedPayloadBytes() > partitionName.length() * 2L); + Assert.assertTrue(partitionInfo.getRetainedPayloadBytes() > partitionName.length()); Assert.assertTrue(partitionInfo.getNameToPartition().containsKey(partitionName)); PartitionItem partitionItem = partitionInfo.getNameToPartitionItem().values().iterator().next(); List actualValues = ((ListPartitionItem) partitionItem).getItems().get(0) @@ -284,7 +285,8 @@ public void testRetainedPayloadCounterTracksSkewedPartitionValues() { Collections.singletonList(partitionEntry(stringPartitionRow(largeValue), 1L))); Assert.assertTrue(large.getRetainedPayloadBytes() - small.getRetainedPayloadBytes() - >= (largeValue.length() - 1L) * 4L); + >= MetaCacheWeightUtils.estimatedStringBytes(largeValue) + - MetaCacheWeightUtils.estimatedStringBytes("x")); } @Test From 3d978950c8b545442dc2b631e7eab7f92a313b98 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Mon, 17 Aug 2026 23:36:55 +0800 Subject: [PATCH 04/45] [fix](fe) Address review findings on cache property lifecycle, statistics and partition-width accounting - Tolerate legacy engine namespaces in persisted cache properties during replay, and validate an ALTER strictly only for the keys it supplies while persisted keys are reduced to what runtime honors; CREATE stays strict for every supplied key. - Re-prepare a catalog whose cache group was retired between preparation and lookup, without blocking on the lifecycle fence. - Expose weight-bound statistics (bounds, estimated/eviction weight, rejections, reason) in information_schema.catalog_meta_cache_statistics on FE and BE. - Require the same table UUID before a retained Iceberg generation retries a commit, and bind snapshot-selectable Iceberg system tables to the relation's frozen generation for their schema. - Charge Iceberg range-partition items and Paimon list-partition items per partition column, and recalibrate the per-partition constants against realistic fixtures. --- ...chema_catalog_meta_cache_stats_scanner.cpp | 10 ++ .../org/apache/doris/catalog/SchemaTable.java | 12 ++ .../apache/doris/datasource/CatalogMgr.java | 36 ++++++ .../doris/datasource/ExternalCatalog.java | 19 +++- .../datasource/ExternalMetaCacheMgr.java | 97 ++++++++++++++++ .../iceberg/IcebergCacheSizeEstimator.java | 4 +- .../iceberg/IcebergPartitionInfo.java | 26 +++++ .../iceberg/IcebergSnapshotCacheValue.java | 9 +- .../iceberg/IcebergSysExternalTable.java | 32 +++++- .../datasource/iceberg/IcebergUtils.java | 4 + .../metacache/AbstractExternalMetaCache.java | 106 ++++++++++++------ .../metacache/ExternalMetaCache.java | 25 +++++ .../paimon/PaimonCacheSizeEstimator.java | 4 +- .../paimon/PaimonPartitionInfo.java | 17 +++ .../doris/datasource/paimon/PaimonUtil.java | 3 + .../tablefunction/MetadataGenerator.java | 18 +++ .../ExternalMetaCacheRouteResolverTest.java | 69 ++++++++++++ .../iceberg/IcebergExternalMetaCacheTest.java | 35 +++++- .../iceberg/IcebergSysExternalTableTest.java | 43 +++++++ .../iceberg/IcebergTransactionTest.java | 25 +++++ .../paimon/PaimonExternalMetaCacheTest.java | 33 +++++- .../test_iceberg_table_meta_cache.groovy | 13 +++ 22 files changed, 584 insertions(+), 56 deletions(-) diff --git a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp index 7e72e6bd1a777a..bffbf40a3e0ca9 100644 --- a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp +++ b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp @@ -53,6 +53,16 @@ std::vector SchemaCatalogMetaCacheStatsScanner::_s_tb {"LAST_LOAD_SUCCESS_TIME", TYPE_STRING, sizeof(StringRef), true}, {"LAST_LOAD_FAILURE_TIME", TYPE_STRING, sizeof(StringRef), true}, {"LAST_ERROR", TYPE_STRING, sizeof(StringRef), true}, + {"WEIGHT_BOUNDED", TYPE_BOOLEAN, sizeof(bool), true}, + {"MAX_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"ESTIMATED_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"EVICTION_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"WEIGHT_REJECT_COUNT", TYPE_BIGINT, sizeof(int64_t), true}, + {"CATALOG_MAX_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"CATALOG_ESTIMATED_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"GLOBAL_MAX_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"GLOBAL_ESTIMATED_WEIGHT", TYPE_BIGINT, sizeof(int64_t), true}, + {"LAST_WEIGHT_REJECT_REASON", TYPE_STRING, sizeof(StringRef), true}, }; SchemaCatalogMetaCacheStatsScanner::SchemaCatalogMetaCacheStatsScanner() diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java index cb4ef35eb002fd..81160fb4e382ae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/SchemaTable.java @@ -635,6 +635,18 @@ public class SchemaTable extends Table { .column("LAST_LOAD_SUCCESS_TIME", ScalarType.createStringType()) .column("LAST_LOAD_FAILURE_TIME", ScalarType.createStringType()) .column("LAST_ERROR", ScalarType.createStringType()) + .column("WEIGHT_BOUNDED", ScalarType.createType(PrimitiveType.BOOLEAN)) + .column("MAX_WEIGHT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("ESTIMATED_WEIGHT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("EVICTION_WEIGHT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("WEIGHT_REJECT_COUNT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("CATALOG_MAX_WEIGHT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("CATALOG_ESTIMATED_WEIGHT", + ScalarType.createType(PrimitiveType.BIGINT)) + .column("GLOBAL_MAX_WEIGHT", ScalarType.createType(PrimitiveType.BIGINT)) + .column("GLOBAL_ESTIMATED_WEIGHT", + ScalarType.createType(PrimitiveType.BIGINT)) + .column("LAST_WEIGHT_REJECT_REASON", ScalarType.createStringType()) .build()) ) .put("backend_kerberos_ticket_cache", diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 3e8a8f813571ea..1ab521987e08f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -539,6 +539,7 @@ private void createCatalogInternal(CatalogIf catalog, boolean isReplay) throws D try { if (!isReplay && catalog instanceof ExternalCatalog) { ((ExternalCatalog) catalog).checkProperties(); + validateSuppliedCacheProperties((ExternalCatalog) catalog, catalog.getProperties()); } Map props = catalog.getProperties(); if (props.containsKey(METADATA_REFRESH_INTERVAL_SEC)) { @@ -623,6 +624,40 @@ public List listCatalogsWithCheckPriv(UserIdentity userIdentity) { } + /** CREATE: every supplied external meta cache property is validated strictly. */ + private static void validateSuppliedCacheProperties(ExternalCatalog catalog, Map properties) + throws DdlException { + Env currentEnv = Env.getCurrentEnv(); + ExternalMetaCacheMgr cacheMgr = currentEnv == null ? null : currentEnv.getExtMetaCacheMgr(); + if (cacheMgr == null) { + return; + } + try { + cacheMgr.validateCatalogCacheProperties(catalog, properties); + } catch (IllegalArgumentException e) { + throw new DdlException(e.getMessage()); + } + } + + /** + * ALTER: newly supplied external meta cache properties are validated strictly, persisted + * ones only as runtime honors them, so a legacy key cannot block an unrelated update. + */ + private static void validateSuppliedCacheProperties(ExternalCatalog catalog, + Map persistedProperties, Map updatedProperties) + throws DdlException { + Env currentEnv = Env.getCurrentEnv(); + ExternalMetaCacheMgr cacheMgr = currentEnv == null ? null : currentEnv.getExtMetaCacheMgr(); + if (cacheMgr == null) { + return; + } + try { + cacheMgr.validateCatalogCachePropertyUpdate(catalog, persistedProperties, updatedProperties); + } catch (IllegalArgumentException e) { + throw new DdlException(e.getMessage()); + } + } + /** * Reply for alter catalog props event. */ @@ -637,6 +672,7 @@ public void replayAlterCatalogProps(CatalogLog log, Map oldPrope boolean tentativelyMutated = false; try { ExternalCatalog externalCatalog = (ExternalCatalog) catalog; + validateSuppliedCacheProperties(externalCatalog, oldProperties, newProps); boolean validatedWithoutMutation = externalCatalog.validatePropertiesBeforeUpdate( oldProperties, newProps); if (!validatedWithoutMutation) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java index f32c77a7015818..bc1e361a4a2c59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java @@ -454,7 +454,10 @@ protected void checkProperties(CatalogProperty property) throws DdlException { // This fallback is only for isolated construction tests before Env is initialized. ExternalMetaCacheBudgetManager.fromConfig().validateCatalogMaxWeight(properties); } else { - extMetaCacheMgr.validateCatalogCacheProperties(this, properties); + // Validate what runtime will honor. Newly supplied keys are validated strictly by + // CatalogMgr for CREATE and ALTER; persisted legacy keys that initialization + // ignores must not reject an unrelated later ALTER. + extMetaCacheMgr.validateEffectiveCatalogCacheProperties(this, properties); } } catch (IllegalArgumentException e) { throw new DdlException(e.getMessage()); @@ -1393,8 +1396,18 @@ public void notifyPropertiesUpdated(Map updatedProps) { } String remainder = key.substring("meta.cache.".length()); int separator = remainder.indexOf('.'); - if (separator > 0) { - extMetaCacheMgr.removeCatalogByEngine(id, remainder.substring(0, separator)); + if (separator <= 0) { + continue; + } + String engine = remainder.substring(0, separator); + try { + extMetaCacheMgr.removeCatalogByEngine(id, engine); + } catch (IllegalArgumentException e) { + // New DDL is validated before it reaches this notification. A persisted key with + // an unknown or legacy engine namespace (edit-log replay, image load) has no cache + // group to retire and must not abort the replay; runtime sanitization ignores it. + LOG.warn("Ignoring external meta cache property '{}' with unknown engine namespace '{}' " + + "for catalog {}: {}", key, engine, id, e.getMessage()); } } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index e58a984eacc462..316e3d3a9fd007 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -44,6 +44,7 @@ import org.apache.logging.log4j.Logger; import java.util.ArrayList; +import java.util.Collections; import java.util.Comparator; import java.util.List; import java.util.Map; @@ -320,6 +321,63 @@ public void validateCatalogCacheProperties(CatalogIf catalog, Map catalog, Map persistedProperties, + Map updatedProperties) { + validateCatalogCacheProperties(catalog, updatedProperties); + Map effective = runtimeEffectiveCacheProperties( + catalog, persistedProperties == null ? Collections.emptyMap() : persistedProperties); + effective.putAll(updatedProperties); + validateRuntimeCacheProperties(catalog, effective); + } + + /** + * Validate persisted properties as runtime will apply them: unknown engine namespaces and + * options no engine honors are dropped, then the remaining set is validated with the + * semantics initialization uses. + */ + public void validateEffectiveCatalogCacheProperties( + CatalogIf catalog, Map catalogProperties) { + validateRuntimeCacheProperties(catalog, runtimeEffectiveCacheProperties(catalog, catalogProperties)); + } + + private void validateRuntimeCacheProperties(CatalogIf catalog, Map effective) { + budgetManager.parseCatalogMaxWeight(effective); + for (ExternalMetaCache cache : routeResolver.resolveCatalogCaches(catalog.getId(), catalog)) { + cache.validateCatalogPropertiesForRuntime(effective); + } + } + + private Map runtimeEffectiveCacheProperties( + CatalogIf catalog, Map catalogProperties) { + Map effective = sanitizeCatalogCachePropertiesForRuntime( + catalog.getId(), catalogProperties); + List routedCaches = routeResolver.resolveCatalogCaches(catalog.getId(), catalog); + Set routedEngines = routedCaches.stream() + .map(ExternalMetaCache::engine) + .collect(Collectors.toSet()); + effective.keySet().removeIf(key -> { + if (key == null || ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY.equals(key) + || !key.startsWith("meta.cache.")) { + return false; + } + String remainder = key.substring("meta.cache.".length()); + int separator = remainder.indexOf('.'); + return separator <= 0 || !routedEngines.contains(remainder.substring(0, separator)); + }); + for (ExternalMetaCache cache : routedCaches) { + effective = cache.sanitizeCatalogPropertiesForRuntime(effective); + } + return effective; + } + public void invalidateCatalog(long catalogId) { routeCatalogEngines(catalogId, cache -> safeInvalidate( cache, catalogId, "invalidateCatalog", @@ -431,6 +489,44 @@ public MetaCacheEntryStats getEntryStats() { private void initEngineCaches() { registerBuiltinEngineCaches(); + bindCatalogPreparers(); + } + + private void bindCatalogPreparers() { + for (ExternalMetaCache cache : cacheRegistry.allCaches()) { + String engine = cache.engine(); + cache.bindCatalogPreparer(catalogId -> tryPrepareCatalogByEngine(catalogId, engine)); + } + } + + /** + * Re-prepare a catalog whose group was retired between a caller's preparation and its lookup. + * Lookups may run inside a cache loader, and retirement holds the lifecycle lock while it + * closes groups, so this never blocks: when the fence is contended the lookup keeps its + * pre-existing failure and the next call prepares under the new policy. + */ + private void tryPrepareCatalogByEngine(long catalogId, String engine) { + ExternalMetaCache targetCache = this.engine(engine); + if (targetCache.isCatalogInitialized(catalogId)) { + return; + } + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + if (!lifecycleLock.tryLock()) { + return; + } + try { + if (targetCache.isCatalogInitialized(catalogId)) { + return; + } + Map catalogProperties = findCatalogProperties(catalogId); + if (catalogProperties == null) { + logMissingCatalogSkip(catalogId, "tryPrepareCatalogByEngine"); + return; + } + prepareCatalogByEngineLocked(catalogId, targetCache, catalogProperties); + } finally { + lifecycleLock.unlock(); + } } private void registerBuiltinEngineCaches() { @@ -545,6 +641,7 @@ public static Map getCacheStats(CacheStats cacheStats, long esti void replaceEngineCachesForTest(List caches) { cacheRegistry.resetForTest(caches); + bindCatalogPreparers(); } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java index 2b11a71f8c8d00..dc3caf2393fffe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java @@ -140,7 +140,9 @@ final class IcebergCacheSizeEstimator { private static final long BLOB_FIELD_BYTES = objectBytes(32L); private static final long PARTITION_STATISTICS_FILE_BYTES = objectBytes(256L); private static final long ENCRYPTED_KEY_BYTES = objectBytes(256L); - private static final long PARTITION_BYTES = objectBytes(640L); + // One retained IcebergPartition (value/transform ArrayLists) or one RangePartitionItem with a + // single partition column plus its map entry; extra columns are charged by IcebergPartitionInfo. + private static final long PARTITION_BYTES = objectBytes(680L); private static final long PARTITION_ALIAS_BYTES = objectBytes(256L); private static final long NAME_MAPPING_ENTRY_BYTES = objectBytes(256L); private static final long MANIFEST_ENTRY_BASE_BYTES = objectBytes(256L); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java index 5c43cc56f8bbf5..a274bd10c29ee5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java @@ -25,6 +25,13 @@ import java.util.Set; public class IcebergPartitionInfo { + // Each RangePartitionItem endpoint holds one LiteralExpr per partition column beyond the + // first (createPartitionKey fills the vacancy with an infinity literal): literal, its lazy + // supplier, children list and array. Calibrated against JOL in IcebergExternalMetaCacheTest. + private static final long RANGE_KEY_EXTRA_COLUMN_BYTES = + MetaCacheWeightUtils.estimatedObjectBytes(208L); + private static final long RANGE_ENDPOINTS_PER_ITEM = 2L; + private final Map nameToPartitionItem; private final Map nameToIcebergPartition; private final Map> nameToIcebergPartitionNames; @@ -85,11 +92,30 @@ private static long retainedPayloadBytes(Map partition if (partition != null) { bytes = MetaCacheWeightUtils.saturatedAdd( bytes, partition.getRetainedPayloadBytes()); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, partitionItemColumnBytes( + partition.getPartitionValues() == null + ? 0 : partition.getPartitionValues().size())); } } return bytes; } + /** + * Structural bytes a partition item retains for every partition column beyond the first; + * the fixed per-partition constants of the estimator cover a single column. The width is + * taken from the loaded metadata generation, so a spec that grew after the related-table + * check was cached is still charged for its full width. + */ + static long partitionItemColumnBytes(long partitionColumnCount) { + if (partitionColumnCount <= 1L) { + return 0L; + } + return MetaCacheWeightUtils.saturatedMultiply( + MetaCacheWeightUtils.saturatedMultiply( + partitionColumnCount - 1L, RANGE_ENDPOINTS_PER_ITEM), + RANGE_KEY_EXTRA_COLUMN_BYTES); + } + public long getLatestSnapshotId(String partitionName) { Set icebergPartitionNames = nameToIcebergPartitionNames.get(partitionName); if (icebergPartitionNames == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index 5d3618b21f01fb..cec5a47ad2a830 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -392,8 +392,8 @@ public TableMetadata refresh() { // fail instead of silently committing files produced for another metadata generation. if (!isWriterCompatible(refreshedMetadata)) { throw new CommitFailedException( - "Cannot retry Iceberg commit after schema, spec, sort order, location, " - + "format version, or table properties changed"); + "Cannot retry Iceberg commit after the table UUID, schema, spec, sort " + + "order, location, format version, or table properties changed"); } currentMetadata = refreshedMetadata; return refreshedMetadata; @@ -407,7 +407,10 @@ public void commit(TableMetadata base, TableMetadata newMetadata) { } private boolean isWriterCompatible(TableMetadata refreshedMetadata) { - return retainedMetadata.formatVersion() == refreshedMetadata.formatVersion() + // A dropped and recreated table can restart schema/spec/order ids at the same + // location; only the same table UUID may absorb a retried commit. + return Objects.equals(retainedMetadata.uuid(), refreshedMetadata.uuid()) + && retainedMetadata.formatVersion() == refreshedMetadata.formatVersion() && retainedMetadata.currentSchemaId() == refreshedMetadata.currentSchemaId() && retainedMetadata.defaultSpecId() == refreshedMetadata.defaultSpecId() && retainedMetadata.defaultSortOrderId() == refreshedMetadata.defaultSortOrderId() diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java index 5bb537340ba6e0..f38d2689dfbbf1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java @@ -23,6 +23,7 @@ import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheKey; import org.apache.doris.datasource.SchemaCacheValue; +import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.datasource.systable.SysTable; import org.apache.doris.statistics.AnalysisInfo; import org.apache.doris.statistics.BaseAnalysisTask; @@ -32,6 +33,7 @@ import org.apache.doris.thrift.TTableDescriptor; import org.apache.doris.thrift.TTableType; +import com.google.common.annotations.VisibleForTesting; import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.MetadataTableUtils; import org.apache.iceberg.Table; @@ -97,14 +99,38 @@ public boolean supportsSnapshotSelection() { } public Table getSysIcebergTable() { - Table baseTable = IcebergUtils.getQueryScopedIcebergTable(sourceTable); MetadataTableType tableType = MetadataTableType.from(sysTableType); if (tableType == null) { throw new IllegalArgumentException("Unknown iceberg system table type: " + sysTableType); } // Metadata tables capture their base operations. Keep them statement-local so exact // previousFiles/history state and stale-generation retry never leak into this table object. - return MetadataTableUtils.createMetadataTableInstance(baseTable, tableType); + return MetadataTableUtils.createMetadataTableInstance(resolveBaseTable(), tableType); + } + + /** + * The base generation this statement binds the metadata table to. Snapshot-selectable + * metadata tables derive both their scan and their schema from the source relation's frozen + * snapshot (the same generation IcebergScanNode scans), so analysis and execution cannot see + * different partition specs or schemas when the table entry refreshes mid-statement. Static + * metadata tables and statements without a bound snapshot read the latest generation. + * + *

    The statement snapshot is looked up by source table (like the scan node's fallback); + * a statement that time-travels the same table under several relations resolves the default + * or, if ambiguous, the latest generation for the schema. + */ + @VisibleForTesting + Table resolveBaseTable() { + if (supportsSnapshotSelection()) { + Optional

    frozenTable = MvccUtil.getSnapshotFromContext(sourceTable) + .filter(IcebergMvccSnapshot.class::isInstance) + .map(IcebergMvccSnapshot.class::cast) + .flatMap(snapshot -> snapshot.getSnapshotCacheValue().getIcebergTable()); + if (frozenTable.isPresent()) { + return frozenTable.get(); + } + } + return IcebergUtils.getQueryScopedIcebergTable(sourceTable); } @Override @@ -172,7 +198,7 @@ private static long generateSysTableId(long sourceTableId, String sysTableType) private SchemaCacheValue loadSchemaCacheValue() { // Metadata-table schemas may change after source schema or partition-spec evolution. - // Resolve the schema from the same latest-generation path instead of permanently pairing + // Resolve the schema from the statement's bound generation instead of permanently pairing // this long-lived system-table object with its first observed generation. return new SchemaCacheValue(IcebergUtils.parseSchema(getSysIcebergTable().schema(), getCatalog().getEnableMappingVarbinary(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index bee37c0c37cea4..c19d61ba328430 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -1770,10 +1770,14 @@ public static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, T List partitionColumns = IcebergUtils.getSchemaCacheValue( dorisTable, schemaId, table).getPartitionColumns(); + long partitionItemColumnBytes = IcebergPartitionInfo.partitionItemColumnBytes( + partitionColumns.size()); for (IcebergPartition partition : icebergPartitions) { nameToPartition.put(partition.getPartitionName(), partition); retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( retainedPayloadBytes, partition.getRetainedPayloadBytes()); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, partitionItemColumnBytes); String transform = table.specs().get(partition.getSpecId()).fields().get(0).transform().toString(); Range partitionRange = getPartitionRange( partition.getPartitionValues().get(0), transform, partitionColumns); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java index 906804e5307c21..7dd629685b12a0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java @@ -37,7 +37,9 @@ import java.util.Objects; import java.util.OptionalLong; import java.util.concurrent.ExecutorService; +import java.util.function.Consumer; import java.util.function.Function; +import java.util.function.LongConsumer; import java.util.function.Predicate; /** @@ -75,6 +77,8 @@ protected AbstractExternalMetaCache(String engine, ExecutorService refreshExecut this(engine, refreshExecutor, new ExternalMetaCacheBudgetManager(OptionalLong.empty())); } + private volatile LongConsumer catalogPreparer; + protected AbstractExternalMetaCache(String engine, ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { this.engine = engine; @@ -99,6 +103,59 @@ public void validateCatalogProperties(Map catalogProperties) { validateMappedCatalogProperties(safeCatalogProperties, true); } + @Override + public Map sanitizeCatalogPropertiesForRuntime(Map catalogProperties) { + return sanitizeCatalogPropertiesForRuntime(catalogProperties, warning -> LOG.debug(warning)); + } + + @Override + public void validateCatalogPropertiesForRuntime(Map catalogProperties) { + Map safeCatalogProperties = CacheSpec.applyCompatibilityMap( + catalogProperties, catalogPropertyCompatibilityMap()); + validateMappedCatalogProperties(safeCatalogProperties, false); + } + + /** + * Exactly what initCatalog keeps: mapped legacy keys, only known entries/options in this + * engine's namespace, a parsable catalog max-weight, and entry max-weights within it. + */ + private Map sanitizeCatalogPropertiesForRuntime( + Map catalogProperties, Consumer warningConsumer) { + Map safeCatalogProperties = CacheSpec.applyCompatibilityMap( + catalogProperties, catalogPropertyCompatibilityMap()); + safeCatalogProperties = CacheSpec.sanitizeEnginePropertiesForRuntime( + safeCatalogProperties, engine, metaCacheEntryDefs, warningConsumer); + try { + budgetManager.parseCatalogMaxWeight(safeCatalogProperties); + } catch (IllegalArgumentException e) { + safeCatalogProperties.remove(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY); + warningConsumer.accept("Ignoring invalid persisted external metadata cache property '" + + ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY + "': " + e.getMessage()); + } + OptionalLong runtimeCatalogMaxWeight = budgetManager.parseCatalogMaxWeight(safeCatalogProperties); + for (MetaCacheEntryDef entryDef : metaCacheEntryDefs.values()) { + if (entryDef.getSizeEstimator() == null) { + continue; + } + String maxWeightKey = CacheSpec.metaCacheKeyPrefix(engine) + + entryDef.getName() + ".max-weight"; + if (!safeCatalogProperties.containsKey(maxWeightKey)) { + continue; + } + CacheSpec cacheSpec = CacheSpec.fromProperties( + safeCatalogProperties, engine, entryDef.getName(), entryDef.getDefaultCacheSpec()); + try { + budgetManager.validateCatalogEntryHierarchy( + runtimeCatalogMaxWeight, cacheSpec.getMaxWeight()); + } catch (IllegalArgumentException e) { + safeCatalogProperties.remove(maxWeightKey); + warningConsumer.accept("Ignoring invalid persisted external metadata cache property '" + + maxWeightKey + "': " + e.getMessage()); + } + } + return safeCatalogProperties; + } + @Override public void initCatalog(long catalogId, Map catalogProperties) { if (catalogEntries.containsKey(catalogId)) { @@ -108,42 +165,9 @@ public void initCatalog(long catalogId, Map catalogProperties) { if (catalogEntries.containsKey(catalogId)) { return; } - Map safeCatalogProperties = CacheSpec.applyCompatibilityMap( - catalogProperties, catalogPropertyCompatibilityMap()); - safeCatalogProperties = CacheSpec.sanitizeEnginePropertiesForRuntime( - safeCatalogProperties, engine, metaCacheEntryDefs, + Map safeCatalogProperties = sanitizeCatalogPropertiesForRuntime( + catalogProperties, warning -> LOG.warn("{} (engine={}, catalog={})", warning, engine, catalogId)); - try { - budgetManager.parseCatalogMaxWeight(safeCatalogProperties); - } catch (IllegalArgumentException e) { - safeCatalogProperties.remove(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY); - LOG.warn("Ignoring invalid persisted external metadata cache property '{}' " - + "for engine {}, catalog {}: {}", - ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY, - engine, catalogId, e.getMessage()); - } - OptionalLong runtimeCatalogMaxWeight = budgetManager.parseCatalogMaxWeight(safeCatalogProperties); - for (MetaCacheEntryDef entryDef : metaCacheEntryDefs.values()) { - if (entryDef.getSizeEstimator() == null) { - continue; - } - String maxWeightKey = CacheSpec.metaCacheKeyPrefix(engine) - + entryDef.getName() + ".max-weight"; - if (!safeCatalogProperties.containsKey(maxWeightKey)) { - continue; - } - CacheSpec cacheSpec = CacheSpec.fromProperties( - safeCatalogProperties, engine, entryDef.getName(), entryDef.getDefaultCacheSpec()); - try { - budgetManager.validateCatalogEntryHierarchy( - runtimeCatalogMaxWeight, cacheSpec.getMaxWeight()); - } catch (IllegalArgumentException e) { - safeCatalogProperties.remove(maxWeightKey); - LOG.warn("Ignoring invalid persisted external metadata cache property '{}' " - + "for engine {}, catalog {}: {}", - maxWeightKey, engine, catalogId, e.getMessage()); - } - } validateMappedCatalogProperties(safeCatalogProperties, false); catalogEntries.put(catalogId, buildCatalogEntryGroup(catalogId, safeCatalogProperties)); } @@ -308,6 +332,13 @@ protected final ExternalTable findExternalTable(NameMapping nameMapping, String private CatalogEntryGroup requireCatalogEntryGroup(long catalogId) { CatalogEntryGroup group = catalogEntries.get(catalogId); + if (group == null && catalogPreparer != null) { + // The caller prepared the catalog before capturing this engine, but a cache-policy + // ALTER retired the group in between. Re-prepare once under the lifecycle fence so + // the lookup observes the new policy instead of failing a valid catalog. + catalogPreparer.accept(catalogId); + group = catalogEntries.get(catalogId); + } if (group == null) { throw new IllegalStateException(String.format( "Catalog %d is not initialized for engine '%s'.", @@ -316,6 +347,11 @@ private CatalogEntryGroup requireCatalogEntryGroup(long catalogId) { return group; } + @Override + public void bindCatalogPreparer(LongConsumer catalogPreparer) { + this.catalogPreparer = catalogPreparer; + } + protected CatalogIf getCatalog(long catalogId) { if (Env.getCurrentEnv() == null || Env.getCurrentEnv().getCatalogMgr() == null) { return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java index 8b874fac8f659e..47e623b219c19a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java @@ -24,6 +24,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.function.LongConsumer; /** * Engine-level abstraction for external metadata cache. @@ -45,6 +46,30 @@ public interface ExternalMetaCache { default void validateCatalogProperties(Map catalogProperties) { } + /** + * Drop the properties in this engine's namespace that runtime initialization would ignore + * (unknown entries, obsolete or unparsable options), returning what the engine will honor. + */ + default Map sanitizeCatalogPropertiesForRuntime(Map catalogProperties) { + return catalogProperties; + } + + /** + * Validate cache properties with the semantics initialization applies to persisted state: + * entry weights must fit their catalog bound, but the catalog bound is not compared with this + * FE's local global bound (runtime clamps it instead). + */ + default void validateCatalogPropertiesForRuntime(Map catalogProperties) { + } + + /** + * Bind the callback that (re)prepares a catalog group under the manager's lifecycle fence. + * A lookup that finds no group (the catalog was retired by a concurrent cache-policy ALTER + * after the caller prepared it) uses it once before failing. + */ + default void bindCatalogPreparer(LongConsumer catalogPreparer) { + } + /** * Initialize all registered entries for one catalog under current engine. * Entry instances are created eagerly at this stage. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java index 8484c4aa477354..85e0bed91b4959 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java @@ -122,7 +122,9 @@ final class PaimonCacheSizeEstimator { .build(); private static final boolean PAIMON_TYPE_LAYOUT_SUPPORTED = checkPaimonTypeLayout(); private static final boolean PAIMON_TABLE_LAYOUT_SUPPORTED = checkPaimonTableLayout(); - private static final long PARTITION_BYTES = objectBytes(160L); + // One Paimon Partition record with its single-column LinkedHashMap spec plus map entry; extra + // columns are charged by PaimonPartitionInfo. + private static final long PARTITION_BYTES = objectBytes(272L); private static final long PARTITION_ITEM_BYTES = objectBytes(640L); private static final long WRAPPER_BYTES = objectBytes(512L); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java index 004adc197061b5..f4570dccf57f6a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java @@ -44,6 +44,12 @@ public enum PruningStatus { UNPRUNABLE } + // Each ListPartitionItem key holds one LiteralExpr (with lazy supplier, children list and + // array) and the Paimon Partition spec one map entry per partition column beyond the first; + // the fixed per-partition constants cover a single column. Calibrated against JOL. + private static final long PARTITION_EXTRA_COLUMN_BYTES = + MetaCacheWeightUtils.estimatedObjectBytes(216L); + public static final PaimonPartitionInfo EMPTY = new PaimonPartitionInfo(PruningStatus.PRUNABLE); public static final PaimonPartitionInfo UNPRUNABLE = new PaimonPartitionInfo(PruningStatus.UNPRUNABLE); @@ -92,6 +98,15 @@ static long addRetainedStringPayload(long bytes, String value) { return addString(bytes, value); } + /** Structural bytes one partition retains for every partition column beyond the first. */ + static long partitionColumnBytes(long partitionColumnCount) { + if (partitionColumnCount <= 1L) { + return 0L; + } + return MetaCacheWeightUtils.saturatedMultiply( + partitionColumnCount - 1L, PARTITION_EXTRA_COLUMN_BYTES); + } + private static long retainedPayloadBytes(Map partitions) { if (partitions == null) { return 0L; @@ -104,6 +119,8 @@ private static long retainedPayloadBytes(Map partitions) { continue; } bytes = addStrings(bytes, partition.spec()); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, partitionColumnBytes( + partition.spec() == null ? 0 : partition.spec().size())); bytes = addString(bytes, partition.createdBy()); bytes = addString(bytes, partition.updatedBy()); bytes = addStrings(bytes, partition.options()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index a1eaf3100cdc28..66511038e7f4c8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -31,6 +31,7 @@ import org.apache.doris.common.UserException; import org.apache.doris.common.util.TimeUtils; import org.apache.doris.datasource.ExternalTable; +import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; import org.apache.doris.thrift.TColumnType; import org.apache.doris.thrift.TPrimitiveType; import org.apache.doris.thrift.schema.external.TArrayField; @@ -187,6 +188,8 @@ public static PaimonPartitionInfo generatePartitionInfo(Table table, List partitionValues = Lists.newArrayListWithExpectedSize(partitionColumns.size()); LinkedHashMap orderedTypedSpec = new LinkedHashMap<>(); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd(retainedPayloadBytes, + PaimonPartitionInfo.partitionColumnBytes(partitionColumns.size())); for (Column partitionColumn : partitionColumns) { String partitionColumnName = partitionColumn.getName(); Preconditions.checkState(typedSpec.containsKey(partitionColumnName), diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index 6217d06b587b46..d4c3c4a0d21261 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -1775,6 +1775,24 @@ private static TFetchSchemaTableDataResult metaCacheStatsMetadataResult(TSchemaT trow.addToColumnValue(new TCell().setStringVal( formatMetaCacheTime(entryStats.getLastLoadFailureTimeMs(), timeZone))); trow.addToColumnValue(new TCell().setStringVal(entryStats.getLastError())); // LAST_ERROR + // Memory governance: -1 for count-bounded entries without a weight budget. + trow.addToColumnValue(new TCell().setBoolVal(entryStats.isWeightBounded())); // WEIGHT_BOUNDED + trow.addToColumnValue(new TCell().setLongVal(entryStats.getMaxWeight())); // MAX_WEIGHT + trow.addToColumnValue( + new TCell().setLongVal(entryStats.getEstimatedWeight())); // ESTIMATED_WEIGHT + trow.addToColumnValue(new TCell().setLongVal(entryStats.getEvictionWeight())); // EVICTION_WEIGHT + trow.addToColumnValue(new TCell().setLongVal( + entryStats.getWeightAdmissionRejectedCount())); // WEIGHT_REJECT_COUNT + trow.addToColumnValue( + new TCell().setLongVal(entryStats.getCatalogMaxWeight())); // CATALOG_MAX_WEIGHT + trow.addToColumnValue(new TCell().setLongVal( + entryStats.getCatalogEstimatedWeight())); // CATALOG_ESTIMATED_WEIGHT + trow.addToColumnValue( + new TCell().setLongVal(entryStats.getGlobalMaxWeight())); // GLOBAL_MAX_WEIGHT + trow.addToColumnValue(new TCell().setLongVal( + entryStats.getGlobalEstimatedWeight())); // GLOBAL_ESTIMATED_WEIGHT + trow.addToColumnValue(new TCell().setStringVal( + entryStats.getLastWeightRejectReason())); // LAST_WEIGHT_REJECT_REASON dataBatch.add(trow); } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java index 607dde99476a5f..111a8173257c3a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/ExternalMetaCacheRouteResolverTest.java @@ -93,6 +93,75 @@ public void testCatalogCachePropertiesRejectEngineNotRoutedByCatalogType() { Assert.assertTrue(exception.getMessage().contains("not supported by catalog type")); } + @Test + public void testCatalogCachePropertyUpdateIgnoresPersistedLegacyKeys() { + ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); + HMSExternalCatalog catalog = new HMSExternalCatalog( + 1L, "hms", null, Collections.emptyMap(), ""); + Map persisted = new HashMap<>(); + // Legacy keys admitted by image/replay: a typo, an unknown engine namespace and an + // option that is valid but stale relative to the update below. + persisted.put("meta.cache.hive.partiton_values.capacity", "10"); + persisted.put("meta.cache.hvie.partition_values.capacity", "10"); + persisted.put("meta.cache.max-weight", "64MB"); + persisted.put("meta.cache.hive.partition_values.max-weight", "16MB"); + + // The persisted map is not valid as a whole ... + Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.validateCatalogCacheProperties(catalog, persisted)); + // ... but runtime only honors the sane subset, so an unrelated ALTER passes ... + metaCacheMgr.validateEffectiveCatalogCacheProperties(catalog, persisted); + metaCacheMgr.validateCatalogCachePropertyUpdate(catalog, persisted, + Collections.singletonMap("meta.cache.hive.partition_values.capacity", "20")); + // ... while newly supplied keys stay strict, alone and against the honored hierarchy. + Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.validateCatalogCachePropertyUpdate(catalog, persisted, + Collections.singletonMap("meta.cache.hive.partiton_values.enable", "true"))); + Assert.assertThrows(IllegalArgumentException.class, + () -> metaCacheMgr.validateCatalogCachePropertyUpdate(catalog, persisted, + Collections.singletonMap("meta.cache.hive.partition_values.max-weight", "128MB"))); + } + + @Test + public void testPropertyNotificationToleratesUnknownEngineNamespace() { + long catalogId = 15L; + HMSExternalCatalog catalog = new HMSExternalCatalog( + catalogId, "hms", null, Collections.emptyMap(), ""); + mockCurrentCatalog(catalogId, catalog); + Map replayedProperties = new HashMap<>(); + replayedProperties.put("meta.cache.hvie.partition_values.capacity", "10"); + replayedProperties.put("meta.cache.hive.partition_values.capacity", "10"); + + // Edit-log replay publishes persisted properties without DDL validation; an unknown + // legacy namespace must be ignored instead of aborting the replay. + catalog.notifyPropertiesUpdated(replayedProperties); + } + + @Test + public void testLookupRepreparesCatalogRetiredByConcurrentPolicyChange() { + ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); + long catalogId = 16L; + HMSExternalCatalog catalog = new HMSExternalCatalog( + catalogId, "hms", null, Collections.emptyMap(), ""); + mockCurrentCatalog(catalogId, catalog); + ExternalMetaCache hive = metaCacheMgr.hive(catalogId); + Assert.assertTrue(hive.isCatalogInitialized(catalogId)); + + // A cache-policy ALTER retires the group after the caller captured the engine ... + metaCacheMgr.removeCatalog(catalogId); + Assert.assertFalse(hive.isCatalogInitialized(catalogId)); + + // ... and the pending lookup re-prepares the catalog instead of failing. + hive.checkCatalogInitialized(catalogId); + Assert.assertTrue(hive.isCatalogInitialized(catalogId)); + metaCacheMgr.removeCatalog(catalogId); + + // A catalog that was really dropped is not re-created by a stale lookup. + mockCurrentCatalog(catalogId, null); + Assert.assertThrows(IllegalStateException.class, () -> hive.checkCatalogInitialized(catalogId)); + Assert.assertFalse(hive.isCatalogInitialized(catalogId)); + } + @Test public void testRuntimePreparationIgnoresInvalidPersistedCacheProperties() { ExternalMetaCacheMgr metaCacheMgr = new ExternalMetaCacheMgr(true); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 180d2b44adf6cc..c850d387bca4f9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -1350,6 +1350,17 @@ public void testTableAndSnapshotFormulasAgainstJolOwnedGraphs() throws Exception EstimatorCalibrationAssertions.assertConservativeDelta( "iceberg snapshot partitions", emptySnapshotEstimate, populatedSnapshotEstimate, emptySnapshot, populatedSnapshot); + + // A spec that widened after the related-table check retains one more literal per range + // endpoint and one more value/transform per partition; the projection charges the width + // it actually loaded instead of the single field the check assumed. + IcebergSnapshotCacheValue wideSnapshot = new IcebergSnapshotCacheValue( + realPartitionInfo(32, 3), new IcebergSnapshot(-1L, 0L)); + long wideSnapshotEstimate = IcebergCacheSizeEstimator.estimateSnapshotEntry( + key, wideSnapshot).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg wide snapshot partitions", populatedSnapshotEstimate, wideSnapshotEstimate, + populatedSnapshot, wideSnapshot); } @Test @@ -2015,19 +2026,31 @@ private IcebergTableCacheValue tableValueWithSnapshotHistory( } private IcebergPartitionInfo realPartitionInfo(int partitionCount) throws Exception { + return realPartitionInfo(partitionCount, 1); + } + + private IcebergPartitionInfo realPartitionInfo(int partitionCount, int partitionColumnCount) + throws Exception { Map partitionItems = new java.util.HashMap<>(); Map partitions = new java.util.HashMap<>(); - List partitionColumns = Collections.singletonList( - new org.apache.doris.catalog.Column( - "part", org.apache.doris.catalog.PrimitiveType.DATETIMEV2)); + List partitionColumns = new ArrayList<>(); + for (int column = 0; column < partitionColumnCount; column++) { + partitionColumns.add(new org.apache.doris.catalog.Column( + "part" + column, org.apache.doris.catalog.PrimitiveType.DATETIMEV2)); + } for (int index = 0; index < partitionCount; index++) { String value = Integer.toString(index); String name = "part=" + value; partitionItems.put(name, new org.apache.doris.catalog.RangePartitionItem( IcebergUtils.getPartitionRange(value, "day", partitionColumns))); - partitions.put(name, new IcebergPartition(name, 0, 1L, 1L, 1L, - 1L, 1L, Collections.singletonList(value), - Collections.singletonList("day"))); + // Loaded partitions own one String per value and transform. + List values = new ArrayList<>(); + List transforms = new ArrayList<>(); + for (int column = 0; column < partitionColumnCount; column++) { + values.add(new String(value)); + transforms.add(new String("day")); + } + partitions.put(name, new IcebergPartition(name, 0, 1L, 1L, 1L, 1L, 1L, values, transforms)); } return new IcebergPartitionInfo( partitionItems, partitions, Collections.emptyMap()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java index 18969f0a535ebb..5e73286fb8260d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java @@ -17,14 +17,20 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.datasource.mvcc.MvccSnapshot; +import org.apache.doris.datasource.mvcc.MvccUtil; + import org.apache.iceberg.MetadataTableType; import org.apache.iceberg.Schema; import org.apache.iceberg.Table; import org.apache.iceberg.types.Types; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; import org.mockito.Mockito; +import java.util.Optional; + public class IcebergSysExternalTableTest { @Test public void testStaticMetadataTablesDoNotSupportSnapshotSelection() { @@ -73,4 +79,41 @@ public void testMetadataSchemaReloadsAfterSourceEvolution() { Assertions.assertEquals(2, sysTable.getFullSchema().size()); Mockito.verify(sysTable, Mockito.times(2)).getSysIcebergTable(); } + + @Test + public void testSnapshotSelectableSchemaFollowsRelationSnapshot() { + IcebergExternalTable sourceTable = Mockito.mock(IcebergExternalTable.class); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + Mockito.when(sourceTable.getId()).thenReturn(1L); + Mockito.when(sourceTable.getName()).thenReturn("table"); + Mockito.when(sourceTable.getRemoteName()).thenReturn("table"); + Mockito.when(sourceTable.getCatalog()).thenReturn(catalog); + Mockito.when(sourceTable.getDatabase()).thenReturn(Mockito.mock(IcebergExternalDatabase.class)); + Table frozenGeneration = Mockito.mock(Table.class); + Table latestGeneration = Mockito.mock(Table.class); + IcebergSnapshotCacheValue snapshotValue = Mockito.mock(IcebergSnapshotCacheValue.class); + Mockito.when(snapshotValue.getIcebergTable()).thenReturn(Optional.of(frozenGeneration)); + Optional relationSnapshot = Optional.of(new IcebergMvccSnapshot(snapshotValue)); + + try (MockedStatic mvccUtil = Mockito.mockStatic(MvccUtil.class); + MockedStatic icebergUtils = Mockito.mockStatic(IcebergUtils.class)) { + mvccUtil.when(() -> MvccUtil.getSnapshotFromContext(sourceTable)).thenReturn(relationSnapshot); + icebergUtils.when(() -> IcebergUtils.getQueryScopedIcebergTable(sourceTable)) + .thenReturn(latestGeneration); + + // $partitions is snapshot-selectable: analysis must see the generation the scan uses. + IcebergSysExternalTable partitions = new IcebergSysExternalTable( + sourceTable, MetadataTableType.PARTITIONS.name()); + Assertions.assertSame(frozenGeneration, partitions.resolveBaseTable()); + + // $snapshots ignores a selected snapshot and keeps reading the latest generation. + IcebergSysExternalTable snapshots = new IcebergSysExternalTable( + sourceTable, MetadataTableType.SNAPSHOTS.name()); + Assertions.assertSame(latestGeneration, snapshots.resolveBaseTable()); + + // Without a bound relation snapshot the latest generation is used. + mvccUtil.when(() -> MvccUtil.getSnapshotFromContext(sourceTable)).thenReturn(Optional.empty()); + Assertions.assertSame(latestGeneration, partitions.resolveBaseTable()); + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java index 2df4ad7229dbe7..897a7d3e23dcf8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java @@ -34,6 +34,7 @@ import org.apache.iceberg.FileFormat; import org.apache.iceberg.FileMetadata; import org.apache.iceberg.FileScanTask; +import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.PartitionSpec; import org.apache.iceberg.RowDelta; import org.apache.iceberg.Schema; @@ -825,6 +826,30 @@ public void testRetainedGenerationRetriesAfterConcurrentDataCommit() throws User Assert.assertEquals(2, refreshedTable.history().size()); } + @Test + public void testRetainedGenerationRefusesRetryAgainstRecreatedTable() { + HadoopCatalog icebergCatalog = (HadoopCatalog) ops.getCatalog(); + TableIdentifier identifier = TableIdentifier.of(dbName, tbWithoutPartition); + Table originalTable = icebergCatalog.loadTable(identifier); + IcebergSnapshotCacheValue cacheValue = new IcebergSnapshotCacheValue( + Mockito.mock(IcebergPartitionInfo.class), Mockito.mock(IcebergSnapshot.class), + Optional.empty(), originalTable); + Table retainedTable = cacheValue.getIcebergTable().get(); + String retainedUuid = ((HasTableOperations) retainedTable).operations().current().uuid(); + + // Drop and recreate at the same location: schema, spec and sort-order ids restart, and + // the writer contract looks identical except for the table UUID. + icebergCatalog.dropTable(identifier, true); + Table recreatedTable = icebergCatalog.createTable(identifier, originalTable.schema()); + Assert.assertNotEquals(retainedUuid, + ((HasTableOperations) recreatedTable).operations().current().uuid()); + + Table writableTable = IcebergSnapshotCacheValue.createWritableTable(retainedTable, recreatedTable); + CommitFailedException failure = Assert.assertThrows(CommitFailedException.class, + () -> ((HasTableOperations) writableTable).operations().refresh()); + Assert.assertTrue(failure.getMessage(), failure.getMessage().contains("table UUID")); + } + @Test public void testStaticPartitionFilterRejectsUnknownKey() { Schema schema = new Schema( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 99bce8a3259c40..ceac5396f18723 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -81,6 +81,7 @@ import java.util.ArrayList; import java.util.Collections; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Optional; import java.util.concurrent.Callable; @@ -205,6 +206,16 @@ public void testSnapshotFormulaAgainstJolOwnedGraph() throws Exception { emptyInts.prepareForCachePublication(intKey).getBytes(), populatedInts.prepareForCachePublication(intKey).getBytes(), emptyInts, populatedInts); + + // Every partition column beyond the first adds a literal to each ListPartitionItem key + // and an entry to the Partition spec; the estimate scales with the loaded width. + PaimonSnapshotCacheValue wideInts = snapshotValueWithRealPartitions( + intTable, 32, 0, Type.INT, 3); + EstimatorCalibrationAssertions.assertConservativeDelta( + "paimon wide snapshot partitions", + populatedInts.prepareForCachePublication(intKey).getBytes(), + wideInts.prepareForCachePublication(intKey).getBytes(), + populatedInts, wideInts); } @Test @@ -1177,6 +1188,12 @@ private long snapshotWeight(PaimonSnapshotEntryKey key, FileStoreTable table, in private PaimonSnapshotCacheValue snapshotValueWithRealPartitions( FileStoreTable table, int partitionCount, int valueLength, Type partitionType) throws AnalysisException { + return snapshotValueWithRealPartitions(table, partitionCount, valueLength, partitionType, 1); + } + + private PaimonSnapshotCacheValue snapshotValueWithRealPartitions( + FileStoreTable table, int partitionCount, int valueLength, Type partitionType, + int partitionColumnCount) throws AnalysisException { Map partitionItems = new HashMap<>(); Map partitions = new HashMap<>(); for (int index = 0; index < partitionCount; index++) { @@ -1184,11 +1201,19 @@ private PaimonSnapshotCacheValue snapshotValueWithRealPartitions( ? Integer.toString(index) : "p" + index + repeatedCharacter('x', valueLength); String name = "part=" + value; - partitionItems.put(name, PaimonUtil.toListPartitionItem( - Collections.singletonList(value), Collections.singletonList(partitionType))); + List values = new ArrayList<>(); + List types = new ArrayList<>(); + Map spec = new java.util.LinkedHashMap<>(); + for (int column = 0; column < partitionColumnCount; column++) { + // Each loaded column owns its own value String. + String columnValue = new String(value); + values.add(columnValue); + types.add(partitionType); + spec.put("part" + column, columnValue); + } + partitionItems.put(name, PaimonUtil.toListPartitionItem(values, types)); partitions.put(name, new org.apache.paimon.partition.Partition( - Collections.singletonMap("part", value), - 100L, 1024L, 1L, 1L, 1, true)); + spec, 100L, 1024L, 1L, 1L, 1, true)); } PaimonPartitionInfo partitionInfo = new PaimonPartitionInfo(partitionItems, partitions); return new PaimonSnapshotCacheValue( diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy index 6079a17ec402ca..0c2bebc52fed56 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy @@ -80,6 +80,19 @@ suite("test_iceberg_table_meta_cache", "p0,external,doris,external_docker,extern sql """refresh table test_iceberg_meta_cache_db.sales""" // select 3 rows sql """select * from test_iceberg_meta_cache_db.sales""" + // The weight-bounded entries expose their budget hierarchy in the statistics view. + def weightStats = sql """ + select entry_name, weight_bounded, max_weight, estimated_weight, catalog_max_weight, + weight_reject_count, last_weight_reject_reason + from internal.information_schema.catalog_meta_cache_statistics + where catalog_name = "${catalog_name}" and engine_name = "iceberg" and weight_bounded = true + order by entry_name; + """ + assertTrue(weightStats.size() > 0) + for (row in weightStats) { + assertTrue((row[2] as long) > 0L) + assertTrue((row[3] as long) >= 0L) + } sql """drop table test_iceberg_meta_cache_db.sales""" // 2. test catalog with meta.cache.iceberg.table.ttl-second From 053a91a2b564899691480550496a1b76b8e4f28a Mon Sep 17 00:00:00 2001 From: guoqiang Date: Tue, 18 Aug 2026 00:58:11 +0800 Subject: [PATCH 05/45] [fix](fe) Address review findings on statistics upgrade tolerance, eviction telemetry and Iceberg payload terms - BE catalog_meta_cache_statistics scanner retries with the legacy column set against an FE that does not know the weight columns and leaves them NULL. - Report the exact reservation weight of Caffeine-driven evictions instead of the int-clamped weigher value, fenced by generation against replacement races. - Charge Iceberg v3 field-default Literal wrappers and values, merged-overlap alias sets per enclosed partition name, and name-mapping alias arrays per historical name. - Document why a snapshot projection charges the frozen table generation it retains even though the table entry that produced it also charges it. --- ...chema_catalog_meta_cache_stats_scanner.cpp | 55 ++++++++---- .../schema_catalog_meta_cache_stats_scanner.h | 2 + .../iceberg/IcebergCacheSizeEstimator.java | 50 +++++++++-- .../iceberg/IcebergPartitionInfo.java | 25 +++++- .../iceberg/IcebergSnapshotCacheValue.java | 5 ++ .../datasource/iceberg/IcebergUtils.java | 2 + .../datasource/metacache/MetaCacheEntry.java | 45 +++++++++- .../metacache/MetaCacheWeightUtils.java | 20 +++++ .../iceberg/IcebergExternalMetaCacheTest.java | 83 ++++++++++++++++++- .../metacache/MetaCacheEntryTest.java | 33 ++++++++ 10 files changed, 289 insertions(+), 31 deletions(-) diff --git a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp index bffbf40a3e0ca9..255813230aa266 100644 --- a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp +++ b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp @@ -65,6 +65,12 @@ std::vector SchemaCatalogMetaCacheStatsScanner::_s_tb {"LAST_WEIGHT_REJECT_REASON", TYPE_STRING, sizeof(StringRef), true}, }; +// Columns that every FE knows. The weight statistics columns appended after LAST_ERROR are +// only served by FEs that carry the memory-governance change; during a rolling upgrade an older +// FE rejects a projection that names them, so the scanner falls back to this prefix and leaves +// the newer columns NULL. +static constexpr size_t kLegacyMetaCacheStatsColumnCount = 23; + SchemaCatalogMetaCacheStatsScanner::SchemaCatalogMetaCacheStatsScanner() : SchemaScanner(_s_tbls_columns, TSchemaTableType::SCH_CATALOG_META_CACHE_STATISTICS) {} @@ -77,9 +83,10 @@ Status SchemaCatalogMetaCacheStatsScanner::start(RuntimeState* state) { return Status::OK(); } -Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { +Status SchemaCatalogMetaCacheStatsScanner::_fetch_from_fe(size_t column_count, + TFetchSchemaTableDataResult* result) { TSchemaTableRequestParams schema_table_request_params; - for (int i = 0; i < _s_tbls_columns.size(); i++) { + for (size_t i = 0; i < column_count; i++) { schema_table_request_params.__isset.columns_name = true; schema_table_request_params.columns_name.emplace_back(_s_tbls_columns[i].name); } @@ -89,20 +96,28 @@ Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { request.__set_schema_table_name(TSchemaTableName::CATALOG_META_CACHE_STATS); request.__set_schema_table_params(schema_table_request_params); - TFetchSchemaTableDataResult result; - RETURN_IF_ERROR(ThriftRpcHelper::rpc( _fe_addr.hostname, _fe_addr.port, - [&request, &result](FrontendServiceConnection& client) { - client->fetchSchemaTableData(result, request); + [&request, result](FrontendServiceConnection& client) { + client->fetchSchemaTableData(*result, request); }, _rpc_timeout)); + return Status::create(result->status); +} - Status status(Status::create(result.status)); +Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { + TFetchSchemaTableDataResult result; + Status status = _fetch_from_fe(_s_tbls_columns.size(), &result); if (!status.ok()) { - LOG(WARNING) << "fetch catalog meta cache stats from FE(" << _fe_addr.hostname - << ") failed, errmsg=" << status; - return status; + LOG(INFO) << "fetch catalog meta cache stats from FE(" << _fe_addr.hostname + << ") with all columns failed, retrying with the legacy column set: " << status; + result = TFetchSchemaTableDataResult(); + status = _fetch_from_fe(kLegacyMetaCacheStatsColumnCount, &result); + if (!status.ok()) { + LOG(WARNING) << "fetch catalog meta cache stats from FE(" << _fe_addr.hostname + << ") failed, errmsg=" << status; + return status; + } } std::vector result_data = result.data_batch; @@ -116,19 +131,29 @@ Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { _block->reserve(_block_rows_limit); + size_t col_size = _s_tbls_columns.size(); if (result_data.size() > 0) { - auto col_size = result_data[0].column_value.size(); - if (col_size != _s_tbls_columns.size()) { + col_size = result_data[0].column_value.size(); + if (col_size != _s_tbls_columns.size() && col_size != kLegacyMetaCacheStatsColumnCount) { return Status::InternalError( "catalog meta cache stats schema is not match for FE and BE"); } } + int available_columns = static_cast(col_size); + int total_columns = static_cast(_s_tbls_columns.size()); for (int i = 0; i < result_data.size(); i++) { TRow row = result_data[i]; - for (int j = 0; j < _s_tbls_columns.size(); j++) { - RETURN_IF_ERROR(insert_block_column(row.column_value[j], j, _block.get(), - _s_tbls_columns[j].type)); + for (int j = 0; j < total_columns; j++) { + if (j < available_columns) { + RETURN_IF_ERROR(insert_block_column(row.column_value[j], j, _block.get(), + _s_tbls_columns[j].type)); + } else { + // Column unknown to the serving FE: NULL. + auto column_guard = _block->mutate_column_scoped(j); + column_guard.mutable_column()->insert_default(); + column_guard.restore(); + } } } return Status::OK(); diff --git a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h index 836500fd97de85..7a2339baf44335 100644 --- a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h +++ b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h @@ -25,6 +25,7 @@ namespace doris { class RuntimeState; class Block; +class TFetchSchemaTableDataResult; class SchemaCatalogMetaCacheStatsScanner : public SchemaScanner { ENABLE_FACTORY_CREATOR(SchemaCatalogMetaCacheStatsScanner); @@ -40,6 +41,7 @@ class SchemaCatalogMetaCacheStatsScanner : public SchemaScanner { private: Status _get_meta_cache_from_fe(); + Status _fetch_from_fe(size_t column_count, TFetchSchemaTableDataResult* result); TNetworkAddress _fe_addr; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java index dc3caf2393fffe..94283ca6b25bb3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java @@ -35,6 +35,7 @@ import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.encryption.EncryptedKey; +import org.apache.iceberg.expressions.Literal; import org.apache.iceberg.transforms.Transform; import org.apache.iceberg.transforms.UnknownTransform; import org.apache.iceberg.types.Type; @@ -42,6 +43,7 @@ import java.lang.reflect.Field; import java.lang.reflect.Modifier; +import java.math.BigDecimal; import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.HashSet; @@ -100,6 +102,14 @@ final class IcebergCacheSizeEstimator { MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L); private static final long LONG_BYTES = MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 8L); + // Literals.BaseLiteral: value plus the transient serialized-buffer slot. + private static final long LITERAL_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); + // JDK 17 HeapByteBuffer: Buffer header fields, address, segment, hb and offset. + private static final long BYTE_BUFFER_BYTES = objectBytes(56L); + private static final long BIG_DECIMAL_BYTES = objectBytes(104L); + private static final long BOXED_DEFAULT_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 16L); private static final String TRUNCATE_TRANSFORM_PREFIX = "truncate["; // Truncate on a decimal source retains a BigInteger width (object plus one-int magnitude). private static final long TRUNCATE_WIDTH_BYTES = MetaCacheWeightUtils.saturatedAdd( @@ -143,7 +153,11 @@ final class IcebergCacheSizeEstimator { // One retained IcebergPartition (value/transform ArrayLists) or one RangePartitionItem with a // single partition column plus its map entry; extra columns are charged by IcebergPartitionInfo. private static final long PARTITION_BYTES = objectBytes(680L); - private static final long PARTITION_ALIAS_BYTES = objectBytes(256L); + // Outer map entry and table share of one merged-overlap group; the alias set itself and its + // contents are charged by IcebergPartitionInfo per enclosed partition name. + private static final long PARTITION_ALIAS_BYTES = objectBytes(144L); + // One name-mapping field: map node, boxed id and list object; alias arrays and Strings are + // charged by IcebergSnapshotCacheValue when the mapping is copied. private static final long NAME_MAPPING_ENTRY_BYTES = objectBytes(256L); private static final long MANIFEST_ENTRY_BASE_BYTES = objectBytes(256L); private static final long DATA_FILE_BYTES = objectBytes(896L); @@ -214,6 +228,11 @@ static MetaCacheSizeEstimate estimateSnapshotEntry( bytes, value.getRetainedNameMappingPayloadBytes()); if (value.getRetainedIcebergTable().isPresent()) { + // The projection keeps its own reference to the frozen table generation. That graph + // is charged here as well as by the table entry that produced it: the two entries have + // independent lifetimes (TTL, weight eviction, soft collection) and either may outlive + // the other, so each must be able to carry the graph on its own. Budgets should be + // sized for the table metadata being counted once per dependent entry. Table table = value.getRetainedIcebergTable().get(); MetaCacheSizeEstimate support = checkSupportedTable(table); if (!support.isComplete()) { @@ -825,8 +844,8 @@ private static long addFieldPayload( bytes = addString(bytes, name); } bytes = addString(bytes, field.doc()); - bytes = addDefaultPayload(bytes, field.initialDefault()); - bytes = addDefaultPayload(bytes, field.writeDefault()); + bytes = addDefaultPayload(bytes, field.initialDefaultLiteral()); + bytes = addDefaultPayload(bytes, field.writeDefaultLiteral()); boolean pushShortName = kind == FieldKind.STRUCT_FIELD || kind == FieldKind.MAP_KEY || !field.type().isStructType(); // Only fields nested through a chain of struct fields get accessors; anything below a @@ -951,16 +970,33 @@ private static long immutableMapTableCapacity(long size) { ? MetaCacheWeightUtils.saturatedMultiply(capacity, 2L) : capacity; } - private static long addDefaultPayload(long bytes, Object value) { + /** + * A v3 field default is retained as an Iceberg Literal wrapper (value plus a transient + * ByteBuffer slot) around its boxed or buffer value. + */ + private static long addDefaultPayload(long bytes, Literal literal) { + if (literal == null) { + return bytes; + } + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, LITERAL_BYTES); + Object value = literal.value(); if (value instanceof CharSequence) { return MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.estimatedCharSequenceBytes((CharSequence) value)); } else if (value instanceof ByteBuffer) { - return MetaCacheWeightUtils.saturatedAdd(bytes, ((ByteBuffer) value).capacity()); + return MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.saturatedAdd( + BYTE_BUFFER_BYTES, + MetaCacheWeightUtils.estimatedByteArrayBytes(((ByteBuffer) value).capacity()))); } else if (value instanceof byte[]) { - return MetaCacheWeightUtils.saturatedAdd(bytes, ((byte[]) value).length); + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedByteArrayBytes(((byte[]) value).length)); + } else if (value instanceof BigDecimal) { + return MetaCacheWeightUtils.saturatedAdd(bytes, BIG_DECIMAL_BYTES); + } else if (value == null || value instanceof Boolean) { + return bytes; } - return bytes; + // Boxed numbers, UUIDs and other small immutable values. + return MetaCacheWeightUtils.saturatedAdd(bytes, BOXED_DEFAULT_BYTES); } private static long addString(long bytes, String value) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java index a274bd10c29ee5..05f469dfdc5229 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java @@ -31,6 +31,9 @@ public class IcebergPartitionInfo { private static final long RANGE_KEY_EXTRA_COLUMN_BYTES = MetaCacheWeightUtils.estimatedObjectBytes(208L); private static final long RANGE_ENDPOINTS_PER_ITEM = 2L; + // A merged-overlap alias group is a HashSet of the enclosed physical partition names; the + // names themselves are shared with the partition maps. + private static final long HASH_SET_BYTES = MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); private final Map nameToPartitionItem; private final Map nameToIcebergPartition; @@ -50,7 +53,9 @@ public IcebergPartitionInfo(Map nameToPartitionItem, Map nameToIcebergPartition, Map> nameToIcebergPartitionNames) { this(nameToPartitionItem, nameToIcebergPartition, nameToIcebergPartitionNames, - retainedPayloadBytes(nameToIcebergPartition)); + MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes(nameToIcebergPartition), + partitionAliasBytes(nameToIcebergPartitionNames))); } public IcebergPartitionInfo(Map nameToPartitionItem, @@ -100,6 +105,24 @@ private static long retainedPayloadBytes(Map partition return bytes; } + /** + * Retained bytes of the merged-overlap alias sets: every group keeps one HashSet with one + * node per enclosed physical partition name (the estimator's per-group constant covers only + * the outer map entry and the empty set object). + */ + static long partitionAliasBytes(Map> nameToIcebergPartitionNames) { + if (nameToIcebergPartitionNames == null) { + return 0L; + } + long bytes = 0L; + for (Set aliases : nameToIcebergPartitionNames.values()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, HASH_SET_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedHashMapBytes(aliases == null ? 0L : aliases.size())); + } + return bytes; + } + /** * Structural bytes a partition item retains for every partition column beyond the first; * the fixed per-partition constants of the estimator cover a single column. The width is diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index cec5a47ad2a830..4b613fa25d1b0c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -96,6 +96,11 @@ private IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSna for (Map.Entry> entry : nameMapping.get().entrySet()) { List names = ImmutableList.copyOf(entry.getValue()); copy.put(entry.getKey(), names); + if (names.size() > 1) { + // A field with several historical names keeps an element array per name. + payloadBytes = MetaCacheWeightUtils.saturatedAdd(payloadBytes, + MetaCacheWeightUtils.estimatedObjectArrayBytes(names.size())); + } for (String name : names) { payloadBytes = MetaCacheWeightUtils.saturatedAdd(payloadBytes, MetaCacheWeightUtils.estimatedStringBytes(name)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index c19d61ba328430..ef408a81e20636 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -1785,6 +1785,8 @@ public static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, T nameToPartitionItem.put(partition.getPartitionName(), item); } Map> partitionNameMap = mergeOverlapPartitions(nameToPartitionItem); + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( + retainedPayloadBytes, IcebergPartitionInfo.partitionAliasBytes(partitionNameMap)); return new IcebergPartitionInfo( nameToPartitionItem, nameToPartition, partitionNameMap, retainedPayloadBytes); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java index de89c7b9d8d6aa..e76bfc0540f6d0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java @@ -125,6 +125,11 @@ public class MetaCacheEntry { private final AtomicLong weightAdmissionRejectedCount = new AtomicLong(0L); private final AtomicLong localEvictionCount = new AtomicLong(0L); private final AtomicLong localEvictionWeight = new AtomicLong(0L); + // Exact byte weight released by Caffeine-driven evictions (size, expiry, soft collection). + // Caffeine's own eviction weight is clamped to the int weigher, so it is not used for stats. + private final AtomicLong automaticEvictionWeight = new AtomicLong(0L); + // Generation reported evicted for a key, consumed by the fenced cleanup of that generation. + private final Map pendingEvictionGenerations = new ConcurrentHashMap<>(); private final AtomicReference lastWeightRejectReason = new AtomicReference<>(""); private final AtomicLong lastWeightRejectLogTimeMs = new AtomicLong(0L); private final AtomicBoolean closed = new AtomicBoolean(false); @@ -453,12 +458,14 @@ public void invalidateAll() { reservations.values().forEach(record -> record.reservation.release()); reservations.clear(); pendingRemovalGenerations.clear(); + pendingEvictionGenerations.clear(); invalidateCount.addAndGet(size); } else { long size = data.estimatedSize(); data.invalidateAll(); refreshRecords.clear(); pendingRemovalGenerations.clear(); + pendingEvictionGenerations.clear(); invalidateCount.addAndGet(size); } } @@ -509,7 +516,7 @@ public MetaCacheEntryStats stats() { weightBounded ? cacheSpec.getMaxWeight().getAsLong() : -1L, weightBounded ? entryBudget.getUsedWeight() : -1L, weightBounded ? MetaCacheWeightUtils.saturatedAdd( - cacheStats.evictionWeight(), localEvictionWeight.get()) : -1L, + automaticEvictionWeight.get(), localEvictionWeight.get()) : -1L, weightBounded ? weightAdmissionRejectedCount.get() : -1L, weightBounded ? entryBudget.getCatalogMaxWeight() : -1L, weightBounded ? entryBudget.getCatalogUsedWeight() : -1L, @@ -570,6 +577,12 @@ private AdmissionResult admitWeightedValue( return AdmissionResult.NOT_CURRENT; } if (oldValue == null && record != null) { + // The previous generation was already removed by Caffeine; if that removal was + // an eviction whose asynchronous cleanup has not run yet, account it here. + if (pendingEvictionGenerations.remove(key, record.generation)) { + automaticEvictionWeight.accumulateAndGet( + record.weight, MetaCacheWeightUtils::saturatedAdd); + } reservations.remove(key, record); record.reservation.release(); record = null; @@ -752,6 +765,10 @@ private void onRemoval(@Nullable K key, @Nullable V value, RemovalCause cause) { if (weightBounded) { ReservationRecord record = reservations.get(key); if (record != null) { + if (cause.wasEvicted()) { + automaticEvictionWeight.accumulateAndGet( + record.weight, MetaCacheWeightUtils::saturatedAdd); + } releaseReservation(key, record.generation); } } else { @@ -770,9 +787,13 @@ private void onRemoval(@Nullable K key, @Nullable V value, RemovalCause cause) { if (closed.get()) { return; } + if (cause.wasEvicted()) { + pendingEvictionGenerations.merge(key, ownerGeneration, Math::max); + } pendingRemovalGenerations.merge(key, ownerGeneration, Math::max); if (closed.get()) { pendingRemovalGenerations.remove(key, ownerGeneration); + pendingEvictionGenerations.remove(key, ownerGeneration); return; } scheduleRemovalCleanup(); @@ -808,12 +829,21 @@ private void drainRemovalCleanups() { if (!pendingRemovalGenerations.remove(key, generation)) { continue; } + boolean evicted = pendingEvictionGenerations.remove(key, generation); + if (!evicted) { + // A newer generation superseded the evicted one; its eviction can no longer + // be attributed, so drop the stale marker instead of retaining the key. + pendingEvictionGenerations.remove(key); + } try { - cleanupRemovedReservation(key, generation); + cleanupRemovedReservation(key, generation, evicted); } catch (RuntimeException e) { // Restore the generation for retry. The finally block requeues one bounded // drain instead of permanently wedging this entry's scheduled flag. pendingRemovalGenerations.merge(key, generation, Math::max); + if (evicted) { + pendingEvictionGenerations.merge(key, generation, Math::max); + } LOG.warn("Failed to clean a removal reservation for external metadata cache entry {}", name, e); } @@ -828,7 +858,7 @@ private void drainRemovalCleanups() { } } - private void cleanupRemovedReservation(K key, long expectedReservationGeneration) { + private void cleanupRemovedReservation(K key, long expectedReservationGeneration, boolean evicted) { beforeRemovalCleanupLockForTest(key); synchronized (admissionLock) { if (weightBounded) { @@ -836,6 +866,11 @@ private void cleanupRemovedReservation(K key, long expectedReservationGeneration if (record != null && record.generation == expectedReservationGeneration && data.asMap().get(key) == null && reservations.remove(key, record)) { + if (evicted) { + // The exact reservation, not Caffeine's int-clamped weigher value. + automaticEvictionWeight.accumulateAndGet( + record.weight, MetaCacheWeightUtils::saturatedAdd); + } record.reservation.release(); } } else { @@ -849,11 +884,13 @@ private void cleanupRemovedReservation(K key, long expectedReservationGeneration afterRemovalCleanupForTest(key); } - private void releaseReservation(K key, long expectedGeneration) { + private boolean releaseReservation(K key, long expectedGeneration) { ReservationRecord record = reservations.get(key); if (record != null && record.generation == expectedGeneration && reservations.remove(key, record)) { record.reservation.release(); + return true; } + return false; } private void releaseRefreshRecord(K key, long expectedGeneration) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java index 828bc28719d4d4..8e32a989588677 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java @@ -199,6 +199,26 @@ public static long estimatedObjectArrayBytes(long length) { return alignedArrayBytes(OBJECT_ARRAY_BASE_BYTES, length, OBJECT_REFERENCE_BYTES); } + /** + * A java.util.HashMap holding {@code entries} mappings: the map object, its power-of-two + * table (allocated on the first put) and one node per entry; keys and values are separate. + */ + public static long estimatedHashMapBytes(long entries) { + long bytes = estimatedObjectLayoutBytes(4L, 16L); + if (entries <= 0L) { + return bytes; + } + long capacity = 16L; + while (entries > capacity - capacity / 4L) { + capacity = saturatedMultiply(capacity, 2L); + if (capacity == Long.MAX_VALUE) { + break; + } + } + bytes = saturatedAdd(bytes, estimatedObjectArrayBytes(capacity)); + return saturatedAdd(bytes, saturatedMultiply(entries, estimatedObjectLayoutBytes(3L, 4L))); + } + /** Size of an object with a known field layout on the active VM. */ public static long estimatedObjectLayoutBytes(long referenceFields, long primitiveBytes) { if (referenceFields < 0L || primitiveBytes < 0L) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index c850d387bca4f9..41cdc2ca8bb966 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -55,7 +55,9 @@ import org.apache.iceberg.TableMetadata; import org.apache.iceberg.TableMetadataParser; import org.apache.iceberg.TableOperations; +import org.apache.iceberg.TableProperties; import org.apache.iceberg.encryption.EncryptedKey; +import org.apache.iceberg.expressions.Expressions; import org.apache.iceberg.hadoop.HadoopTables; import org.apache.iceberg.io.FileIO; import org.apache.iceberg.io.InputFile; @@ -787,14 +789,43 @@ public void testSchemaFormulaCountsBoxedIdsOfUncachedFieldIds() { } private TableMetadata metadataWithAddedSchema(Schema schema) { + return metadataWithAddedSchema(schema, 2); + } + + private TableMetadata metadataWithAddedSchema(Schema schema, int formatVersion) { Schema base = new Schema(0, Types.NestedField.optional(1, "base", Types.IntegerType.get())); TableMetadata metadata = TableMetadata.newTableMetadata( - base, PartitionSpec.unpartitioned(), "file:/warehouse/uncached-ids", - Collections.emptyMap()); + base, PartitionSpec.unpartitioned(), SortOrder.unsorted(), "file:/warehouse/uncached-ids", + Collections.singletonMap(TableProperties.FORMAT_VERSION, Integer.toString(formatVersion))); return TableMetadata.buildFrom(metadata).addSchema(schema) .setCurrentSchema(schema.schemaId()).discardChanges().build(); } + @Test + public void testSchemaFormulaCountsFieldDefaultLiterals() { + // v3 field defaults are retained as Literal wrappers around boxed or String values. + assertRetainedPayloadDelta("defaulted fields", + metadataWithAddedSchema(new Schema(1, defaultedFields(1)), 3), + metadataWithAddedSchema(new Schema(1, defaultedFields(32)), 3), "jol-defaults"); + } + + private List defaultedFields(int fieldCount) { + List fields = new ArrayList<>(); + for (int index = 0; index < fieldCount; index++) { + fields.add(Types.NestedField.optional("text_" + index).withId(10_000 + index) + .ofType(Types.StringType.get()) + .withInitialDefault(Expressions.lit("initial_" + index)) + .withWriteDefault(Expressions.lit("write_" + index)) + .build()); + fields.add(Types.NestedField.optional("number_" + index).withId(20_000 + index) + .ofType(Types.LongType.get()) + .withInitialDefault(Expressions.lit(100_000L + index)) + .withWriteDefault(Expressions.lit(200_000L + index)) + .build()); + } + return fields; + } + @Test public void testTablePayloadAccountsForRetainedHistoricalMetadata() { String largePayload = repeatedCharacter('x', 64 * 1024); @@ -1361,6 +1392,41 @@ public void testTableAndSnapshotFormulasAgainstJolOwnedGraphs() throws Exception EstimatorCalibrationAssertions.assertConservativeDelta( "iceberg wide snapshot partitions", populatedSnapshotEstimate, wideSnapshotEstimate, populatedSnapshot, wideSnapshot); + + // Overlapping physical partitions merge into one Doris partition that keeps every + // enclosed name in a HashSet; the weight follows the set cardinality, not the group count. + IcebergSnapshotCacheValue aliasedSnapshot = new IcebergSnapshotCacheValue( + realPartitionInfo(32, 1, true), new IcebergSnapshot(-1L, 0L)); + long aliasedSnapshotEstimate = IcebergCacheSizeEstimator.estimateSnapshotEntry( + key, aliasedSnapshot).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg partition aliases", populatedSnapshotEstimate, aliasedSnapshotEstimate, + populatedSnapshot, aliasedSnapshot); + + // A name mapping retains an element array per field once it has several historical names. + IcebergSnapshotCacheValue singleNames = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), + Optional.of(nameMappingWithAliases(32, 1))); + IcebergSnapshotCacheValue manyNames = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), + Optional.of(nameMappingWithAliases(32, 8))); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg name mapping aliases", + IcebergCacheSizeEstimator.estimateSnapshotEntry(key, singleNames).getBytes(), + IcebergCacheSizeEstimator.estimateSnapshotEntry(key, manyNames).getBytes(), + singleNames, manyNames); + } + + private Map> nameMappingWithAliases(int fieldCount, int aliasesPerField) { + Map> mapping = new java.util.HashMap<>(); + for (int field = 0; field < fieldCount; field++) { + List names = new ArrayList<>(); + for (int alias = 0; alias < aliasesPerField; alias++) { + names.add("field_" + field + "_v" + alias); + } + mapping.put(1000 + field, names); + } + return mapping; } @Test @@ -2031,6 +2097,11 @@ private IcebergPartitionInfo realPartitionInfo(int partitionCount) throws Except private IcebergPartitionInfo realPartitionInfo(int partitionCount, int partitionColumnCount) throws Exception { + return realPartitionInfo(partitionCount, partitionColumnCount, false); + } + + private IcebergPartitionInfo realPartitionInfo( + int partitionCount, int partitionColumnCount, boolean mergeAllIntoFirst) throws Exception { Map partitionItems = new java.util.HashMap<>(); Map partitions = new java.util.HashMap<>(); List partitionColumns = new ArrayList<>(); @@ -2052,8 +2123,12 @@ private IcebergPartitionInfo realPartitionInfo(int partitionCount, int partition } partitions.put(name, new IcebergPartition(name, 0, 1L, 1L, 1L, 1L, 1L, values, transforms)); } - return new IcebergPartitionInfo( - partitionItems, partitions, Collections.emptyMap()); + Map> aliases = Collections.emptyMap(); + if (mergeAllIntoFirst && partitionCount > 0) { + // mergeOverlapPartitions() shape: the surviving name owns a set of every enclosed name. + aliases = Collections.singletonMap("part=0", new java.util.HashSet<>(partitions.keySet())); + } + return new IcebergPartitionInfo(partitionItems, partitions, aliases); } private org.apache.iceberg.DataFile dataFileWithMetrics(int index) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java index e3052fc42f31fd..5607d604e1b01e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java @@ -1008,6 +1008,39 @@ public void testWeightedCacheUsesSoftValuesAndReleasesCollectedReservation() thr } } + @Test + public void testAutomaticEvictionTelemetryKeepsExactWeightAboveWeigherLimit() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + long hugeEstimate = 3L << 30; // above Caffeine's int weigher limit + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(8L << 30)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "huge-eviction", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "huge-eviction", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 8L << 30), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(hugeEstimate), budget); + try { + entry.put("k", new byte[1]); + Assert.assertEquals(accountedWeight(hugeEstimate), manager.getGlobalUsedWeight()); + LoadingCache loadingCache = extractLoadingCache(entry); + Reference valueReference = extractValueReference(loadingCache); + + // A soft-value collection is an automatic eviction reported through Caffeine, whose + // weigher saw at most Integer.MAX_VALUE; the statistics must report the reservation. + valueReference.clear(); + Assert.assertTrue(valueReference.enqueue()); + loadingCache.cleanUp(); + awaitGlobalWeight(manager, 0L); + + Assert.assertEquals(accountedWeight(hugeEstimate), entry.stats().getEvictionWeight()); + Assert.assertTrue(entry.stats().getEvictionWeight() > Integer.MAX_VALUE); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + @Test public void testStrongQueryReferenceSurvivesSoftValueCollectionChecks() throws Exception { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); From c302b6887634595608b72424eb06979aa71f06bf Mon Sep 17 00:00:00 2001 From: guoqiang Date: Tue, 18 Aug 2026 02:19:59 +0800 Subject: [PATCH 06/45] [fix](fe) Keep query-scoped Iceberg tables writable through live operations and reserve Paimon store graph - IcebergSnapshotCacheValue.retainTableGeneration keeps an already query-scoped table so historical scans never share cached BaseSnapshot instances - IcebergTransaction re-bases query-scoped (weight-bounded) tables onto live operations before commit instead of committing through read-only query operations - PaimonCacheSizeEstimator reserves the lazily built store graph (partition/bucket/key/value RowTypes, merge-engine RowType, prefixed key fields) at publication --- .../iceberg/IcebergSnapshotCacheValue.java | 25 ++- .../iceberg/IcebergTransaction.java | 2 +- .../paimon/PaimonCacheSizeEstimator.java | 160 +++++++++++++++--- .../iceberg/IcebergExternalMetaCacheTest.java | 19 +++ .../iceberg/IcebergTransactionTest.java | 37 ++++ .../paimon/PaimonExternalMetaCacheTest.java | 74 +++++++- 6 files changed, 282 insertions(+), 35 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index 4b613fa25d1b0c..f7dbed2effd183 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -181,6 +181,13 @@ static Table retainTableGeneration(Table table) { if (!(table instanceof HasTableOperations) || isFrozenGeneration(table)) { return table; } + if (table instanceof QueryScopedTable) { + // Already fixed to one admitted metadata generation and isolating its snapshot + // copies from the cached BaseSnapshot instances. Rebuilding it as a plain BaseTable + // would hand historical scans the shared snapshots, whose lazily materialized + // manifest lists would then grow the cached generation past its admitted weight. + return table; + } TableOperations operations = ((HasTableOperations) table).operations(); // Capture current() exactly once so every projection derived from the returned table sees // one metadata generation even when the shared catalog handle refreshes concurrently. @@ -239,6 +246,20 @@ static boolean isFrozenGeneration(Table table) { && ((HasTableOperations) table).operations() instanceof FrozenTableOperations; } + /** + * True for any table bound to a retained metadata generation that cannot commit by itself: + * a frozen generation, or a query-scoped view over one. Writers must re-base such tables onto + * live operations through {@link #createWritableTable}. + */ + static boolean isRetainedGeneration(Table table) { + if (!(table instanceof HasTableOperations)) { + return false; + } + TableOperations operations = ((HasTableOperations) table).operations(); + return operations instanceof FrozenTableOperations + || operations instanceof QueryScopedTableOperations; + } + static TableOperations unwrapRetainedTableOperations(TableOperations operations) { TableOperations current = Objects.requireNonNull(operations, "operations can not be null"); while (current instanceof RetainedTableOperations) { @@ -248,11 +269,11 @@ static TableOperations unwrapRetainedTableOperations(TableOperations operations) } static Table createWritableTable(Table retainedTable, Table liveTable) { - if (!isFrozenGeneration(retainedTable)) { + if (!isRetainedGeneration(retainedTable)) { return retainedTable; } if (!(liveTable instanceof HasTableOperations) - || isFrozenGeneration(liveTable)) { + || isRetainedGeneration(liveTable)) { throw new IllegalArgumentException( "Iceberg commit table must provide writable table operations"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java index 138e4fc4b1289c..50d245ab0ec6d6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java @@ -293,7 +293,7 @@ public void beginDelete(ExternalTable dorisTable, Table targetTable) throws User } private Table createTransactionTable(ExternalTable dorisTable, Table retainedTable) { - if (!IcebergSnapshotCacheValue.isFrozenGeneration(retainedTable)) { + if (!IcebergSnapshotCacheValue.isRetainedGeneration(retainedTable)) { return retainedTable; } // Reads stay on the retained generation; commit refreshes may follow data-only snapshots, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java index 85e0bed91b4959..f60ee09146f10c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java @@ -53,6 +53,7 @@ import org.apache.paimon.types.VectorType; import java.util.List; +import java.util.Locale; import java.util.Map; /** Publication-time retained-weight formula for Paimon snapshot projections. */ @@ -124,6 +125,21 @@ final class PaimonCacheSizeEstimator { private static final boolean PAIMON_TABLE_LAYOUT_SUPPORTED = checkPaimonTableLayout(); // One Paimon Partition record with its single-column LinkedHashMap spec plus map entry; extra // columns are charged by PaimonPartitionInfo. + // FileStoreTable.lazyStore: the store object, its CoreOptions/Options copy, SchemaManager, + // and the partition/bucket-key/row/key/value RowTypes it derives from the TableSchema. It is + // created by the partition projection before publication or by scan planning afterwards. + private static final long STORE_BASE_BYTES = objectBytes(1_536L); + private static final long STORE_OPTION_BYTES = objectBytes(48L); + // KeyValueFileStore keeps prefixed key-field copies and shares the value fields; the + // AppendOnlyFileStore deep copy of the whole type tree is reserved by + // retainedTablePayloadBytes, which already walks that tree. + private static final long STORE_KEY_FIELD_BYTES = objectBytes(112L); + private static final long STORE_LIST_SLOT_BYTES = objectBytes(8L); + private static final String APPEND_ONLY_TABLE_CLASS_NAME = + "org.apache.paimon.table.AppendOnlyFileStoreTable"; + private static final String PRIMARY_KEY_TABLE_CLASS_NAME = + "org.apache.paimon.table.PrimaryKeyFileStoreTable"; + private static final String MERGE_ENGINE_OPTION = "merge-engine"; private static final long PARTITION_BYTES = objectBytes(272L); private static final long PARTITION_ITEM_BYTES = objectBytes(640L); private static final long WRAPPER_BYTES = objectBytes(512L); @@ -228,8 +244,8 @@ private static boolean isSupportedTable(Table table) { return false; } String className = table.getClass().getName(); - return "org.apache.paimon.table.AppendOnlyFileStoreTable".equals(className) - || "org.apache.paimon.table.PrimaryKeyFileStoreTable".equals(className); + return APPEND_ONLY_TABLE_CLASS_NAME.equals(className) + || PRIMARY_KEY_TABLE_CLASS_NAME.equals(className); } /** Uses TableSchema cardinalities only and deliberately never calls FileStoreTable.store(). */ @@ -255,7 +271,70 @@ private static long estimateTable(Table table) { bytes = addCount(bytes, schema.options().size(), TABLE_OPTION_BYTES); bytes = addCount(bytes, schema.partitionKeys().size(), TABLE_KEY_BYTES); bytes = addCount(bytes, schema.primaryKeys().size(), TABLE_KEY_BYTES); - return addCount(bytes, schema.bucketKeys().size(), TABLE_KEY_BYTES); + bytes = addCount(bytes, schema.bucketKeys().size(), TABLE_KEY_BYTES); + return MetaCacheWeightUtils.saturatedAdd(bytes, storeGraphBytes(fileStoreTable, schema)); + } + + /** + * Reserve the store graph the table materializes without opening it: TableSchema + * cardinalities decide its size, and every RowType it derives can grow the four lazy lookup + * maps after admission exactly like nested RowTypes. + */ + private static long storeGraphBytes(FileStoreTable table, TableSchema schema) { + long bytes = STORE_BASE_BYTES; + bytes = addCount(bytes, schema.options().size(), STORE_OPTION_BYTES); + List fields = schema.fields(); + long fieldCount = fields.size(); + long uncachedFieldIds = 0L; + for (DataField field : fields) { + if (isUncachedInteger(field.id())) { + uncachedFieldIds++; + } + } + long partitionKeys = schema.partitionKeys().size(); + long bucketKeys = schema.bucketKeys().size(); + // Partition and bucket key RowTypes reference a subset of the fields; ids beyond the + // Integer cache are counted as if all of them were uncached, which is conservative. + long uncachedPartitionKeys = Math.min(partitionKeys, uncachedFieldIds); + long uncachedBucketKeys = Math.min(bucketKeys, uncachedFieldIds); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, rowTypeBytes(partitionKeys, uncachedPartitionKeys)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, rowTypeBytes(bucketKeys, uncachedBucketKeys)); + if (PRIMARY_KEY_TABLE_CLASS_NAME.equals(table.getClass().getName())) { + long primaryKeys = schema.primaryKeys().size(); + bytes = addCount(bytes, primaryKeys, STORE_KEY_FIELD_BYTES); + for (String primaryKey : schema.primaryKeys()) { + // Each trimmed key field gets a fresh "_KEY_" + name string. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(primaryKey)); + } + bytes = addCount(bytes, fieldCount, STORE_LIST_SLOT_BYTES); + // Key fields are re-numbered above the Integer cache; the value type shares fields. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, rowTypeBytes(primaryKeys, primaryKeys)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, rowTypeBytes(fieldCount, uncachedFieldIds)); + if (retainsMergeFunctionRowType(schema)) { + // partial-update / aggregation merge factories keep a second logical RowType and + // option-derived per-field maps (aggregation also copies CoreOptions). + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, rowTypeBytes(fieldCount, uncachedFieldIds)); + bytes = addCount(bytes, schema.options().size(), STORE_OPTION_BYTES); + } + return bytes; + } + // The copied row type of an append-only store (fields, types and nested lookup maps) is + // reserved by retainedTablePayloadBytes; the top-level RowType wrapper is charged here. + return MetaCacheWeightUtils.saturatedAdd(bytes, rowTypeBytes(fieldCount, uncachedFieldIds)); + } + + private static boolean retainsMergeFunctionRowType(TableSchema schema) { + String mergeEngine = schema.options().get(MERGE_ENGINE_OPTION); + if (mergeEngine == null) { + return false; + } + String normalized = mergeEngine.trim().toLowerCase(Locale.ROOT).replace('_', '-'); + return "partial-update".equals(normalized) || "aggregation".equals(normalized); + } + + private static boolean isUncachedInteger(int value) { + return value < -128 || value > 127; } /** @@ -288,8 +367,15 @@ private static long retainedTablePayloadBytes(Table table, AccountingBudget budg return 0L; } long bytes = addString(0L, schema.comment()); + TypeTreeStructure structure = new TypeTreeStructure(); for (DataField field : schema.fields()) { - bytes = addFieldPayload(bytes, field, false, budget, 0); + bytes = addFieldPayload(bytes, field, false, budget, 0, structure); + } + if (isAppendOnlyTable(table)) { + // AppendOnlyFileStore keeps logicalRowType().notNull(): a deep copy of every field + // and type (names and descriptions are shared), including nested RowTypes with their + // own lazy lookup maps. Reserve that copy without creating the store. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, structure.bytes); } budget.charge(schema.options().size()); for (Map.Entry option : schema.options().entrySet()) { @@ -312,15 +398,17 @@ private static long addStrings( private static long addFieldPayload( long bytes, DataField field, boolean nested, AccountingBudget budget, - int typeDepth) { + int typeDepth, TypeTreeStructure structure) { budget.charge(1L); + // Top-level DataFields are covered by TABLE_FIELD_BYTES; a copied tree owns them all. + structure.add(DATA_FIELD_BYTES); if (nested) { bytes = MetaCacheWeightUtils.saturatedAdd(bytes, DATA_FIELD_BYTES); } bytes = addString(bytes, field.name()); bytes = addString(bytes, field.description()); bytes = addString(bytes, field.defaultValue()); - return addTypePayload(bytes, field.type(), budget, typeDepth); + return addTypePayload(bytes, field.type(), budget, typeDepth, structure); } /** @@ -329,7 +417,8 @@ private static long addFieldPayload( * instead of counting a future composite type as a small primitive. */ private static long addTypePayload( - long bytes, DataType type, AccountingBudget budget, int typeDepth) { + long bytes, DataType type, AccountingBudget budget, int typeDepth, + TypeTreeStructure structure) { if (typeDepth > MAX_TYPE_ACCOUNTING_DEPTH) { throw new IllegalStateException( "Paimon cache accounting type depth exceeded"); @@ -342,40 +431,62 @@ private static long addTypePayload( if (typeClass == RowType.class) { RowType rowType = (RowType) type; List fields = rowType.getFields(); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, rowTypeBytes(fields)); + long rowBytes = rowTypeBytes(fields); + structure.add(rowBytes); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, rowBytes); for (DataField field : fields) { - bytes = addFieldPayload(bytes, field, true, budget, typeDepth + 1); + bytes = addFieldPayload(bytes, field, true, budget, typeDepth + 1, structure); } return bytes; } if (typeClass == ArrayType.class) { + structure.add(ARRAY_TYPE_BYTES); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ARRAY_TYPE_BYTES); return addTypePayload( - bytes, ((ArrayType) type).getElementType(), budget, typeDepth + 1); + bytes, ((ArrayType) type).getElementType(), budget, typeDepth + 1, structure); } if (typeClass == VectorType.class) { + structure.add(VECTOR_TYPE_BYTES); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, VECTOR_TYPE_BYTES); return addTypePayload( - bytes, ((VectorType) type).getElementType(), budget, typeDepth + 1); + bytes, ((VectorType) type).getElementType(), budget, typeDepth + 1, structure); } if (typeClass == MapType.class) { + structure.add(MAP_TYPE_BYTES); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MAP_TYPE_BYTES); bytes = addTypePayload( - bytes, ((MapType) type).getKeyType(), budget, typeDepth + 1); + bytes, ((MapType) type).getKeyType(), budget, typeDepth + 1, structure); return addTypePayload( - bytes, ((MapType) type).getValueType(), budget, typeDepth + 1); + bytes, ((MapType) type).getValueType(), budget, typeDepth + 1, structure); } if (typeClass == MultisetType.class) { + structure.add(MULTISET_TYPE_BYTES); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MULTISET_TYPE_BYTES); - return addTypePayload( - bytes, ((MultisetType) type).getElementType(), budget, typeDepth + 1); + // MultisetType.copy() shares its element type, so a deep copy stops here. + return addTypePayload(bytes, ((MultisetType) type).getElementType(), budget, + typeDepth + 1, new TypeTreeStructure()); } String[] leafFields = LEAF_TYPE_FIELDS.get(typeClass); if (leafFields == null) { throw new IllegalStateException( "Unsupported Paimon data type: " + typeClass.getName()); } - return MetaCacheWeightUtils.saturatedAdd(bytes, leafTypeBytes(leafFields)); + long leafBytes = leafTypeBytes(leafFields); + structure.add(leafBytes); + return MetaCacheWeightUtils.saturatedAdd(bytes, leafBytes); + } + + private static boolean isAppendOnlyTable(Table table) { + return APPEND_ONLY_TABLE_CLASS_NAME.equals(table.getClass().getName()); + } + + /** Non-String bytes of a schema type tree, i.e. what a deep DataType copy allocates again. */ + private static final class TypeTreeStructure { + private long bytes; + + private void add(long structuralBytes) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, structuralBytes); + } } /** @@ -384,7 +495,16 @@ private static long addTypePayload( * so a query cannot grow the retained graph past the admitted weight; nothing is materialized. */ private static long rowTypeBytes(List fields) { - long fieldCount = fields.size(); + long uncachedFieldIds = 0L; + for (DataField field : fields) { + if (isUncachedInteger(field.id())) { + uncachedFieldIds++; + } + } + return rowTypeBytes(fields.size(), uncachedFieldIds); + } + + private static long rowTypeBytes(long fieldCount, long uncachedFieldIds) { long bytes = MetaCacheWeightUtils.saturatedAdd(ROW_TYPE_BYTES, UNMODIFIABLE_LIST_BYTES); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ARRAY_LIST_BYTES); if (fieldCount == 0L) { @@ -392,12 +512,6 @@ private static long rowTypeBytes(List fields) { } bytes = MetaCacheWeightUtils.saturatedAdd( bytes, MetaCacheWeightUtils.estimatedObjectArrayBytes(fieldCount)); - long uncachedFieldIds = 0L; - for (DataField field : fields) { - if (field.id() < -128 || field.id() > 127) { - uncachedFieldIds++; - } - } long uncachedIndexes = fieldCount > 128L ? fieldCount - 128L : 0L; long mapBytes = MetaCacheWeightUtils.saturatedAdd(HASH_MAP_BYTES, MetaCacheWeightUtils.estimatedObjectArrayBytes(hashMapCapacity(fieldCount))); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 41cdc2ca8bb966..419bb991f05de2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -1162,6 +1162,25 @@ public void testWeightedV2ManifestListMaterializesOnlyInQueryView() throws Excep Assert.assertNotSame(firstQuery.currentSnapshot(), secondQuery.currentSnapshot()); Assert.assertNotSame(firstManifests, secondManifests); Assert.assertNotSame(retained.currentSnapshot(), firstQuery.currentSnapshot()); + + // A time-travel projection built from the query-scoped view keeps that isolation: a + // historical manifest-list read must not touch the cached generation's snapshots. + long snapshotId = liveTable.currentSnapshot().snapshotId(); + IcebergSnapshotCacheValue historical = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(snapshotId, 0L), + Optional.empty(), value.getIcebergTable()); + Table historicalTable = historical.getIcebergTable().get(); + Assert.assertEquals(1, historicalTable.snapshot(snapshotId).dataManifests(historicalTable.io()).size()); + Assert.assertNotSame(retained.snapshot(snapshotId), historicalTable.snapshot(snapshotId)); + Snapshot cachedSnapshot = retained.snapshot(snapshotId); + for (Field retainedField : cachedSnapshot.getClass().getDeclaredFields()) { + if (java.lang.reflect.Modifier.isTransient(retainedField.getModifiers()) + && !retainedField.getType().isPrimitive()) { + retainedField.setAccessible(true); + Assert.assertNull(retainedField.getName() + " must stay unmaterialized in the cache", + retainedField.get(cachedSnapshot)); + } + } } @Test diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java index 897a7d3e23dcf8..bd966a75332568 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java @@ -707,6 +707,43 @@ public void testBeginDeleteUsesRetainedTargetTable() throws UserException { Mockito.verify(retainedTable).newTransaction(); } + @Test + public void testQueryScopedGenerationCommitsThroughWritableOperations() throws UserException { + // A weight-bounded snapshot cache hands query-scoped (read-only) tables to the sink; + // commits must still be re-based onto the live table operations. + Table liveTable = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); + IcebergTableCacheValue tableValue = new IcebergTableCacheValue(liveTable); + tableValue.prepareForCachePublication(NameMapping.createForTest(dbName, tbWithoutPartition)); + IcebergSnapshotCacheValue cacheValue = new IcebergSnapshotCacheValue( + Mockito.mock(IcebergPartitionInfo.class), Mockito.mock(IcebergSnapshot.class), + Optional.empty(), tableValue.getRetainedIcebergTable(), + tableValue.getRetainedCurrentSnapshotJson()); + Table queryScopedTable = cacheValue.getIcebergTable().get(); + Assert.assertFalse(IcebergSnapshotCacheValue.isFrozenGeneration(queryScopedTable)); + Assert.assertTrue(IcebergSnapshotCacheValue.isRetainedGeneration(queryScopedTable)); + IcebergExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(dorisTable.getName()).thenReturn(tbWithoutPartition); + + TIcebergCommitData commitData = new TIcebergCommitData(); + commitData.setFilePath("query-scoped-generation.parquet"); + commitData.setFileContent(TFileContent.DATA); + commitData.setRowCount(1); + commitData.setFileSize(1); + + try (MockedStatic mockedUtils = Mockito.mockStatic( + IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(liveTable); + IcebergTransaction txn = getTxn(); + txn.updateIcebergCommitData(Collections.singletonList(commitData)); + txn.beginInsert(dorisTable, queryScopedTable, Optional.empty()); + txn.finishInsert(NameMapping.createForTest(dbName, tbWithoutPartition)); + txn.commit(); + } + + Assert.assertNotNull(ops.getCatalog().loadTable( + TableIdentifier.of(dbName, tbWithoutPartition)).currentSnapshot()); + } + @Test public void testRetainedGenerationCommitsThroughWritableOperations() throws UserException { Table liveTable = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition)); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index ceac5396f18723..723fab322725ed 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -51,6 +51,7 @@ import org.apache.paimon.table.AppendOnlyFileStoreTable; import org.apache.paimon.table.CatalogEnvironment; import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.PrimaryKeyFileStoreTable; import org.apache.paimon.table.Table; import org.apache.paimon.table.sink.StreamTableCommit; import org.apache.paimon.table.sink.StreamTableWrite; @@ -236,10 +237,29 @@ public void testTableSchemaFormulaAgainstJolOwnedGraph() throws Exception { long emptyEstimate = empty.prepareForCachePublication(emptyKey).getBytes(); long populatedEstimate = populated.prepareForCachePublication(populatedKey).getBytes(); + materializeStoreGraph(emptyTable); + materializeStoreGraph(populatedTable); EstimatorCalibrationAssertions.assertConservativeDelta( "paimon table fields", emptyEstimate, populatedEstimate, empty, populated); } + @Test + public void testStoreGraphFormulaAgainstJolOwnedGraph() throws Exception { + // AppendOnlyFileStore copies every field for its non-null row type; KeyValueFileStore + // copies the primary-key fields and shares the rest. Both derive RowTypes with lazy + // lookup maps and copy the table options; the estimate must cover them after + // newReadBuilder().newScan() runs on the admitted table. + assertTableDeltaAgainstJol("paimon append store fields", + newTableWithExtraFields("jol_store_append_narrow", 10, false, 0), + newTableWithExtraFields("jol_store_append_wide", 300, false, 0)); + assertTableDeltaAgainstJol("paimon primary-key store fields", + newTableWithExtraFields("jol_store_pk_narrow", 10, true, 0), + newTableWithExtraFields("jol_store_pk_wide", 300, true, 0)); + assertTableDeltaAgainstJol("paimon store options", + newTableWithExtraFields("jol_store_options_none", 10, false, 0), + newTableWithExtraFields("jol_store_options_many", 10, false, 100)); + } + @Test public void testNestedTableSchemaFormulaAgainstJolOwnedGraph() throws Exception { FileStoreTable smallTable = newTableWithNestedFields("jol_nested_small", 1); @@ -1006,24 +1026,35 @@ private FileStoreTable newPartitionedTableWithNestedField( } private FileStoreTable newTableWithExtraFields(String name, int fieldCount) throws Exception { + return newTableWithExtraFields(name, fieldCount, false, 0); + } + + private FileStoreTable newTableWithExtraFields( + String name, int fieldCount, boolean primaryKey, int optionCount) throws Exception { ArrayList fields = new ArrayList<>(); fields.add(new DataField(0, "part", new IntType())); for (int index = 0; index < fieldCount; index++) { fields.add(new DataField(index + 1, "field_" + index, new IntType())); } + if (primaryKey) { + fields.add(new DataField(fieldCount + 1, "id", new IntType(false))); + } + Map options = new HashMap<>(); + for (int index = 0; index < optionCount; index++) { + options.put("option_" + index, "value_" + index); + } TableSchema schema = new TableSchema( 0, fields, - 1, + fields.size(), Collections.singletonList("part"), - Collections.emptyList(), - Collections.emptyMap(), + primaryKey ? java.util.Arrays.asList("id", "part") : Collections.emptyList(), + options, null); - return new AppendOnlyFileStoreTable( - LocalFileIO.create(), - new Path(temporaryFolder.newFolder(name).toURI()), - schema, - CatalogEnvironment.empty()); + Path location = new Path(temporaryFolder.newFolder(name).toURI()); + return primaryKey + ? new PrimaryKeyFileStoreTable(LocalFileIO.create(), location, schema, CatalogEnvironment.empty()) + : new AppendOnlyFileStoreTable(LocalFileIO.create(), location, schema, CatalogEnvironment.empty()); } private FileStoreTable newTableWithNestedFields(String name, int nestedFieldCount) throws Exception { @@ -1113,13 +1144,38 @@ private void assertTableDeltaAgainstJol( long smallEstimate = small.prepareForCachePublication(smallKey).getBytes(); long populatedEstimate = populated.prepareForCachePublication(populatedKey).getBytes(); // The estimate reserves the lookup maps every nested RowType can materialize after - // admission, so the JOL oracle measures the fully grown graph. + // admission and the store graph scan planning creates, so the JOL oracle measures the + // fully grown graph. materializeRowTypeIndexes(smallTable.schema()); materializeRowTypeIndexes(populatedTable.schema()); + materializeStoreGraph(smallTable); + materializeStoreGraph(populatedTable); EstimatorCalibrationAssertions.assertConservativeDelta( fixture, smallEstimate, populatedEstimate, small, populated); } + /** What scan planning materializes after admission: the store and its RowType indexes. */ + private void materializeStoreGraph(FileStoreTable table) { + table.newReadBuilder().newScan(); + try { + Object store = readField(table, table.getClass(), "lazyStore"); + for (Class owner = store.getClass(); owner != null && owner != Object.class; + owner = owner.getSuperclass()) { + for (Field field : owner.getDeclaredFields()) { + if (RowType.class.isAssignableFrom(field.getType())) { + field.setAccessible(true); + Object rowType = field.get(store); + if (rowType != null) { + materializeRowTypeIndexes((RowType) rowType); + } + } + } + } + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + private static final String[] ROW_TYPE_LAZY_FIELDS = { "laziedNameToField", "laziedNameToIndex", "laziedFieldIdToField", "laziedFieldIdToIndex"}; From b8fcc6db151b2b25dbfcf6fc0cf2996a9bf58848 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Tue, 18 Aug 2026 03:21:19 +0800 Subject: [PATCH 07/45] [fix](fe) Put the Paimon base table entry under the meta cache weight budget - PaimonTableCacheValue gets fail-closed publication sizing (table graph, retained payload and the lazily built store graph), and the table entry registers the estimator so catalog and global weight bounds cover base table handles as well as snapshot projections - Cover base-only scan growth, reservation release, unsupported-table rejection and a weighted catalog in the regression suite --- .../paimon/PaimonCacheSizeEstimator.java | 44 ++++++++-- .../paimon/PaimonExternalMetaCache.java | 1 + .../paimon/PaimonTableCacheValue.java | 35 +++++++- .../paimon/PaimonExternalMetaCacheTest.java | 81 +++++++++++++++++++ .../test_paimon_table_meta_cache.groovy | 58 +++++++++++++ 5 files changed, 209 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java index f60ee09146f10c..4011e6a797b77e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.paimon; +import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; @@ -56,7 +57,7 @@ import java.util.Locale; import java.util.Map; -/** Publication-time retained-weight formula for Paimon snapshot projections. */ +/** Publication-time retained-weight formulas for Paimon table handles and snapshot projections. */ final class PaimonCacheSizeEstimator { // Calibrated against JOL retained-graph deltas in PaimonExternalMetaCacheTest. private static final long MAX_TABLE_ACCOUNTING_ELEMENTS = 50_000L; @@ -64,6 +65,8 @@ final class PaimonCacheSizeEstimator { private static final long KEY_BASE_BYTES = objectBytes(128L); private static final long SNAPSHOT_BASE_BYTES = objectBytes(4L * 1024L); private static final long TABLE_BASE_BYTES = objectBytes(16L * 1024L); + // PaimonTableCacheValue: table ref, generation, payload bytes, estimate ref (+ estimate object). + private static final long TABLE_VALUE_BASE_BYTES = objectBytes(96L); // A top-level DataField, its list slot and shared per-field overhead; the DataType instance // is accounted separately by addTypePayload. private static final long TABLE_FIELD_BYTES = objectBytes(40L); @@ -206,18 +209,43 @@ private static boolean checkPaimonTableLayout() { "lazyStore:KeyValueFileStore"); } - static MetaCacheSizeEstimate estimateSnapshotEntry( - PaimonSnapshotEntryKey key, PaimonSnapshotCacheValue value) { + /** + * Retained weight of the base table entry. The table handle is owned independently of the + * snapshot projections that reference it (they may pin an older generation), so the same + * table graph is charged to both owners rather than shared. + */ + static MetaCacheSizeEstimate estimateTableEntry(NameMapping key, PaimonTableCacheValue value) { + String unsupported = unsupportedReason(value.getPaimonTable()); + if (unsupported != null) { + return MetaCacheSizeEstimate.incomplete(unsupported); + } + long bytes = MetaCacheWeightUtils.saturatedAdd( + KEY_BASE_BYTES, MetaCacheWeightUtils.estimatedNameMappingBytes(key)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, TABLE_VALUE_BASE_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedTablePayloadBytes()); + return MetaCacheSizeEstimate.complete( + MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(value.getPaimonTable()))); + } + + private static String unsupportedReason(Table table) { if (!MetaCacheWeightUtils.isSupportedJvmObjectLayout()) { - return MetaCacheSizeEstimate.incomplete("unsupported_jvm_object_alignment"); + return "unsupported_jvm_object_alignment"; } if (!PAIMON_TYPE_LAYOUT_SUPPORTED || !PAIMON_TABLE_LAYOUT_SUPPORTED) { - return MetaCacheSizeEstimate.incomplete("unsupported_paimon_layout"); + return "unsupported_paimon_layout"; } - Table table = value.getSnapshot().getTable(); if (!isSupportedTable(table)) { - return MetaCacheSizeEstimate.incomplete("unsupported_paimon_table:" - + (table == null ? "null" : table.getClass().getName())); + return "unsupported_paimon_table:" + (table == null ? "null" : table.getClass().getName()); + } + return null; + } + + static MetaCacheSizeEstimate estimateSnapshotEntry( + PaimonSnapshotEntryKey key, PaimonSnapshotCacheValue value) { + Table table = value.getSnapshot().getTable(); + String unsupported = unsupportedReason(table); + if (unsupported != null) { + return MetaCacheSizeEstimate.incomplete(unsupported); } long bytes = MetaCacheWeightUtils.saturatedAdd( diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index a308896bb299e5..ec5436b98e174d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -81,6 +81,7 @@ public PaimonExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCach tableEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_TABLE, NameMapping.class, PaimonTableCacheValue.class, this::loadTableCacheValue, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping)) + .withSizeEstimator((key, value) -> value.prepareForCachePublication(key)) .withReplacementListener(this::retireTableGeneration)); snapshotEntry = registerEntry(MetaCacheEntryDef.contextualOnly(ENTRY_SNAPSHOT, PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class, defaultEntryCacheSpec(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java index 9e381602dcbc9e..9e82df8f8f209c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java @@ -17,20 +17,27 @@ package org.apache.doris.datasource.paimon; +import org.apache.doris.datasource.NameMapping; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; +import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; + import org.apache.paimon.table.Table; import java.util.Objects; import java.util.concurrent.atomic.AtomicLong; /** - * Cache value for a Paimon table handle. Snapshot projections use a separate cache entry so this - * value cannot grow after admission. + * Cache value for a Paimon table handle. Snapshot projections use a separate cache entry; the + * only post-admission growth of this value is the lazily built store graph and RowType lookup + * maps of the table itself, which the publication estimate reserves up front. */ public class PaimonTableCacheValue { private static final AtomicLong NEXT_GENERATION = new AtomicLong(); private final Table paimonTable; private final long generation; + private volatile long retainedTablePayloadBytes; + private volatile MetaCacheSizeEstimate sizeEstimate; public PaimonTableCacheValue(Table paimonTable) { this.paimonTable = paimonTable; @@ -50,4 +57,28 @@ public long getGeneration() { return generation; } + long getRetainedTablePayloadBytes() { + return retainedTablePayloadBytes; + } + + /** + * Compute the retained weight once before the value is published to a weight-bounded cache. + * Never opens the table store; failures fail closed as an incomplete estimate. + */ + synchronized MetaCacheSizeEstimate prepareForCachePublication(NameMapping key) { + if (sizeEstimate == null) { + sizeEstimate = MetaCacheSizeEstimator.estimateSafely("paimon_table_preparation_failed", + () -> { + retainedTablePayloadBytes = + PaimonCacheSizeEstimator.retainedTablePayloadBytes(paimonTable); + return PaimonCacheSizeEstimator.estimateTableEntry(key, this); + }); + } + return sizeEstimate; + } + + public MetaCacheSizeEstimate getSizeEstimate() { + return sizeEstimate == null + ? MetaCacheSizeEstimate.incomplete("not_prepared") : sizeEstimate; + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 723fab322725ed..3d9152e5eb1900 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -333,6 +333,87 @@ public R accept(DataTypeVisitor visitor) { Assert.assertSame(table, value.getSnapshot().getTable()); } + @Test + public void testTableEntryWeightCoversBaseOnlyScanAndReleasesOnInvalidate() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try { + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.singletonMap( + "meta.cache.paimon.table.max-weight", "8MB")); + Assert.assertTrue(cache.stats(catalogId).get(PaimonExternalMetaCache.ENTRY_TABLE).isWeightBounded()); + + NameMapping mapping = new NameMapping(catalogId, "db", "tbl", "db", "tbl"); + FileStoreTable table = newTableWithExtraFields("table_entry_weight", 64, true, 8); + Object lazyStoreBefore = readField(table, table.getClass(), "lazyStore"); + PaimonTableCacheValue value = new PaimonTableCacheValue(table); + org.apache.doris.datasource.metacache.MetaCacheEntry tables = + cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class); + tables.put(mapping, value); + + Assert.assertSame(value, tables.peekIfPresent(mapping)); + Assert.assertTrue(value.getSizeEstimate().getIncompleteReason(), + value.getSizeEstimate().isComplete()); + long estimate = value.getSizeEstimate().getBytes(); + Assert.assertTrue(estimate > 0L); + MetaCacheEntryStats stats = cache.stats(catalogId).get(PaimonExternalMetaCache.ENTRY_TABLE); + // The entry adds a fixed per-record overhead on top of the value estimate. + long reservedWeight = stats.getEstimatedWeight(); + Assert.assertTrue(reservedWeight >= estimate); + Assert.assertSame("table publication must not materialize FileStoreTable.store()", + lazyStoreBefore, readField(table, table.getClass(), "lazyStore")); + + // A base-only scan (fetchRowCount, table-only paths) opens the store and its RowType + // indexes on the admitted handle; the reserved weight already covers that graph. + long beforeScan = EstimatorCalibrationAssertions.graphSize(value); + materializeStoreGraph(table); + materializeRowTypeIndexes(table.schema()); + long afterScan = EstimatorCalibrationAssertions.graphSize(value); + Assert.assertTrue("scan must grow the retained graph", afterScan > beforeScan); + Assert.assertTrue("estimate " + estimate + " must cover the grown graph " + afterScan, + estimate >= afterScan); + Assert.assertEquals(reservedWeight, + cache.stats(catalogId).get(PaimonExternalMetaCache.ENTRY_TABLE).getEstimatedWeight()); + + tables.invalidateKey(mapping); + Assert.assertNull(tables.peekIfPresent(mapping)); + Assert.assertEquals(0L, + cache.stats(catalogId).get(PaimonExternalMetaCache.ENTRY_TABLE).getEstimatedWeight()); + + // Unsupported table implementations fail closed and stay outside the cache. + PaimonTableCacheValue unsupported = new PaimonTableCacheValue(Mockito.mock(Table.class)); + tables.put(mapping, unsupported); + Assert.assertNull(tables.peekIfPresent(mapping)); + Assert.assertFalse(unsupported.getSizeEstimate().isComplete()); + Assert.assertTrue(unsupported.getSizeEstimate().getIncompleteReason() + .startsWith("unsupported_paimon_table:")); + MetaCacheEntryStats rejected = cache.stats(catalogId).get(PaimonExternalMetaCache.ENTRY_TABLE); + Assert.assertEquals(1L, rejected.getWeightAdmissionRejectedCount()); + Assert.assertEquals(0L, rejected.getEstimatedWeight()); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testTableEntryFormulaAgainstJolOwnedGraph() throws Exception { + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + FileStoreTable smallTable = newTableWithExtraFields("jol_table_entry_narrow", 10, true, 0); + FileStoreTable populatedTable = newTableWithExtraFields("jol_table_entry_wide", 300, true, 50); + PaimonTableCacheValue small = new PaimonTableCacheValue(smallTable); + PaimonTableCacheValue populated = new PaimonTableCacheValue(populatedTable); + long smallEstimate = small.prepareForCachePublication(mapping).getBytes(); + long populatedEstimate = populated.prepareForCachePublication(mapping).getBytes(); + materializeRowTypeIndexes(smallTable.schema()); + materializeRowTypeIndexes(populatedTable.schema()); + materializeStoreGraph(smallTable); + materializeStoreGraph(populatedTable); + EstimatorCalibrationAssertions.assertConservativeDelta( + "paimon table entry", smallEstimate, populatedEstimate, small, populated); + } + @Test public void testSnapshotWeightEntryAndPrecomputedEstimate() throws Exception { ExecutorService executor = Executors.newSingleThreadExecutor(); diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_table_meta_cache.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_table_meta_cache.groovy index 2a3176688f505f..ee15e4da2fcd8b 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_table_meta_cache.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_table_meta_cache.groovy @@ -27,10 +27,12 @@ suite("test_paimon_table_meta_cache", "p0,external,doris,external_docker,externa String catalogWithCache = "test_paimon_table_cache_with_cache" String catalogNoCache = "test_paimon_table_cache_no_cache" + String catalogWeighted = "test_paimon_table_cache_weighted" String testDb = "paimon_cache_test_db" sql """drop catalog if exists ${catalogWithCache}""" sql """drop catalog if exists ${catalogNoCache}""" + sql """drop catalog if exists ${catalogWeighted}""" sql """ CREATE CATALOG ${catalogWithCache} PROPERTIES ( @@ -55,6 +57,19 @@ suite("test_paimon_table_meta_cache", "p0,external,doris,external_docker,externa ); """ + // A catalog-level memory bound puts the table handle and snapshot projections under weight. + sql """ + CREATE CATALOG ${catalogWeighted} PROPERTIES ( + 'type' = 'paimon', + 'warehouse' = 's3://warehouse/wh', + 's3.endpoint' = 'http://${externalEnvIp}:${minioPort}', + 's3.access_key' = 'admin', + 's3.secret_key' = 'password', + 's3.path.style.access' = 'true', + 'meta.cache.max-weight' = '128MB' + ); + """ + try { spark_paimon "CREATE DATABASE IF NOT EXISTS paimon.${testDb}" @@ -87,6 +102,48 @@ suite("test_paimon_table_meta_cache", "p0,external,doris,external_docker,externa def result3 = sql """select * from ${testDb}.test_insert order by id""" assertEquals(2, result3.size()) + // ==================== Test 1b: weight-bounded cache ==================== + logger.info("========== Test 1b: weight-bounded cache ==========") + sql """switch ${catalogWeighted}""" + def resultWeighted = sql """select * from ${testDb}.test_insert order by id""" + assertEquals(2, resultWeighted.size()) + // desc/table-only paths admit the base table handle; the scan above admits the snapshot. + sql """desc ${testDb}.test_insert""" + def weightStats = sql """ + select entry_name, weight_bounded, max_weight, estimated_weight, catalog_max_weight, + weight_reject_count, last_weight_reject_reason + from internal.information_schema.catalog_meta_cache_statistics + where catalog_name = "${catalogWeighted}" and engine_name = "paimon" and weight_bounded = true + order by entry_name; + """ + def weightedEntries = weightStats.collect { it[0] as String } + assertTrue(weightedEntries.contains("table")) + assertTrue(weightedEntries.contains("snapshot")) + for (row in weightStats) { + assertTrue((row[2] as long) > 0L) + assertEquals(0L, row[5] as long) + if (row[0] == "table" || row[0] == "snapshot") { + assertTrue((row[3] as long) > 0L) + } + } + // Refresh releases the table handle reservation with its projections; the next scan + // re-admits both without rejections. + sql """refresh table ${testDb}.test_insert""" + def resultWeightedRefreshed = sql """select * from ${testDb}.test_insert order by id""" + assertEquals(2, resultWeightedRefreshed.size()) + def weightStatsRefreshed = sql """ + select entry_name, estimated_weight, weight_reject_count + from internal.information_schema.catalog_meta_cache_statistics + where catalog_name = "${catalogWeighted}" and engine_name = "paimon" + and entry_name in ("table", "snapshot"); + """ + assertEquals(2, weightStatsRefreshed.size()) + for (row in weightStatsRefreshed) { + assertTrue((row[1] as long) > 0L) + assertEquals(0L, row[2] as long) + } + sql """switch ${catalogWithCache}""" + // ==================== Test 2: Schema Change (ADD COLUMN) ==================== logger.info("========== Test 2: Schema Change (ADD COLUMN) ==========") spark_paimon "DROP TABLE IF EXISTS paimon.${testDb}.test_add_column" @@ -124,5 +181,6 @@ suite("test_paimon_table_meta_cache", "p0,external,doris,external_docker,externa } sql """drop catalog if exists ${catalogWithCache}""" sql """drop catalog if exists ${catalogNoCache}""" + sql """drop catalog if exists ${catalogWeighted}""" } } From ec807ce66f46cb641bf33556ee0a903e66831f4b Mon Sep 17 00:00:00 2001 From: guoqiang Date: Tue, 18 Aug 2026 04:26:27 +0800 Subject: [PATCH 08/45] [fix](fe) Charge Iceberg range width for surviving merged partitions and retire Paimon projections of unpublished tables - IcebergUtils.loadPartitionInfo charges the per-column range endpoint width once per Doris partition that survives mergeOverlapPartitions instead of once per physical partition; the range endpoint and partition constants are recalibrated against own-array fixtures - PaimonExternalMetaCache treats an absent (rejected, expired) base table generation as stale so snapshot/schema projections keyed by it never accumulate --- .../iceberg/IcebergCacheSizeEstimator.java | 2 +- .../iceberg/IcebergPartitionInfo.java | 21 +++-- .../datasource/iceberg/IcebergUtils.java | 10 ++- .../paimon/PaimonExternalMetaCache.java | 17 +++- .../iceberg/IcebergExternalMetaCacheTest.java | 65 ++++++++++++-- .../paimon/PaimonExternalMetaCacheTest.java | 88 +++++++++++++++++++ 6 files changed, 184 insertions(+), 19 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java index 94283ca6b25bb3..02f6d30373eb56 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java @@ -152,7 +152,7 @@ final class IcebergCacheSizeEstimator { private static final long ENCRYPTED_KEY_BYTES = objectBytes(256L); // One retained IcebergPartition (value/transform ArrayLists) or one RangePartitionItem with a // single partition column plus its map entry; extra columns are charged by IcebergPartitionInfo. - private static final long PARTITION_BYTES = objectBytes(680L); + private static final long PARTITION_BYTES = objectBytes(696L); // Outer map entry and table share of one merged-overlap group; the alias set itself and its // contents are charged by IcebergPartitionInfo per enclosed partition name. private static final long PARTITION_ALIAS_BYTES = objectBytes(144L); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java index 05f469dfdc5229..0bfa8346036569 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java @@ -26,10 +26,11 @@ public class IcebergPartitionInfo { // Each RangePartitionItem endpoint holds one LiteralExpr per partition column beyond the - // first (createPartitionKey fills the vacancy with an infinity literal): literal, its lazy - // supplier, children list and array. Calibrated against JOL in IcebergExternalMetaCacheTest. + // first (createPartitionKey fills the vacancy with an infinity literal): the MIN literal, its + // lazy supplier/lambda, children list and array, plus the key/type list slots (JOL: 224 bytes + // on a compressed-oops JVM). Calibrated in IcebergExternalMetaCacheTest. private static final long RANGE_KEY_EXTRA_COLUMN_BYTES = - MetaCacheWeightUtils.estimatedObjectBytes(208L); + MetaCacheWeightUtils.estimatedObjectBytes(224L); private static final long RANGE_ENDPOINTS_PER_ITEM = 2L; // A merged-overlap alias group is a HashSet of the enclosed physical partition names; the // names themselves are shared with the partition maps. @@ -54,7 +55,7 @@ public IcebergPartitionInfo(Map nameToPartitionItem, Map> nameToIcebergPartitionNames) { this(nameToPartitionItem, nameToIcebergPartition, nameToIcebergPartitionNames, MetaCacheWeightUtils.saturatedAdd( - retainedPayloadBytes(nameToIcebergPartition), + retainedPayloadBytes(nameToPartitionItem, nameToIcebergPartition), partitionAliasBytes(nameToIcebergPartitionNames))); } @@ -88,7 +89,8 @@ public long getRetainedPayloadBytes() { return retainedPayloadBytes; } - private static long retainedPayloadBytes(Map partitions) { + private static long retainedPayloadBytes( + Map items, Map partitions) { if (partitions == null) { return 0L; } @@ -97,6 +99,15 @@ private static long retainedPayloadBytes(Map partition if (partition != null) { bytes = MetaCacheWeightUtils.saturatedAdd( bytes, partition.getRetainedPayloadBytes()); + } + } + if (items == null) { + return bytes; + } + // Range endpoints exist only for the Doris partitions that survived overlap merging. + for (String name : items.keySet()) { + IcebergPartition partition = partitions.get(name); + if (partition != null) { bytes = MetaCacheWeightUtils.saturatedAdd(bytes, partitionItemColumnBytes( partition.getPartitionValues() == null ? 0 : partition.getPartitionValues().size())); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index ef408a81e20636..078fd00c2b5726 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -1770,14 +1770,10 @@ public static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, T List partitionColumns = IcebergUtils.getSchemaCacheValue( dorisTable, schemaId, table).getPartitionColumns(); - long partitionItemColumnBytes = IcebergPartitionInfo.partitionItemColumnBytes( - partitionColumns.size()); for (IcebergPartition partition : icebergPartitions) { nameToPartition.put(partition.getPartitionName(), partition); retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( retainedPayloadBytes, partition.getRetainedPayloadBytes()); - retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( - retainedPayloadBytes, partitionItemColumnBytes); String transform = table.specs().get(partition.getSpecId()).fields().get(0).transform().toString(); Range partitionRange = getPartitionRange( partition.getPartitionValues().get(0), transform, partitionColumns); @@ -1785,6 +1781,12 @@ public static IcebergPartitionInfo loadPartitionInfo(ExternalTable dorisTable, T nameToPartitionItem.put(partition.getPartitionName(), item); } Map> partitionNameMap = mergeOverlapPartitions(nameToPartitionItem); + // Only the surviving Doris partitions keep their range endpoints; enclosed items were + // dropped by the merge, so the per-column width applies to the merged map size. + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd(retainedPayloadBytes, + MetaCacheWeightUtils.saturatedMultiply( + IcebergPartitionInfo.partitionItemColumnBytes(partitionColumns.size()), + nameToPartitionItem.size())); retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( retainedPayloadBytes, IcebergPartitionInfo.partitionAliasBytes(partitionNameMap)); return new IcebergPartitionInfo( diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index ec5436b98e174d..8f64dbe32d1ed4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -113,8 +113,7 @@ public PaimonSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { ignored -> executeAuthenticated(nameMapping, () -> latestSnapshotProjectionLoader.loadAtFence( nameMapping, fence, tableValue.getGeneration()))); - PaimonTableCacheValue currentTable = tableEntry.get(nameMapping.getCtlId()).peekIfPresent(nameMapping); - if (currentTable != null && currentTable.getGeneration() != tableValue.getGeneration()) { + if (!isCurrentTableGeneration(nameMapping, tableValue.getGeneration())) { entry.invalidateKeyIfSame(key, snapshotValue); } return snapshotValue; @@ -164,13 +163,23 @@ PaimonSchemaCacheValue getPaimonSchemaCacheValue( SchemaCacheValue schemaCacheValue = entry.get(key, ignored -> executeAuthenticated(nameMapping, () -> loadSchemaCacheValue(key, retainedTable))); - PaimonTableCacheValue currentTable = tableEntry.get(nameMapping.getCtlId()).peekIfPresent(nameMapping); - if (currentTable != null && currentTable.getGeneration() != tableGeneration) { + if (!isCurrentTableGeneration(nameMapping, tableGeneration)) { entry.invalidateKeyIfSame(key, schemaCacheValue); } return (PaimonSchemaCacheValue) schemaCacheValue; } + /** + * Snapshot and schema projections are keyed by the synthetic generation of the base table + * handle they were derived from. A generation that is no longer published (replaced, expired, + * or never admitted because its weight estimate was rejected) can never be looked up again, + * so its projections must not stay behind in the child entries. + */ + private boolean isCurrentTableGeneration(NameMapping nameMapping, long tableGeneration) { + PaimonTableCacheValue currentTable = tableEntry.get(nameMapping.getCtlId()).peekIfPresent(nameMapping); + return currentTable != null && currentTable.getGeneration() == tableGeneration; + } + private PaimonTableCacheValue loadTableCacheValue(NameMapping nameMapping) { return new PaimonTableCacheValue(tableLoader.load(nameMapping)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 419bb991f05de2..743a8fac9a6a89 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -1404,13 +1404,18 @@ public void testTableAndSnapshotFormulasAgainstJolOwnedGraphs() throws Exception // A spec that widened after the related-table check retains one more literal per range // endpoint and one more value/transform per partition; the projection charges the width // it actually loaded instead of the single field the check assumed. + // (Both fixtures already carry the shared MIN-literal type singleton of the second column.) + IcebergSnapshotCacheValue twoColumnSnapshot = new IcebergSnapshotCacheValue( + realPartitionInfo(32, 2), new IcebergSnapshot(-1L, 0L)); IcebergSnapshotCacheValue wideSnapshot = new IcebergSnapshotCacheValue( realPartitionInfo(32, 3), new IcebergSnapshot(-1L, 0L)); + long twoColumnSnapshotEstimate = IcebergCacheSizeEstimator.estimateSnapshotEntry( + key, twoColumnSnapshot).getBytes(); long wideSnapshotEstimate = IcebergCacheSizeEstimator.estimateSnapshotEntry( key, wideSnapshot).getBytes(); EstimatorCalibrationAssertions.assertConservativeDelta( - "iceberg wide snapshot partitions", populatedSnapshotEstimate, wideSnapshotEstimate, - populatedSnapshot, wideSnapshot); + "iceberg wide snapshot partitions", twoColumnSnapshotEstimate, wideSnapshotEstimate, + twoColumnSnapshot, wideSnapshot); // Overlapping physical partitions merge into one Doris partition that keeps every // enclosed name in a HashSet; the weight follows the set cardinality, not the group count. @@ -1422,6 +1427,23 @@ public void testTableAndSnapshotFormulasAgainstJolOwnedGraphs() throws Exception "iceberg partition aliases", populatedSnapshotEstimate, aliasedSnapshotEstimate, populatedSnapshot, aliasedSnapshot); + // When mergeOverlapPartitions() really drops the enclosed day ranges, only the surviving + // Doris partition keeps range endpoints: widening the spec must charge the extra + // endpoint columns once, not once per enclosed physical partition. + IcebergSnapshotCacheValue mergedSnapshot = new IcebergSnapshotCacheValue( + mergedPartitionInfo(32, 2), new IcebergSnapshot(-1L, 0L)); + IcebergSnapshotCacheValue mergedWideSnapshot = new IcebergSnapshotCacheValue( + mergedPartitionInfo(32, 3), new IcebergSnapshot(-1L, 0L)); + Assert.assertEquals(1, mergedWideSnapshot.getPartitionInfo().getNameToPartitionItem().size()); + Assert.assertEquals(32, mergedWideSnapshot.getPartitionInfo().getNameToIcebergPartition().size()); + long mergedSnapshotEstimate = IcebergCacheSizeEstimator.estimateSnapshotEntry( + key, mergedSnapshot).getBytes(); + long mergedWideSnapshotEstimate = IcebergCacheSizeEstimator.estimateSnapshotEntry( + key, mergedWideSnapshot).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg merged wide partitions", mergedSnapshotEstimate, mergedWideSnapshotEstimate, + mergedSnapshot, mergedWideSnapshot); + // A name mapping retains an element array per field once it has several historical names. IcebergSnapshotCacheValue singleNames = new IcebergSnapshotCacheValue( IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), @@ -2133,12 +2155,14 @@ private IcebergPartitionInfo realPartitionInfo( String name = "part=" + value; partitionItems.put(name, new org.apache.doris.catalog.RangePartitionItem( IcebergUtils.getPartitionRange(value, "day", partitionColumns))); - // Loaded partitions own one String per value and transform. + // Loaded partitions own one String per value; bucket/truncate transforms own their + // strings too (year/month/day literals are shared and only leave the estimate + // more conservative). List values = new ArrayList<>(); List transforms = new ArrayList<>(); for (int column = 0; column < partitionColumnCount; column++) { - values.add(new String(value)); - transforms.add(new String("day")); + values.add(new String(value.toCharArray())); + transforms.add(new String("day".toCharArray())); } partitions.put(name, new IcebergPartition(name, 0, 1L, 1L, 1L, 1L, 1L, values, transforms)); } @@ -2150,6 +2174,37 @@ private IcebergPartitionInfo realPartitionInfo( return new IcebergPartitionInfo(partitionItems, partitions, aliases); } + /** + * One "year" partition enclosing {@code partitionCount - 1} "day" partitions of that year; + * mergeOverlapPartitions() keeps a single Doris partition item owning every physical name. + */ + private IcebergPartitionInfo mergedPartitionInfo(int partitionCount, int partitionColumnCount) + throws Exception { + Map partitionItems = new java.util.HashMap<>(); + Map partitions = new java.util.HashMap<>(); + List partitionColumns = new ArrayList<>(); + for (int column = 0; column < partitionColumnCount; column++) { + partitionColumns.add(new org.apache.doris.catalog.Column( + "part" + column, org.apache.doris.catalog.PrimitiveType.DATETIMEV2)); + } + for (int index = 0; index < partitionCount; index++) { + String transform = index == 0 ? "year" : "day"; + String value = index == 0 ? "0" : Integer.toString(index); + String name = "part=" + transform + "-" + value; + partitionItems.put(name, new org.apache.doris.catalog.RangePartitionItem( + IcebergUtils.getPartitionRange(value, transform, partitionColumns))); + List values = new ArrayList<>(); + List transforms = new ArrayList<>(); + for (int column = 0; column < partitionColumnCount; column++) { + values.add(new String(value.toCharArray())); + transforms.add(new String(transform.toCharArray())); + } + partitions.put(name, new IcebergPartition(name, 0, 1L, 1L, 1L, 1L, 1L, values, transforms)); + } + Map> aliases = IcebergUtils.mergeOverlapPartitions(partitionItems); + return new IcebergPartitionInfo(partitionItems, partitions, aliases); + } + private org.apache.iceberg.DataFile dataFileWithMetrics(int index) { Map columnSizes = metricLongMap(index, 0); Map valueCounts = metricLongMap(index, 1); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 3d9152e5eb1900..f45ee5744e6889 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -716,6 +716,94 @@ public T execute(Callable task) throws Exception { } } + @Test + public void testRejectedBaseTableDoesNotAccumulateSnapshotOrSchemaProjections() { + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + PaimonExternalDatabase database = Mockito.mock(PaimonExternalDatabase.class); + PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.doReturn(catalog).when(catalogMgr) + .getCatalogOrException(Mockito.eq(1L), Mockito.any()); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog(1L); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + return task.call(); + } + }); + Mockito.doReturn(database).when(catalog).getDbNullable("db"); + Mockito.when(database.getTableNullable("tbl")).thenReturn(externalTable); + Mockito.doReturn(Optional.of(database)).when(catalog).getDb("db"); + Mockito.doReturn(Optional.of(externalTable)).when(database).getTable("tbl"); + Column partitionColumn = new Column("part", Type.INT); + Mockito.doReturn(new PaimonSchemaCacheValue( + Collections.singletonList(partitionColumn), + Collections.singletonList(partitionColumn), null)) + .when(externalTable).loadSchemaForCache(Mockito.any(), Mockito.anyLong()); + + // A mocked table has no supported layout, so its weight estimate is incomplete and every + // load is rejected by the weight-bounded table entry. + FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); + FileStoreTable latestSchemaTable = Mockito.mock(FileStoreTable.class); + FileStoreTable fenceTable = Mockito.mock(FileStoreTable.class); + FileStoreTable snapshotTable = Mockito.mock(FileStoreTable.class); + Snapshot latestSnapshot = Mockito.mock(Snapshot.class); + SchemaManager schemaManager = Mockito.mock(SchemaManager.class); + TableSchema latestSchema = Mockito.mock(TableSchema.class); + ReadBuilder readBuilder = Mockito.mock(ReadBuilder.class); + TableScan tableScan = Mockito.mock(TableScan.class); + Mockito.when(baseTable.copyWithLatestSchema()).thenReturn(latestSchemaTable); + Mockito.when(latestSchemaTable.latestSnapshot()).thenReturn(Optional.of(latestSnapshot)); + Mockito.when(latestSnapshot.id()).thenReturn(7L); + Mockito.when(latestSchemaTable.schemaManager()).thenReturn(schemaManager); + Mockito.when(schemaManager.latest()).thenReturn(Optional.of(latestSchema)); + Mockito.when(latestSchema.id()).thenReturn(3L); + Mockito.when(latestSchemaTable.copyWithoutTimeTravel(Mockito.anyMap())).thenReturn(fenceTable); + Mockito.when(fenceTable.copyWithoutTimeTravel(Mockito.anyMap())).thenReturn(snapshotTable); + Mockito.when(snapshotTable.options()).thenReturn(Collections.emptyMap()); + Mockito.when(snapshotTable.newReadBuilder()).thenReturn(readBuilder); + Mockito.when(readBuilder.newScan()).thenReturn(tableScan); + Mockito.when(tableScan.listPartitionEntries()).thenReturn(Collections.emptyList()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); + Mockito.when(catalog.getPaimonTable(mapping)).thenReturn(baseTable); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + cache.initCatalog(1L, Collections.singletonMap( + "meta.cache.paimon.table.max-weight", "1MB")); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + org.apache.doris.datasource.metacache.MetaCacheEntry tables = + cache.entry(1L, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class); + org.apache.doris.datasource.metacache.MetaCacheEntry< + PaimonSnapshotEntryKey, PaimonSnapshotCacheValue> snapshots = cache.entry( + 1L, PaimonExternalMetaCache.ENTRY_SNAPSHOT, + PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class); + org.apache.doris.datasource.metacache.MetaCacheEntry schemas = + cache.entry(1L, PaimonExternalMetaCache.ENTRY_SCHEMA, + PaimonSchemaCacheKey.class, SchemaCacheValue.class); + + for (int i = 0; i < 5; i++) { + Assert.assertEquals(7L, cache.getSnapshotCache(dorisTable).getSnapshot().getSnapshotId()); + Assert.assertNotNull(cache.getPaimonSchemaCacheValue(mapping, 3L)); + Assert.assertNull("rejected table handle must not be published", tables.peekIfPresent(mapping)); + Assert.assertEquals("projections of an unpublished generation must be retired", + 0L, snapshots.stats().getEstimatedSize()); + Assert.assertEquals(0L, schemas.stats().getEstimatedSize()); + } + Assert.assertEquals(10L, tables.stats().getWeightAdmissionRejectedCount()); + Mockito.verify(catalog, Mockito.times(10)).getPaimonTable(mapping); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + @Test public void testSnapshotEstimateSupportsPrivilegedTableWrapper() throws Exception { FileStoreTable table = newPartitionedTable("privileged_estimate", Collections.emptyMap()); From f5b9250902840718b18472d45f30e78fa62467ba Mon Sep 17 00:00:00 2001 From: guoqiang Date: Tue, 18 Aug 2026 05:56:45 +0800 Subject: [PATCH 09/45] [fix](fe) Rebind Iceberg projections on operational refresh and retire children of removed base tables - Iceberg: a table handle refresh that keeps the metadata generation but renews FileIO, encryption or location provider retires the snapshot projections frozen on the previous handle; post-load guards treat an absent (rejected) base table as stale - metacache: MetaCacheEntryRemovalListener delivers admitted values removed by eviction, expiry, collection or invalidation asynchronously; invalidateIf(BiPredicate) uses quiet lookups - Paimon: retire snapshot/schema projections keyed by a removed table generation, fenced by that generation --- .../iceberg/IcebergExternalMetaCache.java | 50 +++-- .../iceberg/IcebergTableCacheValue.java | 55 +++++ .../metacache/AbstractExternalMetaCache.java | 3 +- .../datasource/metacache/MetaCacheEntry.java | 88 +++++++- .../metacache/MetaCacheEntryDef.java | 26 ++- .../MetaCacheEntryRemovalListener.java | 33 +++ .../paimon/PaimonExternalMetaCache.java | 39 +++- .../iceberg/IcebergExternalMetaCacheTest.java | 206 +++++++++++++++++- .../metacache/MetaCacheEntryTest.java | 34 +++ .../paimon/PaimonExternalMetaCacheTest.java | 61 ++++++ 10 files changed, 569 insertions(+), 26 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryRemovalListener.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index e8b24299d53df2..6ddd8e8b9738df 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -198,10 +198,15 @@ public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { } return value; })); - IcebergTableCacheValue currentTable = tableEntry.get(nameMapping.getCtlId()).peekIfPresent(nameMapping); - if (currentTable != null && !tableValue.isSamePhysicalGeneration(currentTable)) { - // A query may have captured the previous table immediately before refresh publication. - // It can use that immutable value, but must not republish an unreachable old projection. + MetaCacheEntry tables = tableEntry.get(nameMapping.getCtlId()); + IcebergTableCacheValue currentTable = tables.peekIfPresent(nameMapping); + if (tables.isEffectivelyEnabled() + && (currentTable == null || !tableValue.isSameOperationalGeneration(currentTable))) { + // A query may have captured the previous table immediately before refresh publication, + // or loaded through a table handle that was never admitted (weight rejection). It can + // use that immutable value, but must not republish a projection no later lookup can + // reach or one frozen on superseded operational resources. (A disabled table entry + // never publishes; its physically keyed projections stay reusable.) entry.invalidateKeyIfSame(key, snapshotValue); } return snapshotValue; @@ -236,14 +241,16 @@ IcebergSchemaCacheValue getIcebergSchemaCacheValue( MetaCacheEntry entry = schemaEntry.get(nameMapping.getCtlId()); SchemaCacheValue schemaCacheValue = entry .get(key, ignored -> loadSchemaCacheValue(key, retainedTable)); - IcebergTableCacheValue currentTable = tableEntry.get(nameMapping.getCtlId()).peekIfPresent(nameMapping); - if (currentTable != null) { - Optional currentGeneration = IcebergSnapshotEntryKey.tryCreate( - nameMapping, currentTable.getRetainedIcebergTable()); - if (!currentGeneration.isPresent() - || !currentGeneration.get().getTableUuid().equals(generation.get().getTableUuid())) { - entry.invalidateKeyIfSame(key, schemaCacheValue); - } + MetaCacheEntry tables = tableEntry.get(nameMapping.getCtlId()); + IcebergTableCacheValue currentTable = tables.peekIfPresent(nameMapping); + Optional currentGeneration = currentTable == null + ? Optional.empty() + : IcebergSnapshotEntryKey.tryCreate(nameMapping, currentTable.getRetainedIcebergTable()); + if (tables.isEffectivelyEnabled() && (!currentGeneration.isPresent() + || !currentGeneration.get().getTableUuid().equals(generation.get().getTableUuid()))) { + // No published base table (replaced, expired or rejected at admission) can vouch for + // this projection; keep it out of the count-bounded schema cache. + entry.invalidateKeyIfSame(key, schemaCacheValue); } return (IcebergSchemaCacheValue) schemaCacheValue; } @@ -355,14 +362,17 @@ private SchemaCacheValue loadSchemaCacheValue(IcebergSchemaCacheKey key, Table r private void retireTableGeneration(NameMapping nameMapping, @Nullable IcebergTableCacheValue previousValue, IcebergTableCacheValue currentValue) { - if (previousValue != null && previousValue.isSamePhysicalGeneration(currentValue)) { + if (previousValue != null && previousValue.isSameOperationalGeneration(currentValue)) { return; } MetaCacheEntry snapshots = snapshotEntry.getIfInitialized(nameMapping.getCtlId()); if (snapshots != null) { - snapshots.invalidateIf(key -> key.getNameMapping().equals(nameMapping) - && !key.belongsTo(currentValue)); + // Projections of another metadata generation are unreachable. Projections of the same + // generation frozen on a previous handle keep that handle's FileIO (vended credentials) + // and location provider; scans bind to them, so they must be rebuilt from the new handle. + snapshots.invalidateIf((key, value) -> key.getNameMapping().equals(nameMapping) + && (!key.belongsTo(currentValue) || !sharesOperationalResources(currentValue, value))); } Optional currentUuid = currentValue.getTableUuid(); MetaCacheEntry schemas = @@ -373,6 +383,16 @@ private void retireTableGeneration(NameMapping nameMapping, } } + private static boolean sharesOperationalResources( + IcebergTableCacheValue currentValue, @Nullable IcebergSnapshotCacheValue projection) { + if (projection == null) { + return false; + } + Optional
    retainedTable = projection.getRetainedIcebergTable(); + // Count-mode projections do not retain a table handle; nothing to rebind. + return !retainedTable.isPresent() || currentValue.sharesOperationalResources(retainedTable.get()); + } + private IcebergSnapshotCacheValue loadSnapshotProjection( ExternalTable dorisTable, Table projectionTable, Table retainedTable, String retainedCurrentSnapshotJson, boolean isolateForQueries) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java index 0b6bd23c634381..84dcb045d70764 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java @@ -24,6 +24,8 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; +import org.apache.iceberg.io.FileIO; +import org.apache.iceberg.io.SupportsStorageCredentials; import java.util.Objects; import java.util.Optional; @@ -132,6 +134,59 @@ boolean isSamePhysicalGeneration(IcebergTableCacheValue other) { && Objects.equals(left.metadataFileLocation(), right.metadataFileLocation()); } + /** + * Same metadata generation served through the same operational resources. A catalog that + * reloads the same metadata file may still hand out a new FileIO carrying rotated vended + * credentials; projections frozen on the previous handle must not outlive that rotation. + */ + boolean isSameOperationalGeneration(IcebergTableCacheValue other) { + return isSamePhysicalGeneration(other) && sharesOperationalResources(other.icebergTable); + } + + boolean sharesOperationalResources(Table table) { + return sharesOperationalResources(icebergTable, table); + } + + /** True when both tables read and write through the same FileIO, encryption and locations. */ + static boolean sharesOperationalResources(Table left, Table right) { + if (left == right) { + return true; + } + if (left == null || right == null) { + return false; + } + return sameFileIo(left.io(), right.io()) + && sameResource(left.encryption(), right.encryption()) + && sameResource(left.locationProvider(), right.locationProvider()); + } + + private static boolean sameFileIo(FileIO left, FileIO right) { + if (left == right) { + return true; + } + if (left == null || right == null || left.getClass() != right.getClass()) { + return false; + } + try { + // Vended credentials live in the FileIO properties / storage credentials; equal + // configuration means equal credentials even across catalog reload instances. + return Objects.equals(left.properties(), right.properties()) + && Objects.equals(storageCredentials(left), storageCredentials(right)); + } catch (RuntimeException e) { + // A FileIO that cannot expose its configuration cannot prove it is unchanged. + return false; + } + } + + private static Object storageCredentials(FileIO fileIO) { + return fileIO instanceof SupportsStorageCredentials + ? ((SupportsStorageCredentials) fileIO).credentials() : null; + } + + private static boolean sameResource(Object left, Object right) { + return left == right || (left != null && right != null && left.getClass() == right.getClass()); + } + private TableMetadata retainedMetadata() { Table retainedTable = icebergTable; return retainedTable instanceof HasTableOperations diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java index 7dd629685b12a0..2568eaeedf2756 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java @@ -442,7 +442,8 @@ private MetaCacheEntry newMetaCacheEntry( wrapSchemaValidator(entryDef.getLoader(), entryDef.getValueType()), cacheSpec, refreshExecutor, entryDef.isAutoRefresh(), entryDef.isContextualOnly(), - entryDef.getSizeEstimator(), entryBudget, entryDef.getReplacementListener()); + entryDef.getSizeEstimator(), entryBudget, entryDef.getReplacementListener(), + entryDef.getRemovalListener()); } catch (RuntimeException | Error e) { if (entryBudget != null) { entryBudget.close(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java index e76bfc0540f6d0..06b982f283b290 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java @@ -37,6 +37,7 @@ import java.util.OptionalLong; import java.util.Set; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ConcurrentLinkedQueue; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.RejectedExecutionException; @@ -45,6 +46,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; +import java.util.function.BiPredicate; import java.util.function.Function; import java.util.function.Predicate; import javax.annotation.Nullable; @@ -62,7 +64,8 @@ public class MetaCacheEntry { private static final int REMOVAL_CLEANUP_BATCH_SIZE = 256; private static final long WEIGHT_REJECT_LOG_INTERVAL_MS = TimeUnit.MINUTES.toMillis(1L); // Direct Caffeine callbacks must not wait for admissionLock. A daemon drains one coalesced - // generation map per physical entry after callbacks return; cleanup tasks never capture values. + // generation map per physical entry after callbacks return; reservation cleanups never capture + // values, removal-listener notifications hold the removed value only until they are drained. private static final ExecutorService REMOVAL_CLEANUP_EXECUTOR = Executors.newSingleThreadExecutor(runnable -> { Thread thread = new Thread(runnable, "external-meta-cache-removal-cleanup"); thread.setDaemon(true); @@ -86,6 +89,12 @@ public class MetaCacheEntry { private final EntryBudget entryBudget; @Nullable private final MetaCacheEntryReplacementListener replacementListener; + @Nullable + private final MetaCacheEntryRemovalListener removalListener; + // Removed (key, value) pairs awaiting the asynchronous removal listener; drained together with + // the reservation cleanups so Caffeine's synchronous callback stays lock-free. + private final ConcurrentLinkedQueue> pendingRemovalNotifications = + new ConcurrentLinkedQueue<>(); private final boolean weightBounded; // Entries with publication-time work use the same generation-fenced refresh protocol even // before a max-weight is configured. This keeps estimation and dependency retirement on every @@ -159,6 +168,15 @@ public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec ca ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, @Nullable MetaCacheSizeEstimator sizeEstimator, @Nullable EntryBudget entryBudget, @Nullable MetaCacheEntryReplacementListener replacementListener) { + this(name, loader, cacheSpec, refreshExecutor, autoRefresh, contextualOnly, + sizeEstimator, entryBudget, replacementListener, null); + } + + public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, + ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, + @Nullable MetaCacheSizeEstimator sizeEstimator, @Nullable EntryBudget entryBudget, + @Nullable MetaCacheEntryReplacementListener replacementListener, + @Nullable MetaCacheEntryRemovalListener removalListener) { this.name = name; if (contextualOnly) { if (loader != null) { @@ -177,9 +195,10 @@ public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec ca this.sizeEstimator = sizeEstimator; this.entryBudget = entryBudget; this.replacementListener = replacementListener; + this.removalListener = removalListener; this.weightBounded = this.cacheSpec.isWeightBounded(); this.generationFencedRefresh = autoRefresh - && (sizeEstimator != null || replacementListener != null); + && (sizeEstimator != null || replacementListener != null || removalListener != null); if (weightBounded && (sizeEstimator == null || entryBudget == null)) { throw new IllegalArgumentException("weighted cache entry requires both estimator and budget: " + name); } @@ -207,7 +226,7 @@ public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec ca if (weightBounded) { cacheFactory.withSoftValues(); } - if (weightBounded || generationFencedRefresh) { + if (weightBounded || generationFencedRefresh || removalListener != null) { // Direct notification avoids queuing REPLACED values. The listener itself is lock-free // and delegates only current-owner cleanup, so it is safe under Caffeine's eviction lock. this.loadingData = cacheFactory.buildCacheWithSyncRemovalListener( @@ -416,6 +435,19 @@ public void invalidateKey(K key) { } public void invalidateIf(Predicate predicate) { + invalidateIf(predicate, null); + } + + /** + * Invalidate every mapping the predicate accepts. The value is the currently mapped one, or + * null when only a reservation or in-flight mutation record remains for the key. The lookup + * is quiet: it neither counts as an access nor triggers a refresh. + */ + public void invalidateIf(BiPredicate predicate) { + invalidateIf(null, predicate); + } + + private void invalidateIf(@Nullable Predicate keyPredicate, @Nullable BiPredicate predicate) { synchronized (admissionLock) { Set candidates = new HashSet<>(data.asMap().keySet()); candidates.addAll(keyMutationStates.keySet()); @@ -425,7 +457,10 @@ public void invalidateIf(Predicate predicate) { candidates.addAll(refreshRecords.keySet()); } for (K key : candidates) { - if (predicate.test(key)) { + boolean matched = keyPredicate != null + ? keyPredicate.test(key) + : predicate.test(key, data.policy().getIfPresentQuietly(key)); + if (matched) { advanceKeyMutation(key); if (weightBounded) { ReservationRecord record = reservations.get(key); @@ -476,6 +511,7 @@ public void close() { return; } invalidateAll(); + pendingRemovalNotifications.clear(); if (entryBudget != null) { entryBudget.close(); } @@ -529,6 +565,11 @@ public boolean isWeightBounded() { return weightBounded; } + /** True when this entry stores values at all (enabled with a positive capacity or weight). */ + public boolean isEffectivelyEnabled() { + return effectiveEnabled; + } + private AdmissionResult admitWeightedValue( K key, V value, @Nullable V expectedCurrent, boolean requireExpected, @Nullable KeyMutationToken expectedMutation, long expectedReservationGeneration, @@ -747,7 +788,7 @@ private void onRemoval(@Nullable K key, @Nullable V value, RemovalCause cause) { if (key == null) { return; } - if (!weightBounded && !generationFencedRefresh) { + if (!weightBounded && !generationFencedRefresh && removalListener == null) { return; } if (closed.get()) { @@ -758,6 +799,10 @@ private void onRemoval(@Nullable K key, @Nullable V value, RemovalCause cause) { if (cause == RemovalCause.REPLACED) { return; } + if (removalListener != null) { + pendingRemovalNotifications.add(new RemovedValue<>(key, value)); + scheduleRemovalCleanup(); + } if (Thread.holdsLock(admissionLock)) { // Other removals have already removed the Caffeine mapping and can release their owner // inline. A stale callback cannot release a replacement while its mapping is visible. @@ -816,6 +861,7 @@ private void scheduleRemovalCleanup() { private void drainRemovalCleanups() { try { + drainRemovalNotifications(); int processed = 0; for (Map.Entry cleanup : pendingRemovalGenerations.entrySet()) { if (processed++ >= REMOVAL_CLEANUP_BATCH_SIZE) { @@ -850,7 +896,8 @@ private void drainRemovalCleanups() { } } finally { removalCleanupScheduled.set(false); - if (!closed.get() && !pendingRemovalGenerations.isEmpty()) { + if (!closed.get() + && (!pendingRemovalGenerations.isEmpty() || !pendingRemovalNotifications.isEmpty())) { // One bounded task per turn prevents a hot entry from monopolizing the process-wide // cleanup executor; a later task is queued behind already scheduled catalogs. scheduleRemovalCleanup(); @@ -858,6 +905,35 @@ private void drainRemovalCleanups() { } } + private void drainRemovalNotifications() { + if (removalListener == null) { + return; + } + for (int processed = 0; processed < REMOVAL_CLEANUP_BATCH_SIZE; processed++) { + RemovedValue removed = pendingRemovalNotifications.poll(); + if (removed == null || closed.get()) { + return; + } + try { + removalListener.onRemoval(removed.key, removed.value); + } catch (RuntimeException e) { + LOG.warn("Failed to retire dependencies after removing external metadata cache entry {}", + name, e); + } + } + } + + private static final class RemovedValue { + private final K key; + @Nullable + private final V value; + + private RemovedValue(K key, @Nullable V value) { + this.key = key; + this.value = value; + } + } + private void cleanupRemovedReservation(K key, long expectedReservationGeneration, boolean evicted) { beforeRemovalCleanupLockForTest(key); synchronized (admissionLock) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java index 689d3b6dc7ca99..03dbd553f210d5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java @@ -105,11 +105,22 @@ public final class MetaCacheEntryDef { private final MetaCacheSizeEstimator sizeEstimator; @Nullable private final MetaCacheEntryReplacementListener replacementListener; + @Nullable + private final MetaCacheEntryRemovalListener removalListener; private MetaCacheEntryDef(String name, Class keyType, Class valueType, @Nullable Function loader, CacheSpec defaultCacheSpec, boolean autoRefresh, boolean contextualOnly, MetaCacheEntryInvalidation invalidation, @Nullable MetaCacheSizeEstimator sizeEstimator, @Nullable MetaCacheEntryReplacementListener replacementListener) { + this(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, contextualOnly, invalidation, + sizeEstimator, replacementListener, null); + } + + private MetaCacheEntryDef(String name, Class keyType, Class valueType, + @Nullable Function loader, CacheSpec defaultCacheSpec, boolean autoRefresh, boolean contextualOnly, + MetaCacheEntryInvalidation invalidation, @Nullable MetaCacheSizeEstimator sizeEstimator, + @Nullable MetaCacheEntryReplacementListener replacementListener, + @Nullable MetaCacheEntryRemovalListener removalListener) { this.name = Objects.requireNonNull(name, "entry name is required"); this.keyType = Objects.requireNonNull(keyType, "entry key type is required"); this.valueType = Objects.requireNonNull(valueType, "entry value type is required"); @@ -130,6 +141,7 @@ private MetaCacheEntryDef(String name, Class keyType, Class valueType, this.invalidation = Objects.requireNonNull(invalidation, "entry invalidation is required"); this.sizeEstimator = sizeEstimator; this.replacementListener = replacementListener; + this.removalListener = removalListener; } /** @@ -193,7 +205,7 @@ public static MetaCacheEntryDef contextualOnly( public MetaCacheEntryDef withSizeEstimator(MetaCacheSizeEstimator estimator) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, contextualOnly, invalidation, - Objects.requireNonNull(estimator, "estimator"), replacementListener); + Objects.requireNonNull(estimator, "estimator"), replacementListener, removalListener); } /** Return a definition that synchronously retires dependencies after a value replacement. */ @@ -201,6 +213,13 @@ public MetaCacheEntryDef withReplacementListener( MetaCacheEntryReplacementListener listener) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, contextualOnly, invalidation, sizeEstimator, + Objects.requireNonNull(listener, "listener"), removalListener); + } + + /** Return a definition that retires dependencies after an admitted value is removed. */ + public MetaCacheEntryDef withRemovalListener(MetaCacheEntryRemovalListener listener) { + return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, + autoRefresh, contextualOnly, invalidation, sizeEstimator, replacementListener, Objects.requireNonNull(listener, "listener")); } @@ -264,4 +283,9 @@ public MetaCacheSizeEstimator getSizeEstimator() { public MetaCacheEntryReplacementListener getReplacementListener() { return replacementListener; } + + @Nullable + public MetaCacheEntryRemovalListener getRemovalListener() { + return removalListener; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryRemovalListener.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryRemovalListener.java new file mode 100644 index 00000000000000..8fe179b8b90f57 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryRemovalListener.java @@ -0,0 +1,33 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.metacache; + +import javax.annotation.Nullable; + +/** + * Receives a value that left the entry through the normal removal path (capacity or weight + * eviction, expiry, soft-value collection, peer reclaim, explicit invalidation). Replacements are + * reported through {@link MetaCacheEntryReplacementListener} instead. The callback runs + * asynchronously after the removal, so it must be fenced by the removed value itself (its + * generation) rather than by whatever the entry currently publishes; {@code removedValue} is + * null when the value was already collected. + */ +@FunctionalInterface +public interface MetaCacheEntryRemovalListener { + void onRemoval(K key, @Nullable V removedValue); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index 8f64dbe32d1ed4..e953608bce3004 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -82,7 +82,8 @@ public PaimonExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCach this::loadTableCacheValue, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping)) .withSizeEstimator((key, value) -> value.prepareForCachePublication(key)) - .withReplacementListener(this::retireTableGeneration)); + .withReplacementListener(this::retireTableGeneration) + .withRemovalListener(this::retireRemovedTableGeneration)); snapshotEntry = registerEntry(MetaCacheEntryDef.contextualOnly(ENTRY_SNAPSHOT, PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(PaimonSnapshotEntryKey::getNameMapping)) @@ -226,6 +227,42 @@ private void retireTableGeneration(NameMapping nameMapping, } } + /** + * An admitted table handle left the entry through eviction, expiry, collection or explicit + * invalidation without a successor being published. Its synthetic generation can never be + * looked up again, so the projections keyed by it are garbage. The callback is delayed and + * fenced by the removed generation: it never touches the generation currently published. + */ + private void retireRemovedTableGeneration(NameMapping nameMapping, + @Nullable PaimonTableCacheValue removedValue) { + MetaCacheEntry tables = + tableEntry.getIfInitialized(nameMapping.getCtlId()); + PaimonTableCacheValue currentValue = tables == null ? null : tables.peekIfPresent(nameMapping); + long currentGeneration = currentValue == null ? -1L : currentValue.getGeneration(); + long removedGeneration = removedValue == null ? -1L : removedValue.getGeneration(); + if (removedValue != null && removedGeneration == currentGeneration) { + // The removed handle was republished; its projections are addressable again. + return; + } + // A collected (null) value has no generation left: everything that is not derived from + // the currently published handle is unreachable. + java.util.function.LongPredicate retired = removedValue == null + ? generation -> generation != currentGeneration + : generation -> generation == removedGeneration; + MetaCacheEntry snapshots = + snapshotEntry.getIfInitialized(nameMapping.getCtlId()); + if (snapshots != null) { + snapshots.invalidateIf(key -> key.getNameMapping().equals(nameMapping) + && retired.test(key.getTableGeneration())); + } + MetaCacheEntry schemas = + schemaEntry.getIfInitialized(nameMapping.getCtlId()); + if (schemas != null) { + schemas.invalidateIf(key -> key.getNameMapping().equals(nameMapping) + && retired.test(key.getTableGeneration())); + } + } + @Override protected Map catalogPropertyCompatibilityMap() { Map compatibility = new java.util.HashMap<>( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 743a8fac9a6a89..675120de9dcfc0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -216,6 +216,172 @@ public void testReplacingTableGenerationRetiresSnapshotAndSchemaProjection() { } } + @Test + public void testSameGenerationRefreshWithRenewedFileIoRetiresSnapshotProjection() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + try { + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.emptyMap()); + NameMapping mapping = NameMapping.createForTest(catalogId, "db", "tbl"); + TableMetadata metadata = metadataWithLocation("/metadata/rotate-v1.json"); + // Same metadata file, but each catalog reload vends a FileIO with fresh credentials. + IcebergTableCacheValue first = new IcebergTableCacheValue( + tableWithMetadata(metadata, new PropertiesFileIO("token", "one"))); + IcebergTableCacheValue rotated = new IcebergTableCacheValue( + tableWithMetadata(metadata, new PropertiesFileIO("token", "two"))); + IcebergTableCacheValue equivalent = new IcebergTableCacheValue( + tableWithMetadata(metadata, new PropertiesFileIO("token", "two"))); + Assert.assertTrue(first.isSamePhysicalGeneration(rotated)); + Assert.assertFalse(first.isSameOperationalGeneration(rotated)); + Assert.assertTrue(rotated.isSameOperationalGeneration(equivalent)); + + MetaCacheEntry tables = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class); + MetaCacheEntry snapshots = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_SNAPSHOT, + IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class); + MetaCacheEntry schemas = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_SCHEMA, + IcebergSchemaCacheKey.class, SchemaCacheValue.class); + tables.put(mapping, first); + IcebergSnapshotEntryKey snapshotKey = IcebergSnapshotEntryKey.tryCreate( + mapping, first.getRetainedIcebergTable()).get(); + IcebergSnapshotCacheValue firstProjection = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), + Optional.empty(), first.getRetainedIcebergTable()); + snapshots.put(snapshotKey, firstProjection); + IcebergSchemaCacheKey schemaKey = new IcebergSchemaCacheKey(mapping, first.getTableUuid().get(), 0L); + schemas.put(schemaKey, new SchemaCacheValue(Collections.emptyList())); + + // Rotated credentials: the projection frozen on the previous handle must not survive, + // schema projections are generation-keyed and stay. + tables.put(mapping, rotated); + Assert.assertNull(snapshots.peekIfPresent(snapshotKey)); + Assert.assertNotNull(schemas.peekIfPresent(schemaKey)); + + IcebergSnapshotCacheValue rotatedProjection = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), + Optional.empty(), rotated.getRetainedIcebergTable()); + snapshots.put(snapshotKey, rotatedProjection); + // An equivalent reload (same credentials, new FileIO instance) keeps the projection. + tables.put(mapping, equivalent); + Assert.assertSame(rotatedProjection, snapshots.peekIfPresent(snapshotKey)); + + // A count-mode projection retains no handle and is never bound to credentials. + IcebergSnapshotCacheValue countProjection = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L)); + snapshots.put(snapshotKey, countProjection); + tables.put(mapping, new IcebergTableCacheValue( + tableWithMetadata(metadata, new PropertiesFileIO("token", "three")))); + Assert.assertSame(countProjection, snapshots.peekIfPresent(snapshotKey)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testRejectedTableGenerationsDoNotAccumulateSnapshotOrSchemaProjections() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + IcebergMetadataOps metadataOps = Mockito.mock(IcebergMetadataOps.class); + Mockito.when(catalog.getMetadataOps()).thenReturn(metadataOps); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + return task.call(); + } + }); + // Every reload advances the metadata location; every publication is rejected. + Mockito.when(metadataOps.loadTable("remote_db", "remote_tbl")).thenReturn( + tableWithMetadataLocation("/metadata/rejected-v1.json"), + tableWithMetadataLocation("/metadata/rejected-v2.json"), + tableWithMetadataLocation("/metadata/rejected-v3.json")); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor) { + @Override + protected CatalogIf getCatalog(long catalogId) { + return catalog; + } + + @Override + MetaCacheSizeEstimate prepareTableForCachePublication( + NameMapping nameMapping, IcebergTableCacheValue value) { + return MetaCacheSizeEstimate.incomplete("test_rejection"); + } + }; + try { + cache.initCatalog(1L, Collections.singletonMap( + "meta.cache.iceberg.table.max-weight", "4MB")); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); + IcebergExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + MetaCacheEntry tables = cache.entry( + 1L, IcebergExternalMetaCache.ENTRY_TABLE, NameMapping.class, IcebergTableCacheValue.class); + MetaCacheEntry snapshots = cache.entry( + 1L, IcebergExternalMetaCache.ENTRY_SNAPSHOT, + IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class); + MetaCacheEntry schemas = cache.entry( + 1L, IcebergExternalMetaCache.ENTRY_SCHEMA, + IcebergSchemaCacheKey.class, SchemaCacheValue.class); + + for (int i = 1; i <= 3; i++) { + IcebergSnapshotCacheValue projection = cache.getSnapshotCache(dorisTable); + Assert.assertNotNull(projection); + Assert.assertNull("rejected table handle must not be published", tables.peekIfPresent(mapping)); + Assert.assertEquals("projections of an unpublished generation must be retired", + 0L, snapshots.stats().getEstimatedSize()); + Assert.assertEquals(i, tables.stats().getWeightAdmissionRejectedCount()); + } + Mockito.verify(metadataOps, Mockito.times(3)).loadTable("remote_db", "remote_tbl"); + + // Schema projections keyed by a rejected generation are not kept either. + IcebergTableCacheValue rejected = new IcebergTableCacheValue( + tableWithMetadataLocation("/metadata/rejected-schema.json")); + IcebergSchemaCacheKey schemaKey = new IcebergSchemaCacheKey( + mapping, rejected.getTableUuid().get(), 0L); + IcebergSchemaCacheValue schemaValue = new IcebergSchemaCacheValue( + Collections.emptyList(), Collections.emptyList()); + schemas.put(schemaKey, schemaValue); + Assert.assertSame(schemaValue, cache.getIcebergSchemaCacheValue( + mapping, 0L, rejected.getRetainedIcebergTable())); + Assert.assertNull(schemas.peekIfPresent(schemaKey)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testDisabledTableCacheKeepsPhysicallyKeyedSchemaProjections() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + try { + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.singletonMap("meta.cache.iceberg.table.enable", "false")); + NameMapping mapping = NameMapping.createForTest(catalogId, "db", "tbl"); + IcebergTableCacheValue table = new IcebergTableCacheValue( + tableWithMetadataLocation("/metadata/disabled-table-cache.json")); + IcebergSchemaCacheKey schemaKey = new IcebergSchemaCacheKey(mapping, table.getTableUuid().get(), 0L); + IcebergSchemaCacheValue schemaValue = new IcebergSchemaCacheValue( + Collections.emptyList(), Collections.emptyList()); + MetaCacheEntry schemas = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_SCHEMA, + IcebergSchemaCacheKey.class, SchemaCacheValue.class); + schemas.put(schemaKey, schemaValue); + + // No table handle is ever published, but the projection is keyed by the table UUID and + // stays valid for every reload of that table. + Assert.assertSame(schemaValue, cache.getIcebergSchemaCacheValue( + mapping, 0L, table.getRetainedIcebergTable())); + Assert.assertSame(schemaValue, schemas.peekIfPresent(schemaKey)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + @Test public void testOldGenerationSchemaLoadCannotRepopulateAfterTableReplacement() { ExecutorService executor = Executors.newSingleThreadExecutor(); @@ -1974,12 +2140,48 @@ private static String repeatedCharacter(char character, int count) { } private Table tableWithMetadataLocation(String metadataLocation) { + return tableWithMetadata(metadataWithLocation(metadataLocation), null); + } + + private TableMetadata metadataWithLocation(String metadataLocation) { Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); TableMetadata metadata = TableMetadata.newTableMetadata(schema, PartitionSpec.unpartitioned(), "file:/warehouse/db/tbl", Collections.emptyMap()); - metadata = TableMetadata.buildFrom(metadata).discardChanges() + return TableMetadata.buildFrom(metadata).discardChanges() .withMetadataLocation(metadataLocation).build(); - return new BaseTable(new StaticTableOperations(metadata, null), "db.tbl"); + } + + private Table tableWithMetadata(TableMetadata metadata, FileIO io) { + return new BaseTable(new StaticTableOperations(metadata, io), "db.tbl"); + } + + /** A FileIO whose identity is its configuration, like a catalog-vended S3 FileIO. */ + private static final class PropertiesFileIO implements FileIO { + private final Map properties; + + private PropertiesFileIO(String key, String value) { + this.properties = Collections.singletonMap(key, value); + } + + @Override + public Map properties() { + return properties; + } + + @Override + public InputFile newInputFile(String path) { + throw new UnsupportedOperationException(); + } + + @Override + public org.apache.iceberg.io.OutputFile newOutputFile(String path) { + throw new UnsupportedOperationException(); + } + + @Override + public void deleteFile(String path) { + throw new UnsupportedOperationException(); + } } private IcebergTableCacheValue tableValueWithFields(int fieldCount) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java index 5607d604e1b01e..2d8140277e2d98 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java @@ -1844,6 +1844,40 @@ private void awaitValue(MetaCacheEntry entry, String key, String Assert.assertEquals(expected, entry.peekIfPresent(key)); } + @Test + public void testRemovalListenerReceivesRemovedValuesButNotReplacements() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + java.util.List removed = java.util.Collections.synchronizedList(new java.util.ArrayList<>()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "removal-listener", key -> 1, + CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), + refreshExecutor, false, false, null, null, null, + (key, value) -> removed.add(key + "=" + value)); + try { + entry.put("k", 1); + entry.put("k", 2); + entry.put("gone", 7); + entry.invalidateKey("gone"); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); + while (removed.isEmpty() && System.nanoTime() < deadline) { + Thread.sleep(10L); + } + Assert.assertEquals(java.util.Collections.singletonList("gone=7"), removed); + Assert.assertEquals(Integer.valueOf(2), entry.peekIfPresent("k")); + + entry.invalidateIf((key, value) -> Integer.valueOf(2).equals(value)); + Assert.assertNull(entry.peekIfPresent("k")); + deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); + while (removed.size() < 2 && System.nanoTime() < deadline) { + Thread.sleep(10L); + } + Assert.assertEquals(java.util.Arrays.asList("gone=7", "k=2"), removed); + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + private void awaitGlobalWeight(ExternalMetaCacheBudgetManager manager, long expected) throws InterruptedException { long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index f45ee5744e6889..f7d9b44174b058 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -716,6 +716,67 @@ public T execute(Callable task) throws Exception { } } + @Test + public void testExpiredBaseTableRetiresSnapshotAndSchemaProjectionsBeforeReplacement() throws Exception { + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try { + long catalogId = 1L; + Map properties = new HashMap<>(); + properties.put("meta.cache.paimon.table.ttl-second", "1"); + properties.put("meta.cache.paimon.snapshot.ttl-second", "3600"); + cache.initCatalog(catalogId, properties); + NameMapping mapping = new NameMapping(catalogId, "db", "tbl", "db", "tbl"); + PaimonTableCacheValue table = new PaimonTableCacheValue(Mockito.mock(Table.class)); + org.apache.doris.datasource.metacache.MetaCacheEntry tables = + cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class); + org.apache.doris.datasource.metacache.MetaCacheEntry< + PaimonSnapshotEntryKey, PaimonSnapshotCacheValue> snapshots = cache.entry( + catalogId, PaimonExternalMetaCache.ENTRY_SNAPSHOT, + PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class); + org.apache.doris.datasource.metacache.MetaCacheEntry schemas = + cache.entry(catalogId, PaimonExternalMetaCache.ENTRY_SCHEMA, + PaimonSchemaCacheKey.class, SchemaCacheValue.class); + tables.put(mapping, table); + PaimonSnapshotEntryKey snapshotKey = new PaimonSnapshotEntryKey(mapping, 1L, 2L, table.getGeneration()); + PaimonSchemaCacheKey schemaKey = new PaimonSchemaCacheKey(mapping, table.getGeneration(), 2L); + snapshots.put(snapshotKey, new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, 2L, table.getPaimonTable()))); + schemas.put(schemaKey, new SchemaCacheValue(Collections.emptyList())); + + Thread.sleep(1_500L); + com.github.benmanes.caffeine.cache.Cache caffeine = + (com.github.benmanes.caffeine.cache.Cache) readField( + tables, org.apache.doris.datasource.metacache.MetaCacheEntry.class, "loadingData"); + caffeine.cleanUp(); + Assert.assertNull("idle table handle must expire", tables.peekIfPresent(mapping)); + + // Expiry is a plain removal with no successor published: the delayed removal callback + // alone retires the projections keyed by the expired generation. + long deadline = System.nanoTime() + java.util.concurrent.TimeUnit.SECONDS.toNanos(5L); + while ((snapshots.peekIfPresent(snapshotKey) != null || schemas.peekIfPresent(schemaKey) != null) + && System.nanoTime() < deadline) { + Thread.sleep(20L); + } + Assert.assertNull(snapshots.peekIfPresent(snapshotKey)); + Assert.assertNull(schemas.peekIfPresent(schemaKey)); + Assert.assertNull(tables.peekIfPresent(mapping)); + + // A successor published afterwards keeps its own projections. + PaimonTableCacheValue next = new PaimonTableCacheValue(Mockito.mock(Table.class)); + tables.put(mapping, next); + PaimonSnapshotEntryKey nextSnapshotKey = new PaimonSnapshotEntryKey(mapping, 1L, 2L, next.getGeneration()); + snapshots.put(nextSnapshotKey, new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, 2L, next.getPaimonTable()))); + Assert.assertNotNull(snapshots.peekIfPresent(nextSnapshotKey)); + Assert.assertSame(next, tables.peekIfPresent(mapping)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + @Test public void testRejectedBaseTableDoesNotAccumulateSnapshotOrSchemaProjections() { PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); From 29ffdd681c1a7317aaab8462c549c1402a8d70c7 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Tue, 18 Aug 2026 07:28:36 +0800 Subject: [PATCH 10/45] [fix](fe) Queue removal tokens only, revalidate Iceberg projections on ineffective bases and keep one Paimon latest projection per generation - metacache: removal listeners receive a token extracted at removal time so retired values never wait outside the memory budget; isWeightAccounting() gates publication sizing so ineffective weighted entries skip preparation and query isolation - Iceberg: snapshot hits are revalidated against the fresh table handle's operational resources whenever the base entry publishes nothing (max-weight 0, table cache disabled) - Paimon: bypass generation-keyed child caches when the base entry is ineffective; after a load keep only the most recently observed latest-fence projection of a table generation (covers reversed completion and rollback) --- .../iceberg/IcebergExternalMetaCache.java | 32 ++- .../metacache/AbstractExternalMetaCache.java | 2 +- .../datasource/metacache/MetaCacheEntry.java | 63 ++-- .../metacache/MetaCacheEntryDef.java | 31 +- .../MetaCacheEntryRemovalListener.java | 18 +- .../paimon/PaimonExternalMetaCache.java | 124 +++++++- .../iceberg/IcebergExternalMetaCacheTest.java | 134 +++++++++ .../metacache/MetaCacheEntryTest.java | 64 ++++- .../paimon/PaimonExternalMetaCacheTest.java | 272 ++++++++++++++---- 9 files changed, 631 insertions(+), 109 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index 6ddd8e8b9738df..92fe89d2ecf536 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -53,6 +53,7 @@ import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.function.Consumer; +import java.util.function.Function; import javax.annotation.Nullable; /** @@ -154,7 +155,7 @@ Table getQueryScopedIcebergTable(ExternalTable dorisTable) { private Table createQueryTable( NameMapping nameMapping, IcebergTableCacheValue tableValue) { boolean isolateForQueries = tableValue.isQueryIsolationPrepared() - || snapshotEntry.get(nameMapping.getCtlId()).isWeightBounded(); + || snapshotEntry.get(nameMapping.getCtlId()).isWeightAccounting(); if (!isolateForQueries) { return tableValue.getIcebergTable(); } @@ -184,8 +185,8 @@ public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { MetaCacheEntry entry = snapshotEntry.get(nameMapping.getCtlId()); boolean isolateForQueries = tableValue.isQueryIsolationPrepared() - || entry.isWeightBounded(); - IcebergSnapshotCacheValue snapshotValue = entry.get(key, + || entry.isWeightAccounting(); + Function projectionLoader = ignored -> executeAuthenticated(nameMapping.getCtlId(), () -> { Table projectionTable = isolateForQueries ? tableValue.newQueryScopedTable() : tableValue.getIcebergTable(); @@ -193,11 +194,19 @@ public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { dorisTable, projectionTable, tableValue.getRetainedIcebergTable(), tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries); - if (entry.isWeightBounded()) { + if (entry.isWeightAccounting()) { value.prepareForCachePublication(key); } return value; - })); + }); + IcebergSnapshotCacheValue snapshotValue = entry.get(key, projectionLoader); + if (!sharesOperationalResources(tableValue, snapshotValue)) { + // A hit frozen on a previous handle of the same metadata generation keeps that handle's + // FileIO (vended credentials). Whether or not the base entry publishes handles, scans + // bind to the projection, so rebuild it from the handle this lookup just obtained. + entry.invalidateKeyIfSame(key, snapshotValue); + snapshotValue = entry.get(key, projectionLoader); + } MetaCacheEntry tables = tableEntry.get(nameMapping.getCtlId()); IcebergTableCacheValue currentTable = tables.peekIfPresent(nameMapping); if (tables.isEffectivelyEnabled() @@ -205,8 +214,9 @@ public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { // A query may have captured the previous table immediately before refresh publication, // or loaded through a table handle that was never admitted (weight rejection). It can // use that immutable value, but must not republish a projection no later lookup can - // reach or one frozen on superseded operational resources. (A disabled table entry - // never publishes; its physically keyed projections stay reusable.) + // reach or one frozen on superseded operational resources. (An ineffective table entry + // never publishes; its physically keyed projections stay reusable and are revalidated + // against the fresh handle above.) entry.invalidateKeyIfSame(key, snapshotValue); } return snapshotValue; @@ -269,7 +279,7 @@ public ManifestCacheValue getManifestCacheValue(ExternalTable dorisTable, } return manifestEntry.get(key, ignored -> loadManifestCacheValue( - manifest, icebergTable, key.getContent(), manifestEntry.isWeightBounded())); + manifest, icebergTable, key.getContent(), manifestEntry.isWeightAccounting())); } @Override @@ -297,7 +307,7 @@ private IcebergTableCacheValue loadTableCacheValue(NameMapping nameMapping) { IcebergTableCacheValue value = new IcebergTableCacheValue(table); MetaCacheEntry currentEntry = tableEntry.getIfInitialized(nameMapping.getCtlId()); - if (currentEntry != null && currentEntry.isWeightBounded()) { + if (currentEntry != null && currentEntry.isWeightAccounting()) { prepareTableForCachePublication(nameMapping, value); } return value; @@ -388,6 +398,10 @@ private static boolean sharesOperationalResources( if (projection == null) { return false; } + if (projection.getRetainedIcebergTable().map(table -> table == currentValue.getRetainedIcebergTable()) + .orElse(false)) { + return true; + } Optional
    retainedTable = projection.getRetainedIcebergTable(); // Count-mode projections do not retain a table handle; nothing to rebind. return !retainedTable.isPresent() || currentValue.sharesOperationalResources(retainedTable.get()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java index 2568eaeedf2756..30c84cc3715c85 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java @@ -443,7 +443,7 @@ private MetaCacheEntry newMetaCacheEntry( cacheSpec, refreshExecutor, entryDef.isAutoRefresh(), entryDef.isContextualOnly(), entryDef.getSizeEstimator(), entryBudget, entryDef.getReplacementListener(), - entryDef.getRemovalListener()); + entryDef.getRemovalTokenExtractor(), entryDef.getRemovalListener()); } catch (RuntimeException | Error e) { if (entryBudget != null) { entryBudget.close(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java index 06b982f283b290..c95c76ad182f23 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java @@ -64,8 +64,7 @@ public class MetaCacheEntry { private static final int REMOVAL_CLEANUP_BATCH_SIZE = 256; private static final long WEIGHT_REJECT_LOG_INTERVAL_MS = TimeUnit.MINUTES.toMillis(1L); // Direct Caffeine callbacks must not wait for admissionLock. A daemon drains one coalesced - // generation map per physical entry after callbacks return; reservation cleanups never capture - // values, removal-listener notifications hold the removed value only until they are drained. + // generation map per physical entry after callbacks return; cleanup tasks never capture values. private static final ExecutorService REMOVAL_CLEANUP_EXECUTOR = Executors.newSingleThreadExecutor(runnable -> { Thread thread = new Thread(runnable, "external-meta-cache-removal-cleanup"); thread.setDaemon(true); @@ -90,10 +89,14 @@ public class MetaCacheEntry { @Nullable private final MetaCacheEntryReplacementListener replacementListener; @Nullable - private final MetaCacheEntryRemovalListener removalListener; - // Removed (key, value) pairs awaiting the asynchronous removal listener; drained together with - // the reservation cleanups so Caffeine's synchronous callback stays lock-free. - private final ConcurrentLinkedQueue> pendingRemovalNotifications = + private final Function removalTokenExtractor; + @Nullable + private final MetaCacheEntryRemovalListener removalListener; + // Removed (key, token) pairs awaiting the asynchronous removal listener; drained together with + // the reservation cleanups so Caffeine's synchronous callback stays lock-free. Only the token + // is queued: the removed value's reservation is released with the removal, so keeping the + // value here would hold retired graphs outside every budget. + private final ConcurrentLinkedQueue> pendingRemovalNotifications = new ConcurrentLinkedQueue<>(); private final boolean weightBounded; // Entries with publication-time work use the same generation-fenced refresh protocol even @@ -169,14 +172,16 @@ public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec ca @Nullable MetaCacheSizeEstimator sizeEstimator, @Nullable EntryBudget entryBudget, @Nullable MetaCacheEntryReplacementListener replacementListener) { this(name, loader, cacheSpec, refreshExecutor, autoRefresh, contextualOnly, - sizeEstimator, entryBudget, replacementListener, null); + sizeEstimator, entryBudget, replacementListener, null, null); } + @SuppressWarnings("unchecked") public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec cacheSpec, ExecutorService refreshExecutor, boolean autoRefresh, boolean contextualOnly, @Nullable MetaCacheSizeEstimator sizeEstimator, @Nullable EntryBudget entryBudget, @Nullable MetaCacheEntryReplacementListener replacementListener, - @Nullable MetaCacheEntryRemovalListener removalListener) { + @Nullable Function removalTokenExtractor, + @Nullable MetaCacheEntryRemovalListener removalListener) { this.name = name; if (contextualOnly) { if (loader != null) { @@ -195,7 +200,11 @@ public MetaCacheEntry(String name, @Nullable Function loader, CacheSpec ca this.sizeEstimator = sizeEstimator; this.entryBudget = entryBudget; this.replacementListener = replacementListener; - this.removalListener = removalListener; + if ((removalListener == null) != (removalTokenExtractor == null)) { + throw new IllegalArgumentException("removal listener requires a token extractor: " + name); + } + this.removalTokenExtractor = (Function) removalTokenExtractor; + this.removalListener = (MetaCacheEntryRemovalListener) removalListener; this.weightBounded = this.cacheSpec.isWeightBounded(); this.generationFencedRefresh = autoRefresh && (sizeEstimator != null || replacementListener != null || removalListener != null); @@ -570,6 +579,15 @@ public boolean isEffectivelyEnabled() { return effectiveEnabled; } + /** + * True when publication sizing can lead to a weighted admission: an entry may be configured + * with a weight bound yet be ineffective (max-weight 0, disabled, zero TTL or capacity), in + * which case preparing values for publication is pure waste. + */ + public boolean isWeightAccounting() { + return weightBounded && effectiveEnabled; + } + private AdmissionResult admitWeightedValue( K key, V value, @Nullable V expectedCurrent, boolean requireExpected, @Nullable KeyMutationToken expectedMutation, long expectedReservationGeneration, @@ -800,7 +818,7 @@ private void onRemoval(@Nullable K key, @Nullable V value, RemovalCause cause) { return; } if (removalListener != null) { - pendingRemovalNotifications.add(new RemovedValue<>(key, value)); + pendingRemovalNotifications.add(new RemovedToken<>(key, removalToken(value))); scheduleRemovalCleanup(); } if (Thread.holdsLock(admissionLock)) { @@ -905,17 +923,30 @@ private void drainRemovalCleanups() { } } + @Nullable + private Object removalToken(@Nullable V value) { + if (value == null || removalTokenExtractor == null) { + return null; + } + try { + return removalTokenExtractor.apply(value); + } catch (RuntimeException e) { + LOG.warn("Failed to extract the removal token of external metadata cache entry {}", name, e); + return null; + } + } + private void drainRemovalNotifications() { if (removalListener == null) { return; } for (int processed = 0; processed < REMOVAL_CLEANUP_BATCH_SIZE; processed++) { - RemovedValue removed = pendingRemovalNotifications.poll(); + RemovedToken removed = pendingRemovalNotifications.poll(); if (removed == null || closed.get()) { return; } try { - removalListener.onRemoval(removed.key, removed.value); + removalListener.onRemoval(removed.key, removed.token); } catch (RuntimeException e) { LOG.warn("Failed to retire dependencies after removing external metadata cache entry {}", name, e); @@ -923,14 +954,14 @@ private void drainRemovalNotifications() { } } - private static final class RemovedValue { + private static final class RemovedToken { private final K key; @Nullable - private final V value; + private final Object token; - private RemovedValue(K key, @Nullable V value) { + private RemovedToken(K key, @Nullable Object token) { this.key = key; - this.value = value; + this.token = token; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java index 03dbd553f210d5..25f7a21626af73 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryDef.java @@ -106,21 +106,24 @@ public final class MetaCacheEntryDef { @Nullable private final MetaCacheEntryReplacementListener replacementListener; @Nullable - private final MetaCacheEntryRemovalListener removalListener; + private final Function removalTokenExtractor; + @Nullable + private final MetaCacheEntryRemovalListener removalListener; private MetaCacheEntryDef(String name, Class keyType, Class valueType, @Nullable Function loader, CacheSpec defaultCacheSpec, boolean autoRefresh, boolean contextualOnly, MetaCacheEntryInvalidation invalidation, @Nullable MetaCacheSizeEstimator sizeEstimator, @Nullable MetaCacheEntryReplacementListener replacementListener) { this(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, contextualOnly, invalidation, - sizeEstimator, replacementListener, null); + sizeEstimator, replacementListener, null, null); } private MetaCacheEntryDef(String name, Class keyType, Class valueType, @Nullable Function loader, CacheSpec defaultCacheSpec, boolean autoRefresh, boolean contextualOnly, MetaCacheEntryInvalidation invalidation, @Nullable MetaCacheSizeEstimator sizeEstimator, @Nullable MetaCacheEntryReplacementListener replacementListener, - @Nullable MetaCacheEntryRemovalListener removalListener) { + @Nullable Function removalTokenExtractor, + @Nullable MetaCacheEntryRemovalListener removalListener) { this.name = Objects.requireNonNull(name, "entry name is required"); this.keyType = Objects.requireNonNull(keyType, "entry key type is required"); this.valueType = Objects.requireNonNull(valueType, "entry value type is required"); @@ -141,6 +144,7 @@ private MetaCacheEntryDef(String name, Class keyType, Class valueType, this.invalidation = Objects.requireNonNull(invalidation, "entry invalidation is required"); this.sizeEstimator = sizeEstimator; this.replacementListener = replacementListener; + this.removalTokenExtractor = removalTokenExtractor; this.removalListener = removalListener; } @@ -205,7 +209,8 @@ public static MetaCacheEntryDef contextualOnly( public MetaCacheEntryDef withSizeEstimator(MetaCacheSizeEstimator estimator) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, contextualOnly, invalidation, - Objects.requireNonNull(estimator, "estimator"), replacementListener, removalListener); + Objects.requireNonNull(estimator, "estimator"), replacementListener, + removalTokenExtractor, removalListener); } /** Return a definition that synchronously retires dependencies after a value replacement. */ @@ -213,13 +218,18 @@ public MetaCacheEntryDef withReplacementListener( MetaCacheEntryReplacementListener listener) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, contextualOnly, invalidation, sizeEstimator, - Objects.requireNonNull(listener, "listener"), removalListener); + Objects.requireNonNull(listener, "listener"), removalTokenExtractor, removalListener); } - /** Return a definition that retires dependencies after an admitted value is removed. */ - public MetaCacheEntryDef withRemovalListener(MetaCacheEntryRemovalListener listener) { + /** + * Return a definition that retires dependencies after an admitted value is removed. Only the + * token extracted from the removed value is retained until the asynchronous callback runs. + */ + public MetaCacheEntryDef withRemovalListener( + Function tokenExtractor, MetaCacheEntryRemovalListener listener) { return new MetaCacheEntryDef<>(name, keyType, valueType, loader, defaultCacheSpec, autoRefresh, contextualOnly, invalidation, sizeEstimator, replacementListener, + Objects.requireNonNull(tokenExtractor, "tokenExtractor"), Objects.requireNonNull(listener, "listener")); } @@ -285,7 +295,12 @@ public MetaCacheEntryReplacementListener getReplacementListener() { } @Nullable - public MetaCacheEntryRemovalListener getRemovalListener() { + public Function getRemovalTokenExtractor() { + return removalTokenExtractor; + } + + @Nullable + public MetaCacheEntryRemovalListener getRemovalListener() { return removalListener; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryRemovalListener.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryRemovalListener.java index 8fe179b8b90f57..0c8c173d415b11 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryRemovalListener.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntryRemovalListener.java @@ -20,14 +20,16 @@ import javax.annotation.Nullable; /** - * Receives a value that left the entry through the normal removal path (capacity or weight - * eviction, expiry, soft-value collection, peer reclaim, explicit invalidation). Replacements are - * reported through {@link MetaCacheEntryReplacementListener} instead. The callback runs - * asynchronously after the removal, so it must be fenced by the removed value itself (its - * generation) rather than by whatever the entry currently publishes; {@code removedValue} is - * null when the value was already collected. + * Receives the token of a value that left the entry through the normal removal path (capacity or + * weight eviction, expiry, soft-value collection, peer reclaim, explicit invalidation). + * Replacements are reported through {@link MetaCacheEntryReplacementListener} instead. The + * callback runs asynchronously after the removal; only the small token extracted from the value + * at removal time is queued, never the value itself, so retired graphs are not kept alive outside + * the memory budget. The listener must therefore fence on that token (for example a generation) + * rather than on whatever the entry currently publishes; the token is null when the value was + * already collected. */ @FunctionalInterface -public interface MetaCacheEntryRemovalListener { - void onRemoval(K key, @Nullable V removedValue); +public interface MetaCacheEntryRemovalListener { + void onRemoval(K key, @Nullable T removedToken); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index e953608bce3004..de3cea5af3f2d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -34,7 +34,10 @@ import org.apache.paimon.table.Table; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicLong; import javax.annotation.Nullable; /** @@ -68,6 +71,10 @@ public class PaimonExternalMetaCache extends AbstractExternalMetaCache { private final EntryHandle schemaEntry; private final PaimonTableLoader tableLoader; private final PaimonLatestSnapshotProjectionLoader latestSnapshotProjectionLoader; + // Most recently observed latest fence per (table, generation); see getSnapshotCache. + private final AtomicLong fenceObservations = new AtomicLong(); + private final ConcurrentHashMap latestObservedFences = + new ConcurrentHashMap<>(); public PaimonExternalMetaCache(ExecutorService refreshExecutor) { this(refreshExecutor, new ExternalMetaCacheBudgetManager(java.util.OptionalLong.empty())); @@ -83,7 +90,7 @@ public PaimonExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCach MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping)) .withSizeEstimator((key, value) -> value.prepareForCachePublication(key)) .withReplacementListener(this::retireTableGeneration) - .withRemovalListener(this::retireRemovedTableGeneration)); + .withRemovalListener(PaimonTableCacheValue::getGeneration, this::retireRemovedTableGeneration)); snapshotEntry = registerEntry(MetaCacheEntryDef.contextualOnly(ENTRY_SNAPSHOT, PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(PaimonSnapshotEntryKey::getNameMapping)) @@ -104,22 +111,99 @@ public Table getPaimonTable(NameMapping nameMapping) { public PaimonSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - PaimonTableCacheValue tableValue = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + MetaCacheEntry tables = tableEntry.get(nameMapping.getCtlId()); + PaimonTableCacheValue tableValue = tables.get(nameMapping); PaimonSnapshot fence = loadLatestSnapshotFence(nameMapping, tableValue.getPaimonTable()).getSnapshot(); + if (!tables.isEffectivelyEnabled()) { + // Projections are keyed by the synthetic generation of a published table handle. An + // ineffective table entry publishes nothing, so nothing keyed by this load could ever + // be looked up again: serve it directly instead of churning the snapshot entry. + return executeAuthenticated(nameMapping, + () -> latestSnapshotProjectionLoader.loadAtFence( + nameMapping, fence, tableValue.getGeneration())); + } + // Order fence observations, not snapshot ids: a rollback moves the latest snapshot + // backwards, and a concurrent call may finish after a later observation (reversed + // completion). Either way the most recently observed fence is the one future lookups read. + long observation = fenceObservations.incrementAndGet(); PaimonSnapshotEntryKey key = PaimonSnapshotEntryKey.of( nameMapping, fence, tableValue.getGeneration()); MetaCacheEntry entry = snapshotEntry.get(nameMapping.getCtlId()); + AtomicBoolean loaded = new AtomicBoolean(); PaimonSnapshotCacheValue snapshotValue = entry.get(key, - ignored -> executeAuthenticated(nameMapping, - () -> latestSnapshotProjectionLoader.loadAtFence( - nameMapping, fence, tableValue.getGeneration()))); + ignored -> executeAuthenticated(nameMapping, () -> { + loaded.set(true); + return latestSnapshotProjectionLoader.loadAtFence( + nameMapping, fence, tableValue.getGeneration()); + })); + LatestFenceOwner owner = new LatestFenceOwner(nameMapping, tableValue.getGeneration()); + ObservedFence latest = latestObservedFences.compute(owner, (ignored, current) -> + current == null || current.observation < observation ? new ObservedFence(observation, key) : current); + if (loaded.get()) { + retireSupersededLatestProjections(entry, owner, latest.key); + } if (!isCurrentTableGeneration(nameMapping, tableValue.getGeneration())) { entry.invalidateKeyIfSame(key, snapshotValue); } return snapshotValue; } + /** + * Only the projection of the most recently observed latest fence of a table generation is + * reachable: every later call re-reads the fence and looks up that key. After a load, retire + * every other projection of the generation, including this load itself when a concurrent call + * observed a later fence and finished first, so a busy table never accumulates projections. + */ + private static void retireSupersededLatestProjections( + MetaCacheEntry entry, + LatestFenceOwner owner, PaimonSnapshotEntryKey latestKey) { + entry.invalidateIf(key -> owner.owns(key) && !key.equals(latestKey)); + } + + private void forgetObservedFences(NameMapping nameMapping, java.util.function.LongPredicate retiredGeneration) { + latestObservedFences.keySet().removeIf(owner -> owner.nameMapping.equals(nameMapping) + && retiredGeneration.test(owner.generation)); + } + + private static final class LatestFenceOwner { + private final NameMapping nameMapping; + private final long generation; + + private LatestFenceOwner(NameMapping nameMapping, long generation) { + this.nameMapping = nameMapping; + this.generation = generation; + } + + private boolean owns(PaimonSnapshotEntryKey key) { + return key.getTableGeneration() == generation && key.getNameMapping().equals(nameMapping); + } + + @Override + public boolean equals(Object object) { + if (!(object instanceof LatestFenceOwner)) { + return false; + } + LatestFenceOwner that = (LatestFenceOwner) object; + return generation == that.generation && nameMapping.equals(that.nameMapping); + } + + @Override + public int hashCode() { + return java.util.Objects.hash(nameMapping, generation); + } + } + + private static final class ObservedFence { + private final long observation; + private final PaimonSnapshotEntryKey key; + + private ObservedFence(long observation, PaimonSnapshotEntryKey key) { + this.observation = observation; + this.key = key; + } + } + public PaimonSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTable, Table effectiveTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); return executeAuthenticated(nameMapping, @@ -156,7 +240,9 @@ public PaimonSchemaCacheValue getPaimonSchemaCacheValue(NameMapping nameMapping, PaimonSchemaCacheValue getPaimonSchemaCacheValue( NameMapping nameMapping, long schemaId, long tableGeneration, Table retainedTable) { PaimonSchemaCacheKey key = new PaimonSchemaCacheKey(nameMapping, tableGeneration, schemaId); - if (tableGeneration <= 0L) { + if (tableGeneration <= 0L || !tableEntry.get(nameMapping.getCtlId()).isEffectivelyEnabled()) { + // See getSnapshotCache: without a published table handle no generation-keyed + // projection is reachable again. return (PaimonSchemaCacheValue) executeAuthenticated(nameMapping, () -> loadSchemaCacheValue(key, retainedTable)); } @@ -213,6 +299,7 @@ private T executeAuthenticated(NameMapping nameMapping, java.util.concurrent private void retireTableGeneration(NameMapping nameMapping, @Nullable PaimonTableCacheValue previousValue, PaimonTableCacheValue currentValue) { + forgetObservedFences(nameMapping, generation -> generation != currentValue.getGeneration()); MetaCacheEntry snapshots = snapshotEntry.getIfInitialized(nameMapping.getCtlId()); if (snapshots != null) { @@ -233,22 +320,21 @@ private void retireTableGeneration(NameMapping nameMapping, * looked up again, so the projections keyed by it are garbage. The callback is delayed and * fenced by the removed generation: it never touches the generation currently published. */ - private void retireRemovedTableGeneration(NameMapping nameMapping, - @Nullable PaimonTableCacheValue removedValue) { + private void retireRemovedTableGeneration(NameMapping nameMapping, @Nullable Long removedGeneration) { MetaCacheEntry tables = tableEntry.getIfInitialized(nameMapping.getCtlId()); PaimonTableCacheValue currentValue = tables == null ? null : tables.peekIfPresent(nameMapping); long currentGeneration = currentValue == null ? -1L : currentValue.getGeneration(); - long removedGeneration = removedValue == null ? -1L : removedValue.getGeneration(); - if (removedValue != null && removedGeneration == currentGeneration) { + if (removedGeneration != null && removedGeneration == currentGeneration) { // The removed handle was republished; its projections are addressable again. return; } - // A collected (null) value has no generation left: everything that is not derived from - // the currently published handle is unreachable. - java.util.function.LongPredicate retired = removedValue == null + // A collected value left no generation behind: everything that is not derived from the + // currently published handle is unreachable. + java.util.function.LongPredicate retired = removedGeneration == null ? generation -> generation != currentGeneration : generation -> generation == removedGeneration; + forgetObservedFences(nameMapping, retired); MetaCacheEntry snapshots = snapshotEntry.getIfInitialized(nameMapping.getCtlId()); if (snapshots != null) { @@ -263,6 +349,18 @@ private void retireRemovedTableGeneration(NameMapping nameMapping, } } + @Override + public void invalidateCatalog(long catalogId) { + latestObservedFences.keySet().removeIf(owner -> owner.nameMapping.getCtlId() == catalogId); + super.invalidateCatalog(catalogId); + } + + @Override + public void invalidateCatalogEntries(long catalogId) { + latestObservedFences.keySet().removeIf(owner -> owner.nameMapping.getCtlId() == catalogId); + super.invalidateCatalogEntries(catalogId); + } + @Override protected Map catalogPropertyCompatibilityMap() { Map compatibility = new java.util.HashMap<>( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 675120de9dcfc0..aa078607b6103a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -74,6 +74,7 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.Collections; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -353,6 +354,139 @@ MetaCacheSizeEstimate prepareTableForCachePublication( } } + @Test + public void testIneffectiveTableEntryRevalidatesSnapshotResourcesOnEveryLookup() { + // A base entry can be ineffective while the snapshot entry caches: the physical key hits, + // but the projection must be rebuilt as soon as the fresh handle rotates credentials. + assertIneffectiveBaseRebindsRotatedCredentials( + Collections.singletonMap("meta.cache.iceberg.table.max-weight", "0")); + Map explicitSnapshot = new HashMap<>(); + explicitSnapshot.put("meta.cache.iceberg.table.enable", "false"); + explicitSnapshot.put("meta.cache.iceberg.snapshot.enable", "true"); + assertIneffectiveBaseRebindsRotatedCredentials(explicitSnapshot); + } + + private void assertIneffectiveBaseRebindsRotatedCredentials(Map catalogProperties) { + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + IcebergMetadataOps metadataOps = Mockito.mock(IcebergMetadataOps.class); + Mockito.when(catalog.getMetadataOps()).thenReturn(metadataOps); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + return task.call(); + } + }); + TableMetadata metadata = metadataWithLocation("/metadata/ineffective-base.json"); + Table firstHandle = tableWithMetadata(metadata, new PropertiesFileIO("token", "one")); + Table sameCredentials = tableWithMetadata(metadata, new PropertiesFileIO("token", "one")); + Table rotatedHandle = tableWithMetadata(metadata, new PropertiesFileIO("token", "two")); + Mockito.when(metadataOps.loadTable("remote_db", "remote_tbl")) + .thenReturn(firstHandle, sameCredentials, rotatedHandle); + AtomicInteger preparations = new AtomicInteger(); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor) { + @Override + protected CatalogIf getCatalog(long catalogId) { + return catalog; + } + + @Override + MetaCacheSizeEstimate prepareTableForCachePublication( + NameMapping nameMapping, IcebergTableCacheValue value) { + preparations.incrementAndGet(); + return super.prepareTableForCachePublication(nameMapping, value); + } + }; + try { + cache.initCatalog(1L, catalogProperties); + MetaCacheEntry tables = cache.entry( + 1L, IcebergExternalMetaCache.ENTRY_TABLE, NameMapping.class, IcebergTableCacheValue.class); + MetaCacheEntry snapshots = cache.entry( + 1L, IcebergExternalMetaCache.ENTRY_SNAPSHOT, + IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class); + Assert.assertFalse(tables.isEffectivelyEnabled()); + Assert.assertTrue(snapshots.isEffectivelyEnabled()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); + IcebergExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + + IcebergSnapshotCacheValue first = cache.getSnapshotCache(dorisTable); + Assert.assertNull(tables.peekIfPresent(mapping)); + Assert.assertEquals(1L, snapshots.stats().getEstimatedSize()); + // Same credentials on a new handle instance: the physically keyed projection is reused. + Assert.assertSame(first, cache.getSnapshotCache(dorisTable)); + // Rotated credentials: the projection frozen on the first handle is rebuilt. + IcebergSnapshotCacheValue rebound = cache.getSnapshotCache(dorisTable); + Assert.assertNotSame(first, rebound); + Assert.assertSame(rotatedHandle.io(), rebound.getIcebergTable().get().io()); + Assert.assertEquals(1L, snapshots.stats().getEstimatedSize()); + Mockito.verify(metadataOps, Mockito.times(3)).loadTable("remote_db", "remote_tbl"); + // An ineffective weighted entry never admits, so publication sizing is skipped. + Assert.assertEquals(0, preparations.get()); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testIneffectiveWeightedEntriesSkipPublicationSizing() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + IcebergMetadataOps metadataOps = Mockito.mock(IcebergMetadataOps.class); + Mockito.when(catalog.getMetadataOps()).thenReturn(metadataOps); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + return task.call(); + } + }); + Mockito.when(metadataOps.loadTable("remote_db", "remote_tbl")).thenAnswer( + invocation -> tableWithMetadataLocation("/metadata/ineffective-weighted.json")); + AtomicInteger preparations = new AtomicInteger(); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor) { + @Override + protected CatalogIf getCatalog(long catalogId) { + return catalog; + } + + @Override + MetaCacheSizeEstimate prepareTableForCachePublication( + NameMapping nameMapping, IcebergTableCacheValue value) { + preparations.incrementAndGet(); + return super.prepareTableForCachePublication(nameMapping, value); + } + }; + try { + Map properties = new HashMap<>(); + properties.put("meta.cache.iceberg.table.max-weight", "0"); + properties.put("meta.cache.iceberg.snapshot.max-weight", "0"); + properties.put("meta.cache.iceberg.manifest.enable", "true"); + properties.put("meta.cache.iceberg.manifest.max-weight", "0"); + cache.initCatalog(1L, properties); + for (String entryName : new String[] {IcebergExternalMetaCache.ENTRY_TABLE, + IcebergExternalMetaCache.ENTRY_SNAPSHOT, IcebergExternalMetaCache.ENTRY_MANIFEST}) { + MetaCacheEntryStats stats = cache.stats(1L).get(entryName); + Assert.assertTrue(entryName, stats.isWeightBounded()); + Assert.assertFalse(entryName, stats.isEffectiveEnabled()); + } + NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); + IcebergExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + + IcebergSnapshotCacheValue projection = cache.getSnapshotCache(dorisTable); + IcebergTableCacheValue tableValue = cache.entry(1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class).get(mapping); + Assert.assertEquals(0, preparations.get()); + Assert.assertFalse(tableValue.isQueryIsolationPrepared()); + Assert.assertFalse(projection.getSizeEstimate().isComplete()); + Assert.assertEquals("not_prepared", projection.getSizeEstimate().getIncompleteReason()); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + @Test public void testDisabledTableCacheKeepsPhysicallyKeyedSchemaProjections() { ExecutorService executor = Executors.newSingleThreadExecutor(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java index 2d8140277e2d98..00c7db74fba19d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java @@ -1852,7 +1852,7 @@ public void testRemovalListenerReceivesRemovedValuesButNotReplacements() throws "removal-listener", key -> 1, CacheSpec.of(true, CacheSpec.CACHE_NO_TTL, 10L), refreshExecutor, false, false, null, null, null, - (key, value) -> removed.add(key + "=" + value)); + value -> value, (key, token) -> removed.add(key + "=" + token)); try { entry.put("k", 1); entry.put("k", 2); @@ -1878,6 +1878,68 @@ public void testRemovalListenerReceivesRemovedValuesButNotReplacements() throws } } + @Test + public void testQueuedRemovalNotificationsDoNotRetainRemovedValues() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + CountDownLatch listenerEntered = new CountDownLatch(1); + CountDownLatch releaseListener = new CountDownLatch(1); + java.util.List tokens = java.util.Collections.synchronizedList(new java.util.ArrayList<>()); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(1L << 20)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "removal-token", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "removal-token", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1L << 20), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget, null, + value -> (long) value.length, (key, token) -> { + tokens.add(token); + listenerEntered.countDown(); + try { + releaseListener.await(5L, TimeUnit.SECONDS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + } + }); + try { + // Block the cleanup thread inside the first callback, then churn remove/refill cycles. + entry.put("blocker", new byte[8]); + entry.invalidateKey("blocker"); + Assert.assertTrue(listenerEntered.await(3L, TimeUnit.SECONDS)); + + java.util.List> retired = new java.util.ArrayList<>(); + for (int i = 0; i < 8; i++) { + byte[] value = new byte[64 * 1024]; + retired.add(new WeakReference<>(value)); + entry.put("k", value); + entry.invalidateKey("k"); + value = null; + } + Assert.assertEquals("reservations are released with the removal", 0L, manager.getGlobalUsedWeight()); + for (int attempt = 0; attempt < 5 && retired.stream().anyMatch(ref -> ref.get() != null); attempt++) { + System.gc(); + Thread.sleep(50L); + } + // The queued notifications carry only the extracted tokens; the removed values are + // collectable while their reservations are already released. + Assert.assertTrue("queued removal notifications must not retain removed values", + retired.stream().allMatch(ref -> ref.get() == null)); + + releaseListener.countDown(); + long deadline = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); + while (tokens.size() < 9 && System.nanoTime() < deadline) { + Thread.sleep(10L); + } + Assert.assertEquals(9, tokens.size()); + Assert.assertEquals(8L, tokens.get(0)); + Assert.assertEquals((long) (64 * 1024), tokens.get(8)); + } finally { + releaseListener.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + private void awaitGlobalWeight(ExternalMetaCacheBudgetManager manager, long expected) throws InterruptedException { long deadlineNanos = System.nanoTime() + TimeUnit.SECONDS.toNanos(3L); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index f7d9b44174b058..1497efaeef3552 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -777,67 +777,99 @@ public void testExpiredBaseTableRetiresSnapshotAndSchemaProjectionsBeforeReplace } } + /** Mocked catalog/env/table graph for the base-table + projection flows. */ + private static final class MockedPaimonCatalog { + private final Env env = Mockito.mock(Env.class); + private final PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + private final FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); + private final java.util.concurrent.atomic.AtomicLong latestSnapshotId = + new java.util.concurrent.atomic.AtomicLong(7L); + private final NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); + private final AtomicInteger partitionEnumerations = new AtomicInteger(); + private final AtomicInteger schemaLoads = new AtomicInteger(); + // When set, the next partition enumeration signals enumerationEntered and blocks on it. + private volatile java.util.concurrent.CountDownLatch blockNextEnumeration; + private final java.util.concurrent.CountDownLatch enumerationEntered = + new java.util.concurrent.CountDownLatch(1); + + private MockedPaimonCatalog() { + PaimonExternalDatabase database = Mockito.mock(PaimonExternalDatabase.class); + PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.doReturn(catalog).when(catalogMgr) + .getCatalogOrException(Mockito.eq(1L), Mockito.any()); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog(1L); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + return task.call(); + } + }); + Mockito.doReturn(database).when(catalog).getDbNullable("db"); + Mockito.when(database.getTableNullable("tbl")).thenReturn(externalTable); + Mockito.doReturn(Optional.of(database)).when(catalog).getDb("db"); + Mockito.doReturn(Optional.of(externalTable)).when(database).getTable("tbl"); + Column partitionColumn = new Column("part", Type.INT); + Mockito.doAnswer(invocation -> { + schemaLoads.incrementAndGet(); + return new PaimonSchemaCacheValue( + Collections.singletonList(partitionColumn), + Collections.singletonList(partitionColumn), null); + }).when(externalTable).loadSchemaForCache(Mockito.any(), Mockito.anyLong()); + + FileStoreTable latestSchemaTable = Mockito.mock(FileStoreTable.class); + FileStoreTable fenceTable = Mockito.mock(FileStoreTable.class); + FileStoreTable snapshotTable = Mockito.mock(FileStoreTable.class); + Snapshot latestSnapshot = Mockito.mock(Snapshot.class); + SchemaManager schemaManager = Mockito.mock(SchemaManager.class); + TableSchema latestSchema = Mockito.mock(TableSchema.class); + ReadBuilder readBuilder = Mockito.mock(ReadBuilder.class); + TableScan tableScan = Mockito.mock(TableScan.class); + Mockito.when(baseTable.copyWithLatestSchema()).thenReturn(latestSchemaTable); + Mockito.when(latestSchemaTable.latestSnapshot()).thenReturn(Optional.of(latestSnapshot)); + Mockito.when(latestSnapshot.id()).thenAnswer(invocation -> latestSnapshotId.get()); + Mockito.when(latestSchemaTable.schemaManager()).thenReturn(schemaManager); + Mockito.when(schemaManager.latest()).thenReturn(Optional.of(latestSchema)); + Mockito.when(latestSchema.id()).thenReturn(3L); + Mockito.when(latestSchemaTable.copyWithoutTimeTravel(Mockito.anyMap())).thenReturn(fenceTable); + Mockito.when(fenceTable.copyWithoutTimeTravel(Mockito.anyMap())).thenReturn(snapshotTable); + Mockito.when(snapshotTable.options()).thenReturn(Collections.emptyMap()); + Mockito.when(snapshotTable.newReadBuilder()).thenReturn(readBuilder); + Mockito.when(readBuilder.newScan()).thenReturn(tableScan); + Mockito.when(tableScan.listPartitionEntries()).thenAnswer(invocation -> { + partitionEnumerations.incrementAndGet(); + java.util.concurrent.CountDownLatch block = blockNextEnumeration; + if (block != null) { + blockNextEnumeration = null; + enumerationEntered.countDown(); + Assert.assertTrue(block.await(5L, java.util.concurrent.TimeUnit.SECONDS)); + } + return Collections.emptyList(); + }); + Mockito.when(catalog.getPaimonTable(mapping)).thenReturn(baseTable); + } + + private ExternalTable dorisTable() { + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + return dorisTable; + } + } + @Test public void testRejectedBaseTableDoesNotAccumulateSnapshotOrSchemaProjections() { - PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); - PaimonExternalDatabase database = Mockito.mock(PaimonExternalDatabase.class); - PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class); - CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); - Env env = Mockito.mock(Env.class); - Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); - Mockito.doReturn(catalog).when(catalogMgr) - .getCatalogOrException(Mockito.eq(1L), Mockito.any()); - Mockito.doReturn(catalog).when(catalogMgr).getCatalog(1L); - Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(new ExecutionAuthenticator() { - @Override - public T execute(Callable task) throws Exception { - return task.call(); - } - }); - Mockito.doReturn(database).when(catalog).getDbNullable("db"); - Mockito.when(database.getTableNullable("tbl")).thenReturn(externalTable); - Mockito.doReturn(Optional.of(database)).when(catalog).getDb("db"); - Mockito.doReturn(Optional.of(externalTable)).when(database).getTable("tbl"); - Column partitionColumn = new Column("part", Type.INT); - Mockito.doReturn(new PaimonSchemaCacheValue( - Collections.singletonList(partitionColumn), - Collections.singletonList(partitionColumn), null)) - .when(externalTable).loadSchemaForCache(Mockito.any(), Mockito.anyLong()); - // A mocked table has no supported layout, so its weight estimate is incomplete and every // load is rejected by the weight-bounded table entry. - FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); - FileStoreTable latestSchemaTable = Mockito.mock(FileStoreTable.class); - FileStoreTable fenceTable = Mockito.mock(FileStoreTable.class); - FileStoreTable snapshotTable = Mockito.mock(FileStoreTable.class); - Snapshot latestSnapshot = Mockito.mock(Snapshot.class); - SchemaManager schemaManager = Mockito.mock(SchemaManager.class); - TableSchema latestSchema = Mockito.mock(TableSchema.class); - ReadBuilder readBuilder = Mockito.mock(ReadBuilder.class); - TableScan tableScan = Mockito.mock(TableScan.class); - Mockito.when(baseTable.copyWithLatestSchema()).thenReturn(latestSchemaTable); - Mockito.when(latestSchemaTable.latestSnapshot()).thenReturn(Optional.of(latestSnapshot)); - Mockito.when(latestSnapshot.id()).thenReturn(7L); - Mockito.when(latestSchemaTable.schemaManager()).thenReturn(schemaManager); - Mockito.when(schemaManager.latest()).thenReturn(Optional.of(latestSchema)); - Mockito.when(latestSchema.id()).thenReturn(3L); - Mockito.when(latestSchemaTable.copyWithoutTimeTravel(Mockito.anyMap())).thenReturn(fenceTable); - Mockito.when(fenceTable.copyWithoutTimeTravel(Mockito.anyMap())).thenReturn(snapshotTable); - Mockito.when(snapshotTable.options()).thenReturn(Collections.emptyMap()); - Mockito.when(snapshotTable.newReadBuilder()).thenReturn(readBuilder); - Mockito.when(readBuilder.newScan()).thenReturn(tableScan); - Mockito.when(tableScan.listPartitionEntries()).thenReturn(Collections.emptyList()); - NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); - Mockito.when(catalog.getPaimonTable(mapping)).thenReturn(baseTable); - + MockedPaimonCatalog mocked = new MockedPaimonCatalog(); + NameMapping mapping = mocked.mapping; ExecutorService executor = Executors.newSingleThreadExecutor(); PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { - mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + mockedEnv.when(Env::getCurrentEnv).thenReturn(mocked.env); cache.initCatalog(1L, Collections.singletonMap( "meta.cache.paimon.table.max-weight", "1MB")); - ExternalTable dorisTable = Mockito.mock(ExternalTable.class); - Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + ExternalTable dorisTable = mocked.dorisTable(); org.apache.doris.datasource.metacache.MetaCacheEntry tables = cache.entry(1L, PaimonExternalMetaCache.ENTRY_TABLE, NameMapping.class, PaimonTableCacheValue.class); @@ -858,7 +890,141 @@ public T execute(Callable task) throws Exception { Assert.assertEquals(0L, schemas.stats().getEstimatedSize()); } Assert.assertEquals(10L, tables.stats().getWeightAdmissionRejectedCount()); - Mockito.verify(catalog, Mockito.times(10)).getPaimonTable(mapping); + Mockito.verify(mocked.catalog, Mockito.times(10)).getPaimonTable(mapping); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testIneffectiveBaseTableServesProjectionsWithoutChildCaching() { + // table.max-weight=0 (and table.enable=false with snapshot re-enabled) leave the base entry + // ineffective while the children report enabled: nothing keyed by an unpublished + // generation is reachable, so the children are bypassed instead of loaded and discarded. + assertIneffectiveBaseBypassesChildren(Collections.singletonMap("meta.cache.paimon.table.max-weight", "0")); + Map explicitSnapshot = new HashMap<>(); + explicitSnapshot.put("meta.cache.paimon.table.enable", "false"); + explicitSnapshot.put("meta.cache.paimon.snapshot.enable", "true"); + assertIneffectiveBaseBypassesChildren(explicitSnapshot); + } + + private void assertIneffectiveBaseBypassesChildren(Map catalogProperties) { + MockedPaimonCatalog mocked = new MockedPaimonCatalog(); + NameMapping mapping = mocked.mapping; + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(mocked.env); + cache.initCatalog(1L, catalogProperties); + ExternalTable dorisTable = mocked.dorisTable(); + org.apache.doris.datasource.metacache.MetaCacheEntry tables = + cache.entry(1L, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class); + org.apache.doris.datasource.metacache.MetaCacheEntry< + PaimonSnapshotEntryKey, PaimonSnapshotCacheValue> snapshots = cache.entry( + 1L, PaimonExternalMetaCache.ENTRY_SNAPSHOT, + PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class); + org.apache.doris.datasource.metacache.MetaCacheEntry schemas = + cache.entry(1L, PaimonExternalMetaCache.ENTRY_SCHEMA, + PaimonSchemaCacheKey.class, SchemaCacheValue.class); + Assert.assertFalse(tables.isEffectivelyEnabled()); + Assert.assertTrue(snapshots.isEffectivelyEnabled()); + Assert.assertTrue(schemas.isEffectivelyEnabled()); + + for (int i = 0; i < 3; i++) { + Assert.assertEquals(7L, cache.getSnapshotCache(dorisTable).getSnapshot().getSnapshotId()); + Assert.assertNotNull(cache.getPaimonSchemaCacheValue(mapping, 3L)); + Assert.assertNull(tables.peekIfPresent(mapping)); + Assert.assertEquals(0L, snapshots.stats().getEstimatedSize()); + Assert.assertEquals(0L, schemas.stats().getEstimatedSize()); + } + Assert.assertEquals("no projection was ever admitted", 0L, snapshots.stats().getInvalidateCount()); + Assert.assertEquals(0L, schemas.stats().getInvalidateCount()); + Assert.assertTrue("no admission is attempted", tables.stats().getWeightAdmissionRejectedCount() <= 0L); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + @Test + public void testAdvancingLatestFenceKeepsOnlyNewestProjectionOfTableGeneration() throws Exception { + MockedPaimonCatalog mocked = new MockedPaimonCatalog(); + NameMapping mapping = mocked.mapping; + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(mocked.env); + cache.initCatalog(1L, Collections.emptyMap()); + ExternalTable dorisTable = mocked.dorisTable(); + org.apache.doris.datasource.metacache.MetaCacheEntry tables = + cache.entry(1L, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class); + org.apache.doris.datasource.metacache.MetaCacheEntry< + PaimonSnapshotEntryKey, PaimonSnapshotCacheValue> snapshots = cache.entry( + 1L, PaimonExternalMetaCache.ENTRY_SNAPSHOT, + PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class); + + PaimonSnapshotCacheValue at7 = cache.getSnapshotCache(dorisTable); + PaimonTableCacheValue tableValue = tables.peekIfPresent(mapping); + Assert.assertNotNull(tableValue); + PaimonSnapshotEntryKey key7 = new PaimonSnapshotEntryKey(mapping, 7L, 3L, tableValue.getGeneration()); + Assert.assertSame(at7, snapshots.peekIfPresent(key7)); + Assert.assertSame(at7, cache.getSnapshotCache(dorisTable)); + + // Commits observed before the table handle refreshes advance the fence: only the + // newest projection of this generation stays reachable. + mocked.latestSnapshotId.set(8L); + PaimonSnapshotCacheValue at8 = cache.getSnapshotCache(dorisTable); + PaimonSnapshotEntryKey key8 = new PaimonSnapshotEntryKey(mapping, 8L, 3L, tableValue.getGeneration()); + Assert.assertEquals(8L, at8.getSnapshot().getSnapshotId()); + Assert.assertNull(snapshots.peekIfPresent(key7)); + Assert.assertSame(at8, snapshots.peekIfPresent(key8)); + Assert.assertEquals(1L, snapshots.stats().getEstimatedSize()); + Assert.assertSame("the table handle itself is not replaced", tableValue, tables.peekIfPresent(mapping)); + + // Reversed completion: a call that observed fence 8 is still enumerating partitions + // while a later call observes fence 9 and finishes first. The most recently observed + // fence wins; the older load must not survive next to it. + snapshots.invalidateKey(key8); + java.util.concurrent.CountDownLatch releaseOlderLoad = new java.util.concurrent.CountDownLatch(1); + mocked.blockNextEnumeration = releaseOlderLoad; + ExecutorService olderCall = Executors.newSingleThreadExecutor(); + java.util.concurrent.Future older; + PaimonSnapshotEntryKey key9 = new PaimonSnapshotEntryKey(mapping, 9L, 3L, tableValue.getGeneration()); + try { + older = olderCall.submit(() -> { + try (MockedStatic workerEnv = Mockito.mockStatic(Env.class)) { + workerEnv.when(Env::getCurrentEnv).thenReturn(mocked.env); + return cache.getSnapshotCache(dorisTable); + } + }); + Assert.assertTrue(mocked.enumerationEntered.await(5L, java.util.concurrent.TimeUnit.SECONDS)); + mocked.latestSnapshotId.set(9L); + PaimonSnapshotCacheValue at9 = cache.getSnapshotCache(dorisTable); + Assert.assertSame(at9, snapshots.peekIfPresent(key9)); + releaseOlderLoad.countDown(); + Assert.assertEquals(8L, older.get(5L, java.util.concurrent.TimeUnit.SECONDS) + .getSnapshot().getSnapshotId()); + Assert.assertNull(snapshots.peekIfPresent(key8)); + Assert.assertSame(at9, snapshots.peekIfPresent(key9)); + Assert.assertEquals(1L, snapshots.stats().getEstimatedSize()); + } finally { + releaseOlderLoad.countDown(); + olderCall.shutdownNow(); + } + + // Rollback: the latest snapshot moves backwards; the newly observed fence replaces the + // projection of the higher snapshot id instead of being retired by it. + mocked.latestSnapshotId.set(8L); + PaimonSnapshotCacheValue rolledBack = cache.getSnapshotCache(dorisTable); + Assert.assertEquals(8L, rolledBack.getSnapshot().getSnapshotId()); + Assert.assertSame(rolledBack, snapshots.peekIfPresent(key8)); + Assert.assertNull(snapshots.peekIfPresent(key9)); + Assert.assertSame(rolledBack, cache.getSnapshotCache(dorisTable)); + Assert.assertEquals(1L, snapshots.stats().getEstimatedSize()); + Assert.assertEquals(5, mocked.partitionEnumerations.get()); } finally { cache.close(); executor.shutdownNow(); From 950d87dafe9d970ab8bb1c611e936d8052b3ec0c Mon Sep 17 00:00:00 2001 From: guoqiang Date: Thu, 20 Aug 2026 12:54:31 +0800 Subject: [PATCH 11/45] [fix](fe) Charge shared Paimon partition column names once and drop latest-fence owners of unpublished generations - Partition column names are schema-owned strings shared by every partition's typed spec; charge each distinct reference once so wide partition sets are not overcounted out of the weight budget, and calibrate fixtures with production-style shared name identities - Remove the latest-fence owner registered by a lookup whose table generation was never published (rejected admission, replaced or invalidated mid-load) so persistently rejected tables cannot grow the owner map --- .../paimon/PaimonExternalMetaCache.java | 5 ++++ .../paimon/PaimonPartitionInfo.java | 15 ++++++++---- .../doris/datasource/paimon/PaimonUtil.java | 10 ++++++-- .../paimon/PaimonExternalMetaCacheTest.java | 23 +++++++++++++++++-- 4 files changed, 45 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index de3cea5af3f2d8..f8b5f46c2bb661 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -145,6 +145,11 @@ public PaimonSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { } if (!isCurrentTableGeneration(nameMapping, tableValue.getGeneration())) { entry.invalidateKeyIfSame(key, snapshotValue); + // A generation that is not published (rejected admission, replaced or invalidated + // mid-load) can never be observed again; drop the owner this call registered so + // persistently rejected tables cannot grow the map, and so a delayed old-generation + // load cannot resurrect an owner that catalog cleanup already removed. + latestObservedFences.remove(owner); } return snapshotValue; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java index f4570dccf57f6a..b7f3caca190db8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonPartitionInfo.java @@ -23,7 +23,9 @@ import org.apache.paimon.partition.Partition; import java.util.Collections; +import java.util.IdentityHashMap; import java.util.Map; +import java.util.Set; /** * Snapshot-scoped Paimon partition metadata used by Doris. @@ -112,28 +114,33 @@ private static long retainedPayloadBytes(Map partitions) { return 0L; } long bytes = 0L; + // Spec keys are the schema-owned partition column names, shared by reference across + // partitions; charge each distinct reference once, exactly like the retained graph. + Set seenSpecKeys = Collections.newSetFromMap(new IdentityHashMap<>()); for (Map.Entry entry : partitions.entrySet()) { bytes = addString(bytes, entry.getKey()); Partition partition = entry.getValue(); if (partition == null) { continue; } - bytes = addStrings(bytes, partition.spec()); + bytes = addStrings(bytes, partition.spec(), seenSpecKeys); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, partitionColumnBytes( partition.spec() == null ? 0 : partition.spec().size())); bytes = addString(bytes, partition.createdBy()); bytes = addString(bytes, partition.updatedBy()); - bytes = addStrings(bytes, partition.options()); + bytes = addStrings(bytes, partition.options(), null); } return bytes; } - private static long addStrings(long bytes, Map values) { + private static long addStrings(long bytes, Map values, Set seenKeys) { if (values == null) { return bytes; } for (Map.Entry entry : values.entrySet()) { - bytes = addString(bytes, entry.getKey()); + if (seenKeys == null || seenKeys.add(entry.getKey())) { + bytes = addString(bytes, entry.getKey()); + } bytes = addString(bytes, entry.getValue()); } return bytes; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index de6b24fd8cf368..a793c59da149d4 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -180,6 +180,14 @@ public static PaimonPartitionInfo generatePartitionInfo(Table table, List candidates = Lists.newArrayListWithExpectedSize(partitionEntries.size()); Map> displayNameToTypedSpec = Maps.newHashMap(); long retainedPayloadBytes = 0L; + if (!partitionEntries.isEmpty()) { + for (Column partitionColumn : partitionColumns) { + // Every partition's typed spec keys the same schema-owned name reference; the + // retained graph holds one string per column, not one per partition. + retainedPayloadBytes = PaimonPartitionInfo.addRetainedStringPayload( + retainedPayloadBytes, partitionColumn.getName()); + } + } for (PartitionEntry partitionEntry : partitionEntries) { Map typedSpec = getPartitionInfoMap( @@ -199,8 +207,6 @@ public static PaimonPartitionInfo generatePartitionInfo(Table table, List partitionItems = new HashMap<>(); Map partitions = new HashMap<>(); + // Production typed specs key the schema-owned column names: one shared String reference + // across every partition, so the fixture must share them too. + String[] columnNames = new String[partitionColumnCount]; + for (int column = 0; column < partitionColumnCount; column++) { + columnNames[column] = new String(("part" + column).toCharArray()); + } for (int index = 0; index < partitionCount; index++) { String value = partitionType == Type.INT ? Integer.toString(index) @@ -1658,10 +1668,10 @@ private PaimonSnapshotCacheValue snapshotValueWithRealPartitions( Map spec = new java.util.LinkedHashMap<>(); for (int column = 0; column < partitionColumnCount; column++) { // Each loaded column owns its own value String. - String columnValue = new String(value); + String columnValue = new String(value.toCharArray()); values.add(columnValue); types.add(partitionType); - spec.put("part" + column, columnValue); + spec.put(columnNames[column], columnValue); } partitionItems.put(name, PaimonUtil.toListPartitionItem(values, types)); partitions.put(name, new org.apache.paimon.partition.Partition( @@ -1679,6 +1689,15 @@ private Map sizeOnlyMap(int size) { return map; } + private int observedFenceOwnerCount(PaimonExternalMetaCache cache) { + try { + return ((java.util.Map) readField( + cache, PaimonExternalMetaCache.class, "latestObservedFences")).size(); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + private Object readField(RowType rowType, String fieldName) throws Exception { return readField(rowType, RowType.class, fieldName); } From f6812356af7ea4be7aab54ed743ea548953aaf27 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Thu, 20 Aug 2026 19:42:53 +0800 Subject: [PATCH 12/45] [fix](fe) Charge the Hive partition-value key width and retire catalog budget buckets in O(1) - HiveCacheSizeEstimator charges the key's retained type list (singleton list or backing array per column) independently of the partition count, so an empty partitioned table's admission scales with its column width; calibrated at ratio 1.000 - ExternalMetaCacheBudgetManager tracks live entry budgets per catalog bucket so closing an entry no longer scans every scope under the global accounting lock --- .../hive/HiveCacheSizeEstimator.java | 22 ++++++++++ .../hive/HiveExternalMetaCache.java | 4 ++ .../ExternalMetaCacheBudgetManager.java | 9 ++-- .../hive/HiveMetaStoreCacheTest.java | 30 +++++++++++++ .../ExternalMetaCacheBudgetManagerTest.java | 43 +++++++++++++++++++ 5 files changed, 105 insertions(+), 3 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java index 0d5f8a8343415f..fa74f86c429319 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java @@ -28,6 +28,12 @@ final class HiveCacheSizeEstimator { // name plus derived value/literal strings and therefore remains skew-sensitive. private static final long ENTRY_BASE_BYTES = objectBytes(2L * 1024L); private static final long PARTITION_BASE_BYTES = objectBytes(896L); + // PartitionValueCacheKey retains an ImmutableList over the partition column types; the Type + // instances themselves are shared catalog singletons and are not charged. A single-element + // list is Guava's SingletonImmutableList (one reference, no array); wider lists add a + // backing array with one slot per column. + private static final long KEY_TYPE_LIST_BYTES = + MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); private static final long PARTITION_COLUMN_BYTES = objectBytes(256L); // One copy is retained as the partition name and another in the decoded partition values. private static final long PARTITION_NAME_PAYLOAD_COPIES = 2L; @@ -52,6 +58,9 @@ static MetaCacheSizeEstimate estimatePartitionValuesEntry( value.getPartitionColumnCount(), PARTITION_COLUMN_BYTES)); long bytes = MetaCacheWeightUtils.saturatedAdd( ENTRY_BASE_BYTES, MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping())); + // The retained key width does not depend on how many partitions the table has today; + // an empty partitioned table still retains one type list slot per partition column. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, keyTypeListBytes(key.retainedTypeCount())); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.saturatedMultiply(partitionCount, perPartitionBytes)); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, @@ -59,4 +68,17 @@ static MetaCacheSizeEstimate estimatePartitionValuesEntry( value.getPartitionNamePayloadBytes(), PARTITION_NAME_PAYLOAD_COPIES)); return MetaCacheSizeEstimate.complete(bytes); } + + private static long keyTypeListBytes(int typeCount) { + if (typeCount <= 0) { + // ImmutableList.of() is a shared singleton. + return 0L; + } + long bytes = KEY_TYPE_LIST_BYTES; + if (typeCount > 1) { + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedObjectArrayBytes(typeCount)); + } + return bytes; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java index caf507e6017101..07df89cacab1b5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java @@ -974,6 +974,10 @@ public PartitionValueCacheKey(NameMapping nameMapping, List types) { this.types = types == null ? null : ImmutableList.copyOf(types); } + int retainedTypeCount() { + return types == null ? 0 : types.size(); + } + @Override public boolean equals(Object obj) { if (this == obj) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java index 904a3fb0ad8f75..bbdd108f0a585a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java @@ -136,6 +136,7 @@ public EntryBudget createEntryBudget(long catalogId, String engine, String entry this, scope, catalogBucket, entryBucket, effectiveMax.getAsLong()); entryBuckets.put(scope, entryBucket); entryBudgets.put(scope, entryBudget); + catalogBucket.liveEntries++; return entryBudget; } } @@ -270,9 +271,8 @@ private void close(EntryBudget entryBudget) { entryBuckets.remove(entryBudget.scope, entryBudget.entryBucket); entryBudgets.remove(entryBudget.scope, entryBudget); Bucket catalogBucket = entryBudget.catalogBucket; - boolean catalogStillReferenced = entryBuckets.keySet().stream() - .anyMatch(scope -> scope.catalogId == entryBudget.scope.catalogId); - if (!catalogStillReferenced && catalogBucket.usedWeight == 0L) { + catalogBucket.liveEntries--; + if (catalogBucket.liveEntries == 0 && catalogBucket.usedWeight == 0L) { catalogBuckets.remove(entryBudget.scope.catalogId, catalogBucket); } } @@ -441,6 +441,9 @@ private static void checkWeight(long bytes) { private static final class Bucket { private final long maxWeight; private long usedWeight; + // Live entry budgets charging into this catalog bucket; maintained under the manager + // lock so catalog retirement can decide teardown in O(1) instead of scanning all scopes. + private int liveEntries; private Bucket(long maxWeight) { this.maxWeight = maxWeight; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java index 1a6736feccda7f..5753820523a928 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java @@ -311,6 +311,36 @@ public void testPartitionValuesFormulaAgainstJolOwnedGraph() throws Exception { "hive long-tail partition", shortTailEstimate, longTailEstimate, shortTail, longTail); } + @Test + public void testEmptyTableKeyWidthFormulaAgainstJolOwnedGraph() throws Exception { + // An empty partitioned table still retains the key's immutable type list; the estimate + // must scale with the partition column width even when partitionCount is zero. Type + // instances are shared catalog singletons, present on both sides of the delta. + NameMapping mapping = NameMapping.createForTest("db", "tbl"); + List narrowTypes = Collections.singletonList(Type.STRING); + List wideTypes = new java.util.ArrayList<>(); + for (int i = 0; i < 16; i++) { + wideTypes.add(Type.STRING); + } + HiveExternalMetaCache.PartitionValueCacheKey narrowKey = + new HiveExternalMetaCache.PartitionValueCacheKey(mapping, narrowTypes); + HiveExternalMetaCache.PartitionValueCacheKey wideKey = + new HiveExternalMetaCache.PartitionValueCacheKey(mapping, wideTypes); + HiveExternalMetaCache.HivePartitionValues narrowValues = new HiveExternalMetaCache.HivePartitionValues( + new HashMap<>(), HashBiMap.create(), new HashMap<>(), 0L, narrowTypes.size()); + HiveExternalMetaCache.HivePartitionValues wideValues = new HiveExternalMetaCache.HivePartitionValues( + new HashMap<>(), HashBiMap.create(), new HashMap<>(), 0L, wideTypes.size()); + + long narrowEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry( + narrowKey, narrowValues).getBytes(); + long wideEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry( + wideKey, wideValues).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "hive empty table key width", narrowEstimate, wideEstimate, + java.util.Arrays.asList(narrowKey, narrowValues), + java.util.Arrays.asList(wideKey, wideValues)); + } + private void putCache( MetaCacheEntry fileCache, MetaCacheEntry partitionCache, diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java index d4b76b97c0131e..a5841e523298e3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManagerTest.java @@ -288,4 +288,47 @@ private static void await(CountDownLatch latch) { throw new RuntimeException(e); } } + + @Test + public void testCatalogBucketLifecycleTracksLiveEntriesWithoutScanning() throws Exception { + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(1L << 20)); + java.util.List budgets = new java.util.ArrayList<>(); + for (long catalogId = 1L; catalogId <= 3L; catalogId++) { + for (String entry : new String[] {"table", "snapshot"}) { + budgets.add(manager.createEntryBudget( + catalogId, "paimon", entry, OptionalLong.empty(), OptionalLong.empty())); + } + } + java.util.Map catalogBuckets = readCatalogBuckets(manager); + Assert.assertEquals(3, catalogBuckets.size()); + + // Closing one of a catalog's entries keeps its bucket; closing the last removes it, and + // unrelated catalogs keep reserving while the retirement is in progress. + budgets.get(0).close(); + Assert.assertEquals(3, readCatalogBuckets(manager).size()); + AdmissionReservation unrelated = budgets.get(2).tryReserve(64L).get(); + budgets.get(1).close(); + Assert.assertEquals(2, readCatalogBuckets(manager).size()); + unrelated.release(); + + // A re-created catalog starts from a fresh bucket and can reserve again. + ExternalMetaCacheBudgetManager.EntryBudget recreated = manager.createEntryBudget( + 1L, "paimon", "table", OptionalLong.empty(), OptionalLong.empty()); + Assert.assertEquals(3, readCatalogBuckets(manager).size()); + AdmissionReservation recreatedReservation = recreated.tryReserve(128L).get(); + recreatedReservation.release(); + recreated.close(); + for (int i = 2; i < budgets.size(); i++) { + budgets.get(i).close(); + } + Assert.assertEquals(0, readCatalogBuckets(manager).size()); + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } + + private static java.util.Map readCatalogBuckets(ExternalMetaCacheBudgetManager manager) + throws Exception { + java.lang.reflect.Field field = ExternalMetaCacheBudgetManager.class.getDeclaredField("catalogBuckets"); + field.setAccessible(true); + return (java.util.Map) field.get(manager); + } } From 66a94dd487b928bff0172aa98c60c943adf119ad Mon Sep 17 00:00:00 2001 From: guoqiang Date: Thu, 20 Aug 2026 21:58:31 +0800 Subject: [PATCH 13/45] [refactor](fe) Replace structural cache size estimators with coarse cardinality formulas Converge the estimator design as requested in review: max-weight is an estimated retained-cache admission budget, not an exact heap promise. - Iceberg/Paimon/Hive estimators now charge a rounded-up constant per stable logical dimension (snapshot, schema field, partition field, type node, metadata entry, ...) plus the loader-materialized string payload; per-node constants absorb post-admission lazy growth (schema indexes, fieldsBySourceId, RowType lookup maps, store-derived copies) - Remove SDK-private layout fingerprints and exact JVM layout modeling; unknown SDK subtypes (content files, partition containers, logical types) are charged generic conservative weights instead of disabling weighted caching; the only remaining SDK-private access is the TableMetadata snapshotsLoaded probe that prevents IO - Drop the GPL-licensed jol-core test dependency; calibration now uses a dependency-free reflective probe with an order-of-magnitude acceptance band instead of a 10% JOL bound - Keep the admission/lifecycle framework unchanged: no IO or lazy materialization during weighing, bounded publication work, saturated arithmetic, O(1) hits and removals --- fe/fe-core/pom.xml | 7 +- .../hive/HiveCacheSizeEstimator.java | 12 +- .../iceberg/IcebergCacheSizeEstimator.java | 1053 +++-------------- .../iceberg/IcebergPartitionInfo.java | 2 +- .../iceberg/cache/ManifestCacheValue.java | 64 +- .../doris/datasource/metacache/CacheSpec.java | 4 +- .../metacache/MetaCacheWeightUtils.java | 343 +----- .../paimon/PaimonCacheSizeEstimator.java | 604 +++------- .../iceberg/IcebergExternalMetaCacheTest.java | 59 +- .../EstimatorCalibrationAssertions.java | 193 +-- .../metacache/MetaCacheEntryTest.java | 8 +- .../paimon/PaimonExternalMetaCacheTest.java | 12 +- fe/pom.xml | 6 - 13 files changed, 573 insertions(+), 1794 deletions(-) diff --git a/fe/fe-core/pom.xml b/fe/fe-core/pom.xml index 494f51ceb25148..2a0cc1d8224e1c 100644 --- a/fe/fe-core/pom.xml +++ b/fe/fe-core/pom.xml @@ -849,11 +849,6 @@ under the License. mockito-inline test - - org.openjdk.jol - jol-core - test - @@ -928,7 +923,7 @@ under the License. false false - -Xmx1024m --add-reads=org.apache.arrow.flight.core=ALL-UNNAMED --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED -javaagent:${settings.localRepository}/org/jmockit/jmockit/${jmockit.version}/jmockit-${jmockit.version}.jar @{argLine} + -Xmx1024m --add-reads=org.apache.arrow.flight.core=ALL-UNNAMED --add-opens=java.base/java.nio=org.apache.arrow.memory.core,ALL-UNNAMED --add-opens=java.base/java.lang=ALL-UNNAMED --add-opens=java.base/java.util=ALL-UNNAMED --add-opens=java.base/java.util.concurrent=ALL-UNNAMED --add-opens=java.base/java.math=ALL-UNNAMED -javaagent:${settings.localRepository}/org/jmockit/jmockit/${jmockit.version}/jmockit-${jmockit.version}.jar @{argLine} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java index fa74f86c429319..51007656960f46 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java @@ -28,12 +28,9 @@ final class HiveCacheSizeEstimator { // name plus derived value/literal strings and therefore remains skew-sensitive. private static final long ENTRY_BASE_BYTES = objectBytes(2L * 1024L); private static final long PARTITION_BASE_BYTES = objectBytes(896L); - // PartitionValueCacheKey retains an ImmutableList over the partition column types; the Type - // instances themselves are shared catalog singletons and are not charged. A single-element - // list is Guava's SingletonImmutableList (one reference, no array); wider lists add a - // backing array with one slot per column. - private static final long KEY_TYPE_LIST_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + // PartitionValueCacheKey retains an immutable list over the partition column types; the Type + // instances themselves are shared catalog singletons and are not charged. + private static final long KEY_TYPE_LIST_BYTES = 24L; private static final long PARTITION_COLUMN_BYTES = objectBytes(256L); // One copy is retained as the partition name and another in the decoded partition values. private static final long PARTITION_NAME_PAYLOAD_COPIES = 2L; @@ -47,9 +44,6 @@ private static long objectBytes(long bytes) { static MetaCacheSizeEstimate estimatePartitionValuesEntry( PartitionValueCacheKey key, HivePartitionValues value) { - if (!MetaCacheWeightUtils.isSupportedJvmObjectLayout()) { - return MetaCacheSizeEstimate.incomplete("unsupported_jvm_object_alignment"); - } long partitionCount = value.getIdToPartitionItem() == null ? 0L : value.getIdToPartitionItem().size(); long perPartitionBytes = MetaCacheWeightUtils.saturatedAdd( diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java index 02f6d30373eb56..5dbac8eaf5f644 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java @@ -29,173 +29,109 @@ import org.apache.iceberg.PartitionStatisticsFile; import org.apache.iceberg.Schema; import org.apache.iceberg.Snapshot; -import org.apache.iceberg.SortField; import org.apache.iceberg.SortOrder; import org.apache.iceberg.StatisticsFile; import org.apache.iceberg.Table; import org.apache.iceberg.TableMetadata; import org.apache.iceberg.encryption.EncryptedKey; -import org.apache.iceberg.expressions.Literal; -import org.apache.iceberg.transforms.Transform; -import org.apache.iceberg.transforms.UnknownTransform; import org.apache.iceberg.types.Type; import org.apache.iceberg.types.Types; import java.lang.reflect.Field; import java.lang.reflect.Modifier; -import java.math.BigDecimal; -import java.nio.ByteBuffer; -import java.util.ArrayList; import java.util.HashSet; import java.util.List; -import java.util.Locale; import java.util.Map; import java.util.Set; -/** Publication-time retained-weight formulas for Iceberg cache entries. */ +/** + * Publication-time approximate weight formulas for Iceberg cache entries. + * + *

    The formulas follow a coarse cardinality model: a rounded-up constant per stable logical + * dimension (snapshot, schema field, partition field, metadata entry, ...) plus the + * skew-sensitive string payload the loader already materialized. Constants are calibrated + * offline and deliberately absorb the lazy state Iceberg materializes after admission (schema + * name/id/lower-case/accessor indexes, partition-type graphs) instead of modeling those objects + * individually, so weights track metadata size without depending on SDK-private layouts. + * {@code max-weight} is an estimated admission budget, not an exact heap limit. + */ final class IcebergCacheSizeEstimator { - // Calibrated against JOL retained-graph deltas in IcebergExternalMetaCacheTest. - // Every metadata element visited (field, type, snapshot, summary entry, ...) costs a few - // reads; the bound only guards against pathological metadata and is far above real tables - // (a 10,000-snapshot history with 15 summary keys each is 160,000 elements). Exceeding it + // Every metadata element visited (field, snapshot, summary entry, ...) costs a few reads; + // the bound only guards against pathological metadata and is far above real tables (a + // 10,000-snapshot history with 15 summary keys each is 160,000 elements). Exceeding it // rejects weighted admission, so it must not be reachable by ordinary long-lived tables. private static final long MAX_TABLE_ACCOUNTING_ELEMENTS = 2_000_000L; - // Total name characters the estimator may lower-case while reserving case-insensitive indexes. + // Total name characters the estimator may account for retained name indexes. private static final long MAX_TABLE_ACCOUNTING_CHARACTERS = 4_000_000L; private static final int MAX_TYPE_ACCOUNTING_DEPTH = 128; - private static final long KEY_BASE_BYTES = objectBytes(128L); - private static final long TABLE_BASE_BYTES = objectBytes(16L * 1024L); - // TableMetadata-side share of one schema version: schemas list slot and schemasById entry, - // including the growth of both from their singleton to their regular immutable shapes. - private static final long SCHEMA_VERSION_BYTES = objectBytes(128L); - private static final long PARTITION_SPEC_BYTES = objectBytes(256L); - // Exact active-layout sizes of the Iceberg/Guava objects that lazy partition, sort and - // schema state allocates. Iceberg 1.10.1 field layouts are pinned by ICEBERG_LAZY_LAYOUT_SUPPORTED. - private static final long PARTITION_FIELD_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 8L); - private static final long SORT_FIELD_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L); - // Identity/Bucket/Truncate transforms are allocated per parsed field; time transforms are enums. - private static final long TRANSFORM_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); - private static final long NESTED_FIELD_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(5L, 5L); - private static final long STRUCT_TYPE_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(6L, 0L); - private static final long SCHEMA_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(11L, 8L); - private static final long IMMUTABLE_LIST_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); - private static final long IMMUTABLE_MAP_KEY_SET_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); - private static final long SINGLETON_IMMUTABLE_SET_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); - private static final long REGULAR_IMMUTABLE_SET_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 8L); - private static final long ARRAY_LIST_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 8L); - private static final long HASH_MAP_NODE_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L); - private static final long HASH_MAP_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(4L, 16L); - private static final long INTEGER_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L); - private static final long LONG_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 8L); - // Literals.BaseLiteral: value plus the transient serialized-buffer slot. - private static final long LITERAL_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); - // JDK 17 HeapByteBuffer: Buffer header fields, address, segment, hb and offset. - private static final long BYTE_BUFFER_BYTES = objectBytes(56L); - private static final long BIG_DECIMAL_BYTES = objectBytes(104L); - private static final long BOXED_DEFAULT_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 16L); - private static final String TRUNCATE_TRANSFORM_PREFIX = "truncate["; - // Truncate on a decimal source retains a BigInteger width (object plus one-int magnitude). - private static final long TRUNCATE_WIDTH_BYTES = MetaCacheWeightUtils.saturatedAdd( - MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 20L), - MetaCacheWeightUtils.estimatedIntArrayBytes(1L)); - private static final long LIST_MULTIMAP_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(9L, 0L); - private static final long CAPTURING_SUPPLIER_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); - private static final long POSITION_ACCESSOR_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 4L); - // One WrappedPositionAccessor (1 ref + int) per optional struct ancestor. Required ancestors - // collapse into a single Position2/3Accessor that replaces the inner accessor, which retains - // less than this per-level reservation. - private static final long WRAPPED_ACCESSOR_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 4L); - private static final long LIST_TYPE_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); - private static final long MAP_TYPE_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 0L); - private static final long DECIMAL_TYPE_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 8L); - private static final long FIXED_TYPE_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L); - private static final long GEOMETRY_TYPE_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); - private static final long GEOGRAPHY_TYPE_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); - private static final long SORT_ORDER_BYTES = objectBytes(256L); - private static final long TABLE_PROPERTY_BYTES = objectBytes(40L); - private static final long CURRENT_SNAPSHOT_BYTES = objectBytes(512L); - private static final long HISTORICAL_SNAPSHOT_BYTES = objectBytes(176L); - private static final long SNAPSHOT_LOG_ENTRY_BYTES = objectBytes(38L); - private static final long METADATA_LOG_ENTRY_BYTES = objectBytes(128L); - private static final long SNAPSHOT_REF_BYTES = objectBytes(128L); - private static final long STATISTICS_FILE_BYTES = objectBytes(512L); - private static final long BLOB_METADATA_BYTES = objectBytes(128L); - private static final long BLOB_FIELD_BYTES = objectBytes(32L); - private static final long PARTITION_STATISTICS_FILE_BYTES = objectBytes(256L); - private static final long ENCRYPTED_KEY_BYTES = objectBytes(256L); - // One retained IcebergPartition (value/transform ArrayLists) or one RangePartitionItem with a + + private static final long KEY_BASE_WEIGHT = 256L; + private static final long TABLE_BASE_WEIGHT = 16L * 1024L; + // One schema version: the Schema object, its lists, the schemasById entry and the struct it + // wraps, including their growth from singleton to regular immutable shapes. + private static final long SCHEMA_WEIGHT = 1024L; + // One nested field at any depth: the NestedField and its type node plus this field's share + // of every eager and lazy schema index (idToName, nameToId, idToField, lowerCaseNameToId, + // idToAccessor, struct field indexes and the secondary partition-type schema graph). + private static final long FIELD_WEIGHT = 1408L; + // Retained copies of one field name across the case-sensitive and lower-cased name indexes. + private static final long NAME_INDEX_COPIES = 4L; + // One partition spec: the spec object, its field list, javaClasses and the partitionType() + // graph with its own indexes. + private static final long SPEC_WEIGHT = 2048L; + // One partition field including its transform, index entries and the per-field share of the + // lazily built fieldsBySourceId multimap. + private static final long PARTITION_FIELD_WEIGHT = 1024L; + // The lazily built fieldsBySourceId multimap grows O(distinctSourceIds * fieldCount). + private static final long FIELDS_BY_SOURCE_SLOT_WEIGHT = 64L; + private static final long SORT_ORDER_WEIGHT = 512L; + private static final long SORT_FIELD_WEIGHT = 256L; + // One entry of any retained string map (table properties, snapshot summaries, blob or key + // properties): map node plus boxed/list slack; the strings are charged separately. + private static final long METADATA_ENTRY_WEIGHT = 128L; + private static final long SNAPSHOT_WEIGHT = 512L; + private static final long CURRENT_SNAPSHOT_WEIGHT = 1024L; + private static final long SNAPSHOT_LOG_WEIGHT = 64L; + private static final long METADATA_LOG_WEIGHT = 160L; + private static final long SNAPSHOT_REF_WEIGHT = 256L; + private static final long STATISTICS_FILE_WEIGHT = 640L; + private static final long BLOB_METADATA_WEIGHT = 256L; + private static final long BLOB_FIELD_WEIGHT = 32L; + private static final long PARTITION_STATISTICS_FILE_WEIGHT = 320L; + private static final long ENCRYPTED_KEY_WEIGHT = 320L; + // One retained IcebergPartition (value/transform lists) or one RangePartitionItem with a // single partition column plus its map entry; extra columns are charged by IcebergPartitionInfo. - private static final long PARTITION_BYTES = objectBytes(696L); + private static final long PARTITION_WEIGHT = 768L; // Outer map entry and table share of one merged-overlap group; the alias set itself and its // contents are charged by IcebergPartitionInfo per enclosed partition name. - private static final long PARTITION_ALIAS_BYTES = objectBytes(144L); + private static final long PARTITION_ALIAS_WEIGHT = 160L; // One name-mapping field: map node, boxed id and list object; alias arrays and Strings are // charged by IcebergSnapshotCacheValue when the mapping is copied. - private static final long NAME_MAPPING_ENTRY_BYTES = objectBytes(256L); - private static final long MANIFEST_ENTRY_BASE_BYTES = objectBytes(256L); - private static final long DATA_FILE_BYTES = objectBytes(896L); - private static final long DELETE_FILE_BYTES = objectBytes(1024L); - private static final long FILE_METRIC_ENTRY_BYTES = objectBytes(104L); - private static final String BASE_SNAPSHOT_CLASS_NAME = "org.apache.iceberg.BaseSnapshot"; - private static final Field[] BASE_SNAPSHOT_RETAINED_CACHE_FIELDS = - loadBaseSnapshotRetainedCacheFields(); + private static final long NAME_MAPPING_ENTRY_WEIGHT = 256L; + private static final long MANIFEST_ENTRY_BASE_WEIGHT = 512L; + private static final long DATA_FILE_WEIGHT = 1024L; + private static final long DELETE_FILE_WEIGHT = 1024L; + private static final long FILE_METRIC_ENTRY_WEIGHT = 128L; + // TableMetadata.snapshots()/snapshot(id) load lazily through a catalog supplier - // (REST snapshot-loading-mode=refs). Publication must not perform that IO. + // (REST snapshot-loading-mode=refs). Publication must not perform that IO, so this single + // reflective probe is retained as an IO guard rather than a layout model. private static final Field TABLE_METADATA_SNAPSHOTS_LOADED_FIELD = loadTableMetadataField("snapshotsLoaded", boolean.class); private static final Field TABLE_METADATA_SNAPSHOTS_SUPPLIER_FIELD = loadTableMetadataField("snapshotsSupplier", null); - // The formulas above are built on the Iceberg 1.10.1 instance-field layouts of the classes a - // cached table retains. Every non-static field is pinned, not only the transient lazy ones: a - // library upgrade that adds a retained reference makes weighted admission fail closed. - private static final boolean ICEBERG_LAZY_LAYOUT_SUPPORTED = checkIcebergLayout(); private IcebergCacheSizeEstimator() { } - private static long objectBytes(long bytes) { - return MetaCacheWeightUtils.estimatedObjectBytes(bytes); - } - static MetaCacheSizeEstimate estimateTableEntry(NameMapping key, IcebergTableCacheValue value) { - MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout(); - if (!layoutSupport.isComplete()) { - return layoutSupport; - } Table table = value.getRetainedIcebergTable(); MetaCacheSizeEstimate support = checkSupportedTable(table); if (!support.isComplete()) { return support; } long bytes = MetaCacheWeightUtils.saturatedAdd( - KEY_BASE_BYTES, MetaCacheWeightUtils.estimatedNameMappingBytes(key)); + KEY_BASE_WEIGHT, MetaCacheWeightUtils.estimatedNameMappingBytes(key)); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table)); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedTablePayloadBytes()); bytes = MetaCacheWeightUtils.saturatedAdd( @@ -205,11 +141,7 @@ static MetaCacheSizeEstimate estimateTableEntry(NameMapping key, IcebergTableCac static MetaCacheSizeEstimate estimateSnapshotEntry( IcebergSnapshotEntryKey key, IcebergSnapshotCacheValue value) { - MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout(); - if (!layoutSupport.isComplete()) { - return layoutSupport; - } - long bytes = KEY_BASE_BYTES; + long bytes = KEY_BASE_WEIGHT; bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping())); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, @@ -218,12 +150,13 @@ static MetaCacheSizeEstimate estimateSnapshotEntry( MetaCacheWeightUtils.estimatedStringBytes(key.getMetadataFileLocation())); IcebergPartitionInfo partitionInfo = value.getPartitionInfo(); - bytes = addCount(bytes, partitionInfo.getNameToPartitionItem().size(), PARTITION_BYTES); - bytes = addCount(bytes, partitionInfo.getNameToIcebergPartition().size(), PARTITION_BYTES); - bytes = addCount(bytes, partitionInfo.getNameToIcebergPartitionNames().size(), PARTITION_ALIAS_BYTES); + bytes = addCount(bytes, partitionInfo.getNameToPartitionItem().size(), PARTITION_WEIGHT); + bytes = addCount(bytes, partitionInfo.getNameToIcebergPartition().size(), PARTITION_WEIGHT); + bytes = addCount(bytes, partitionInfo.getNameToIcebergPartitionNames().size(), + PARTITION_ALIAS_WEIGHT); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, partitionInfo.getRetainedPayloadBytes()); bytes = addCount(bytes, value.getNameMapping().map(Map::size).orElse(0), - NAME_MAPPING_ENTRY_BYTES); + NAME_MAPPING_ENTRY_WEIGHT); bytes = MetaCacheWeightUtils.saturatedAdd( bytes, value.getRetainedNameMappingPayloadBytes()); @@ -249,34 +182,21 @@ static MetaCacheSizeEstimate estimateSnapshotEntry( static MetaCacheSizeEstimate estimateManifestEntry( IcebergManifestEntryKey key, ManifestCacheValue value) { - MetaCacheSizeEstimate layoutSupport = checkJvmObjectLayout(); - if (!layoutSupport.isComplete()) { - return layoutSupport; - } if (!value.isAccountingComplete()) { return MetaCacheSizeEstimate.incomplete("iceberg_manifest_accounting_incomplete"); } long bytes = MetaCacheWeightUtils.saturatedAdd( - MANIFEST_ENTRY_BASE_BYTES, + MANIFEST_ENTRY_BASE_WEIGHT, MetaCacheWeightUtils.estimatedStringBytes(key.getManifestPath())); - bytes = addCount(bytes, value.getDataFiles().size(), DATA_FILE_BYTES); - bytes = addCount(bytes, value.getDeleteFiles().size(), DELETE_FILE_BYTES); - bytes = addCount(bytes, value.getDataFileMetricEntryCount(), FILE_METRIC_ENTRY_BYTES); - bytes = addCount(bytes, value.getDeleteFileMetricEntryCount(), FILE_METRIC_ENTRY_BYTES); + bytes = addCount(bytes, value.getDataFiles().size(), DATA_FILE_WEIGHT); + bytes = addCount(bytes, value.getDeleteFiles().size(), DELETE_FILE_WEIGHT); + bytes = addCount(bytes, value.getDataFileMetricEntryCount(), FILE_METRIC_ENTRY_WEIGHT); + bytes = addCount(bytes, value.getDeleteFileMetricEntryCount(), FILE_METRIC_ENTRY_WEIGHT); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedPayloadBytes()); return MetaCacheSizeEstimate.complete(bytes); } - private static MetaCacheSizeEstimate checkJvmObjectLayout() { - return MetaCacheWeightUtils.isSupportedJvmObjectLayout() - ? MetaCacheSizeEstimate.complete(1L) - : MetaCacheSizeEstimate.incomplete("unsupported_jvm_object_alignment"); - } - private static MetaCacheSizeEstimate checkSupportedTable(Table table) { - if (!ICEBERG_LAZY_LAYOUT_SUPPORTED) { - return MetaCacheSizeEstimate.incomplete("unsupported_iceberg_lazy_layout"); - } if (table == null) { return MetaCacheSizeEstimate.incomplete("missing_iceberg_table"); } @@ -302,23 +222,21 @@ private static MetaCacheSizeEstimate checkSupportedTable(Table table) { private static long estimateTable(Table table) { TableMetadata metadata = ((HasTableOperations) table).operations().current(); long bytes = MetaCacheWeightUtils.saturatedAdd( - TABLE_BASE_BYTES, MetaCacheWeightUtils.estimatedStringBytes(table.name())); + TABLE_BASE_WEIGHT, MetaCacheWeightUtils.estimatedStringBytes(table.name())); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.estimatedStringBytes(metadata.location())); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.estimatedStringBytes(metadata.metadataFileLocation())); - - bytes = addCount(bytes, metadata.properties().size(), TABLE_PROPERTY_BYTES); + bytes = addCount(bytes, metadata.properties().size(), METADATA_ENTRY_WEIGHT); if (metadata.currentSnapshot() != null) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, CURRENT_SNAPSHOT_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, CURRENT_SNAPSHOT_WEIGHT); } return bytes; } /** - * Fully accounts variable payload with a bounded amount of publication-time work. Only - * already-parsed metadata is read; the SDK state it touches on the way (StructType, ListType - * and MapType fieldList copies, the identifier field set) is small, accounted and O(N). + * Weight of everything a retained table generation's metadata can grow into, computed from + * already-parsed metadata with bounded publication-time work and no IO. */ static long retainedTablePayloadBytes(Table table) { if (!(table instanceof HasTableOperations)) { @@ -340,12 +258,16 @@ static long retainedTablePayloadBytes(Table table) { bytes = MetaCacheWeightUtils.saturatedAdd(bytes, partitionSpecBytes(spec, budget)); } for (SortOrder sortOrder : metadata.sortOrders()) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, sortOrderBytes(sortOrder, budget)); + budget.chargeElements(MetaCacheWeightUtils.saturatedAdd( + 1L, sortOrder.fields().size())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SORT_ORDER_WEIGHT); + bytes = addCount(bytes, sortOrder.fields().size(), SORT_FIELD_WEIGHT); } for (Schema schema : metadata.schemas()) { bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaBytes(schema, budget)); } budget.chargeElements(metadata.properties().size()); + bytes = addCount(bytes, metadata.properties().size(), METADATA_ENTRY_WEIGHT); for (Map.Entry property : metadata.properties().entrySet()) { bytes = addString(bytes, property.getKey()); bytes = addString(bytes, property.getValue()); @@ -354,363 +276,135 @@ static long retainedTablePayloadBytes(Table table) { bytes = MetaCacheWeightUtils.saturatedAdd(bytes, snapshotBytes(snapshot, budget)); } budget.chargeElements(metadata.snapshotLog().size()); - bytes = addCount(bytes, metadata.snapshotLog().size(), SNAPSHOT_LOG_ENTRY_BYTES); + bytes = addCount(bytes, metadata.snapshotLog().size(), SNAPSHOT_LOG_WEIGHT); budget.chargeElements(metadata.previousFiles().size()); for (TableMetadata.MetadataLogEntry previousFile : metadata.previousFiles()) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, METADATA_LOG_ENTRY_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, METADATA_LOG_WEIGHT); bytes = addString(bytes, previousFile.file()); } budget.chargeElements(metadata.refs().size()); - bytes = addCount(bytes, metadata.refs().size(), SNAPSHOT_REF_BYTES); + bytes = addCount(bytes, metadata.refs().size(), SNAPSHOT_REF_WEIGHT); for (String refName : metadata.refs().keySet()) { bytes = addString(bytes, refName); } budget.chargeElements(metadata.statisticsFiles().size()); for (StatisticsFile statisticsFile : metadata.statisticsFiles()) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, STATISTICS_FILE_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, STATISTICS_FILE_WEIGHT); bytes = addString(bytes, statisticsFile.path()); for (BlobMetadata blob : statisticsFile.blobMetadata()) { budget.chargeElements(MetaCacheWeightUtils.saturatedAdd(1L, MetaCacheWeightUtils.saturatedAdd( blob.fields().size(), blob.properties().size()))); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, BLOB_METADATA_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, BLOB_METADATA_WEIGHT); bytes = addString(bytes, blob.type()); - bytes = addCount(bytes, blob.fields().size(), BLOB_FIELD_BYTES); - bytes = addStringMap(bytes, blob.properties(), TABLE_PROPERTY_BYTES); + bytes = addCount(bytes, blob.fields().size(), BLOB_FIELD_WEIGHT); + bytes = addStringMap(bytes, blob.properties()); } } budget.chargeElements(metadata.partitionStatisticsFiles().size()); for (PartitionStatisticsFile statisticsFile : metadata.partitionStatisticsFiles()) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, PARTITION_STATISTICS_FILE_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, PARTITION_STATISTICS_FILE_WEIGHT); bytes = addString(bytes, statisticsFile.path()); } budget.chargeElements(metadata.encryptionKeys().size()); for (EncryptedKey encryptedKey : metadata.encryptionKeys()) { budget.chargeElements(encryptedKey.properties().size()); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ENCRYPTED_KEY_BYTES); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ENCRYPTED_KEY_WEIGHT); bytes = addString(bytes, encryptedKey.keyId()); bytes = addString(bytes, encryptedKey.encryptedById()); - bytes = addBufferPayload(bytes, encryptedKey.encryptedKeyMetadata()); - bytes = addStringMap(bytes, encryptedKey.properties(), TABLE_PROPERTY_BYTES); + if (encryptedKey.encryptedKeyMetadata() != null) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedByteArrayBytes( + encryptedKey.encryptedKeyMetadata().remaining())); + } + bytes = addStringMap(bytes, encryptedKey.properties()); } bytes = addString(bytes, metadata.uuid()); return bytes; } - /** - * Account a PartitionSpec together with the lazy state that a normal scan materializes after - * admission: fieldList, javaClasses, partitionType() with its StructType indexes, the secondary - * Schema/Binder graph behind partitionType().asSchema() and fieldsBySourceId. Iceberg 1.10.1 - * allocates one Object[fieldCount] per distinct source id inside fieldsBySourceId, so that - * retained graph is O(distinctSourceIds * fieldCount); it is reserved here in O(fieldCount) - * publication work without materializing any of it. - */ private static long partitionSpecBytes(PartitionSpec spec, AccountingBudget budget) { List fields = spec.fields(); - long fieldCount = fields.size(); - budget.chargeElements(MetaCacheWeightUtils.saturatedAdd(1L, fieldCount)); - long bytes = PARTITION_SPEC_BYTES; - if (fieldCount == 0L) { - return bytes; - } - Set distinctSourceIds = new HashSet<>(); - long uncachedSourceIds = 0L; - long uncachedFieldIds = 0L; - long lowerCaseNameBytes = 0L; + budget.chargeElements(MetaCacheWeightUtils.saturatedAdd(1L, fields.size())); + long bytes = SPEC_WEIGHT; + bytes = addCount(bytes, fields.size(), PARTITION_FIELD_WEIGHT); + Set sourceIds = new HashSet<>(); for (PartitionField field : fields) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, PARTITION_FIELD_BYTES); - bytes = addTransformPayload(bytes, field.transform()); - bytes = addString(bytes, field.name()); - lowerCaseNameBytes = MetaCacheWeightUtils.saturatedAdd( - lowerCaseNameBytes, generatedLowerCaseNameBytes(field.name(), budget)); - if (isUncachedInteger(field.fieldId())) { - uncachedFieldIds++; - } - if (distinctSourceIds.add(field.sourceId()) && isUncachedInteger(field.sourceId())) { - uncachedSourceIds++; - } - } - // Eager PartitionField[] plus lazy fieldList and javaClasses. - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, objectArrayGrowthBytes(fieldCount)); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, immutableListBytes(fieldCount)); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, objectArrayGrowthBytes(fieldCount)); - // partitionType(): the StructType itself also exists for an unpartitioned spec. - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, objectArrayGrowthBytes(fieldCount)); - bytes = addCount(bytes, fieldCount, NESTED_FIELD_BYTES); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, - structTypeIndexBytes(fieldCount, uncachedFieldIds, lowerCaseNameBytes)); - if (!spec.schema().idsToOriginal().isEmpty()) { - // rawPartitionType() rebuilds the struct with original ids. - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, STRUCT_TYPE_BYTES); - bytes = MetaCacheWeightUtils.saturatedAdd( - bytes, MetaCacheWeightUtils.estimatedObjectArrayBytes(fieldCount)); - bytes = addCount(bytes, fieldCount, NESTED_FIELD_BYTES); - } - // A partition filter binds against partitionType().asSchema(). - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, secondarySchemaBytes( - SchemaShape.flat(fieldCount, uncachedFieldIds, lowerCaseNameBytes))); - // fieldsBySourceId: HashMap. - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, LIST_MULTIMAP_BYTES); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, CAPTURING_SUPPLIER_BYTES); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, - hashIdMapBytes(distinctSourceIds.size(), uncachedSourceIds)); - return addCount(bytes, distinctSourceIds.size(), - MetaCacheWeightUtils.saturatedAdd(ARRAY_LIST_BYTES, - MetaCacheWeightUtils.estimatedObjectArrayBytes(fieldCount))); - } - - /** Account a SortOrder: SortField[] with per-field transforms plus the lazy fieldList copy. */ - private static long sortOrderBytes(SortOrder sortOrder, AccountingBudget budget) { - long fieldCount = sortOrder.fields().size(); - budget.chargeElements(MetaCacheWeightUtils.saturatedAdd(1L, fieldCount)); - long bytes = SORT_ORDER_BYTES; - if (fieldCount == 0L) { - return bytes; - } - for (SortField field : sortOrder.fields()) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SORT_FIELD_BYTES); - bytes = addTransformPayload(bytes, field.transform()); - } - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, objectArrayGrowthBytes(fieldCount)); - return MetaCacheWeightUtils.saturatedAdd(bytes, immutableListBytes(fieldCount)); - } - - /** Transform instance plus the payload only some transforms retain. */ - private static long addTransformPayload(long bytes, Transform transform) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, TRANSFORM_BYTES); - if (transform instanceof UnknownTransform) { - return addString(bytes, transform.toString()); - } - if (transform.toString().startsWith(TRUNCATE_TRANSFORM_PREFIX)) { - // Truncate is package-private; its serialized name is the SPI contract. - return MetaCacheWeightUtils.saturatedAdd(bytes, TRUNCATE_WIDTH_BYTES); - } + sourceIds.add(field.sourceId()); + budget.chargeCharacters(field.name().length()); + // The name is retained by the field and again by the partition-type name indexes. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.saturatedMultiply( + MetaCacheWeightUtils.estimatedStringBytes(field.name()), NAME_INDEX_COPIES)); + } + // Reserve the O(distinctSources * fields) growth of the lazy fieldsBySourceId index. + bytes = addCount(bytes, + MetaCacheWeightUtils.saturatedMultiply(sourceIds.size(), fields.size()), + FIELDS_BY_SOURCE_SLOT_WEIGHT); return bytes; } - /** Lazy StructType indexes: fieldList, fieldsByName, fieldsByLowerCaseName and fieldsById. */ - private static long structTypeIndexBytes( - long fieldCount, long uncachedFieldIds, long lowerCaseNameBytes) { - long bytes = immutableListBytes(fieldCount); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, - immutableNameMapBytes(fieldCount, 0L, 0L)); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, - immutableNameMapBytes(fieldCount, 0L, lowerCaseNameBytes)); - return MetaCacheWeightUtils.saturatedAdd(bytes, - immutableNameMapBytes(fieldCount, uncachedFieldIds, 0L)); - } - - /** - * The Schema created by StructType.asSchema(): its constructor materializes idToName and two - * empty id maps; Binder and projection paths add nameToId, lowerCaseNameToId, idToField and - * idToAccessor; its own StructType copy grows the same lazy indexes as the root struct. - */ - private static long secondarySchemaBytes(SchemaShape shape) { - long bytes = schemaObjectBytes(shape); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaLookupBytes(shape)); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaLazyIndexBytes(shape)); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, STRUCT_TYPE_BYTES); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, - MetaCacheWeightUtils.estimatedObjectArrayBytes(shape.topLevelFieldCount)); - return MetaCacheWeightUtils.saturatedAdd(bytes, structTypeIndexBytes( - shape.topLevelFieldCount, shape.uncachedTopLevelFieldIdCount, - shape.topLevelLowerCaseStringBytes)); - } - - /** Schema object, empty identifier int[], the two empty id maps and the eager idToName keySet. */ - private static long schemaObjectBytes(SchemaShape shape) { - long bytes = MetaCacheWeightUtils.saturatedAdd( - SCHEMA_BYTES, MetaCacheWeightUtils.estimatedIntArrayBytes(0L)); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, HASH_MAP_BYTES); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, HASH_MAP_BYTES); - if (shape.fieldCount == 1L) { - return MetaCacheWeightUtils.saturatedAdd(bytes, SINGLETON_IMMUTABLE_SET_BYTES); - } - return shape.fieldCount > 1L - ? MetaCacheWeightUtils.saturatedAdd(bytes, IMMUTABLE_MAP_KEY_SET_BYTES) : bytes; - } - /** - * idToName (eager in the constructor), nameToId and idToField. Every map boxes uncached ids - * itself; idToName and nameToId each retain their own copy of every nested canonical name and - * nameToId also retains the short aliases. - */ - private static long schemaLookupBytes(SchemaShape shape) { - long bytes = immutableNameMapBytes( - shape.fieldCount, shape.uncachedFieldIdCount, shape.pathStringBytes); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, immutableNameMapBytes( - shape.nameEntryCount, shape.uncachedNameIdCount, - MetaCacheWeightUtils.saturatedAdd( - shape.pathStringBytes, shape.aliasStringBytes))); - return MetaCacheWeightUtils.saturatedAdd(bytes, - hashIdMapBytes(shape.fieldCount, shape.uncachedFieldIdCount)); - } - - /** lowerCaseNameToId and idToAccessor, materialized by case-insensitive lookups and Binder. */ - private static long schemaLazyIndexBytes(SchemaShape shape) { - long bytes = immutableNameMapBytes( - shape.nameEntryCount, shape.uncachedNameIdCount, shape.lowerCaseStringBytes); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, - hashIdMapBytes(shape.accessorFieldCount, shape.uncachedAccessorIdCount)); - bytes = addCount(bytes, shape.accessorFieldCount, POSITION_ACCESSOR_BYTES); - return addCount(bytes, shape.wrappedAccessorCount, WRAPPED_ACCESSOR_BYTES); - } - - /** - * One table schema version with every index a normal scan can materialize afterwards. Only - * metadata already parsed is read; nothing lazy is touched, and each field is visited once. + * One schema version: a constant per nested field at any depth plus the retained name/doc + * payload. The per-field constant absorbs the type node and this field's share of every + * eager and lazily materialized schema index. */ private static long schemaBytes(Schema schema, AccountingBudget budget) { - budget.chargeElements(1L); - SchemaShape shape = new SchemaShape(); - long bytes = SCHEMA_VERSION_BYTES; + long bytes = SCHEMA_WEIGHT; + bytes = addCount(bytes, schema.identifierFieldIds().size(), METADATA_ENTRY_WEIGHT); for (Types.NestedField field : schema.columns()) { - bytes = addFieldPayload( - bytes, field, PathState.ROOT, FieldKind.STRUCT_FIELD, budget, shape); - } - Set identifierFieldIds = schema.identifierFieldIds(); - budget.chargeElements(identifierFieldIds.size()); - bytes = addIdentifierFieldPayload(bytes, identifierFieldIds); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, shape.typeObjectBytes); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaObjectBytes(shape)); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, STRUCT_TYPE_BYTES); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, - MetaCacheWeightUtils.estimatedObjectArrayBytes(shape.topLevelFieldCount)); - if (shape.fieldCount == 0L) { - // Nothing can be looked up in an empty schema; its indexes stay shared singletons. - return bytes; - } - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaLookupBytes(shape)); - // Future lazy growth: main lookups, root struct indexes and the asSchema() secondary graph. - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaLazyIndexBytes(shape)); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, structTypeIndexBytes( - shape.topLevelFieldCount, shape.uncachedTopLevelFieldIdCount, - shape.topLevelLowerCaseStringBytes)); - return MetaCacheWeightUtils.saturatedAdd(bytes, secondarySchemaBytes(shape)); - } - - /** ImmutableList.copyOf(array): shared empty, singleton, or a list object plus copied array. */ - private static long immutableListBytes(long elementCount) { - if (elementCount <= 0L) { - return 0L; - } - if (elementCount == 1L) { - return IMMUTABLE_LIST_BYTES; - } - return MetaCacheWeightUtils.saturatedAdd(IMMUTABLE_LIST_BYTES, - MetaCacheWeightUtils.estimatedObjectArrayBytes(elementCount)); - } - - /** Growth of a reference array that replaces an empty array retained by the empty shape. */ - private static long objectArrayGrowthBytes(long elementCount) { - long populated = MetaCacheWeightUtils.estimatedObjectArrayBytes(elementCount); - long empty = MetaCacheWeightUtils.estimatedObjectArrayBytes(0L); - return populated == Long.MAX_VALUE ? populated : populated - empty; - } - - /** Boxed Integer keys outside the JVM Integer cache are retained per lookup map. */ - private static boolean isUncachedInteger(int value) { - return value < -128 || value > 127; - } - - /** - * Retained bytes of one lower-cased copy of a name, or 0 when the name is already lower case - * and the index reuses it. Every case-insensitive index (partition StructType, secondary - * Schema and secondary StructType) allocates its own copy, so callers add this per index. - */ - private static long generatedLowerCaseNameBytes(String name, AccountingBudget budget) { - budget.chargeCharacters(name.length()); - String lowerName = name.toLowerCase(Locale.ROOT); - if (lowerName.equals(name)) { - return 0L; + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, fieldBytes(field, budget, 0)); } - return MetaCacheWeightUtils.estimatedGeneratedStringBytes( - lowerName.length(), MetaCacheWeightUtils.isLatin1String(lowerName)); + return bytes; } - private static long hashIdMapBytes(long entryCount, long uncachedIds) { - long bytes = HASH_MAP_BYTES; - if (entryCount <= 0L) { - // HashMap allocates its table on the first put. - return bytes; + private static long fieldBytes(Types.NestedField field, AccountingBudget budget, int depth) { + if (depth > MAX_TYPE_ACCOUNTING_DEPTH) { + throw new IllegalStateException("Iceberg schema type nesting is too deep"); } - bytes = addCount(bytes, entryCount, HASH_MAP_NODE_BYTES); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, - MetaCacheWeightUtils.estimatedObjectArrayBytes( - hashMapCapacity(entryCount))); - return addCount(bytes, uncachedIds, INTEGER_BYTES); - } - - private static long immutableNameMapBytes( - long entryCount, long uncachedIds, long generatedStringBytes) { - long bytes = 0L; - if (entryCount == 1L) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, - MetaCacheWeightUtils.estimatedObjectLayoutBytes(8L, 0L)); - } else if (entryCount > 1L) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, - MetaCacheWeightUtils.estimatedObjectLayoutBytes(6L, 4L)); - bytes = addCount(bytes, entryCount, - MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 0L)); - bytes = MetaCacheWeightUtils.saturatedAdd( - bytes, MetaCacheWeightUtils.estimatedObjectArrayBytes(entryCount)); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, - MetaCacheWeightUtils.estimatedObjectArrayBytes( - immutableMapTableCapacity(entryCount))); + budget.chargeElements(1L); + long bytes = FIELD_WEIGHT; + String name = field.name(); + if (name != null) { + budget.chargeCharacters(name.length()); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.saturatedMultiply( + MetaCacheWeightUtils.estimatedStringBytes(name), NAME_INDEX_COPIES)); + } + if (field.doc() != null) { + budget.chargeCharacters(field.doc().length()); + bytes = addString(bytes, field.doc()); + } + Type type = field.type(); + if (type != null && type.isNestedType()) { + for (Types.NestedField nested : type.asNestedType().fields()) { + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, fieldBytes(nested, budget, depth + 1)); + } } - bytes = addCount(bytes, uncachedIds, INTEGER_BYTES); - return MetaCacheWeightUtils.saturatedAdd(bytes, generatedStringBytes); + return bytes; } private static long snapshotBytes(Snapshot snapshot, AccountingBudget budget) { - rejectMaterializedSnapshotPayload(snapshot); Map summary = snapshot.summary(); budget.chargeElements(MetaCacheWeightUtils.saturatedAdd( 1L, summary == null ? 0L : summary.size())); - long bytes = HISTORICAL_SNAPSHOT_BYTES; - // The parsed parent id is never inside the Long cache; the row-id fields are boxed too - // and only tiny values would share a cached instance, so each present field is charged. - bytes = addBoxedLong(bytes, snapshot.parentId()); - bytes = addBoxedLong(bytes, snapshot.firstRowId()); - bytes = addBoxedLong(bytes, snapshot.addedRows()); + long bytes = SNAPSHOT_WEIGHT; bytes = addString(bytes, snapshot.operation()); String manifestListLocation = snapshot.manifestListLocation(); if (manifestListLocation == null) { - // A snapshot serialized with an inline "manifests" array (legacy writers) retains a - // String[] of manifest locations that is only exposed through ManifestFile wrappers. + // A snapshot serialized with an inline "manifests" array (legacy v1 writers) retains + // a manifest-location list that is only exposed through FileIO-backed wrappers. // Reject weighted admission instead of doing IO or admitting an underestimate. throw new IllegalStateException( "Iceberg snapshot with inline manifest list is unsupported"); } bytes = addString(bytes, manifestListLocation); bytes = addString(bytes, snapshot.keyId()); - return addStringMap(bytes, summary, TABLE_PROPERTY_BYTES); - } - - private static void rejectMaterializedSnapshotPayload(Snapshot snapshot) { - if (!BASE_SNAPSHOT_CLASS_NAME.equals(snapshot.getClass().getName())) { - throw new IllegalStateException( - "Unsupported Iceberg snapshot implementation: " - + snapshot.getClass().getName()); - } - if (BASE_SNAPSHOT_RETAINED_CACHE_FIELDS == null) { - throw new IllegalStateException( - "Iceberg BaseSnapshot retained-cache inspection is unavailable"); - } - try { - // The field list is resolved once per process. Publication only performs a bounded - // number of O(1) reads and never walks a retained manifest/file graph. - for (Field retainedCacheField : BASE_SNAPSHOT_RETAINED_CACHE_FIELDS) { - if (retainedCacheField.get(snapshot) != null) { - throw new IllegalStateException( - "Iceberg snapshot has materialized retained payload: " - + retainedCacheField.getName()); - } - } - } catch (IllegalAccessException e) { - throw new IllegalStateException( - "Cannot inspect Iceberg BaseSnapshot retained payload", e); + if (summary != null) { + bytes = addCount(bytes, summary.size(), METADATA_ENTRY_WEIGHT); + bytes = addStringMap(bytes, summary); } + return bytes; } /** Iceberg marks snapshots loaded at construction unless a lazy supplier was configured. */ @@ -741,262 +435,9 @@ private static Field loadTableMetadataField(String name, Class expectedType) } } - private static Field[] loadBaseSnapshotRetainedCacheFields() { - try { - Class snapshotClass = Class.forName( - BASE_SNAPSHOT_CLASS_NAME, false, Snapshot.class.getClassLoader()); - List retainedCacheFields = new ArrayList<>(); - for (Field field : snapshotClass.getDeclaredFields()) { - int modifiers = field.getModifiers(); - if (Modifier.isTransient(modifiers) && !Modifier.isStatic(modifiers) - && !field.getType().isPrimitive()) { - field.setAccessible(true); - retainedCacheFields.add(field); - } - } - return retainedCacheFields.isEmpty() - ? null : retainedCacheFields.toArray(new Field[0]); - } catch (ReflectiveOperationException | RuntimeException e) { - return null; - } - } - - private static boolean checkIcebergLayout() { - ClassLoader loader = Snapshot.class.getClassLoader(); - return MetaCacheWeightUtils.hasExpectedInstanceFields(Schema.class, - "struct:StructType", "schemaId:int", "identifierFieldIds:int[]", - "highestFieldId:int", "aliasToId:BiMap", "idToField:Map", "nameToId:Map", - "lowerCaseNameToId:Map", "idToAccessor:Map", "idToName:Map", - "identifierFieldIdSet:Set", "idsToReassigned:Map", "idsToOriginal:Map") - && MetaCacheWeightUtils.hasExpectedInstanceFields(PartitionSpec.class, - "schema:Schema", "specId:int", "fields:PartitionField[]", - "fieldsBySourceId:ListMultimap", "lazyJavaClasses:Class[]", - "lazyPartitionType:StructType", "lazyRawPartitionType:StructType", - "fieldList:List", "lastAssignedFieldId:int") - && MetaCacheWeightUtils.hasExpectedInstanceFields(PartitionField.class, - "sourceId:int", "fieldId:int", "name:String", "transform:Transform") - && MetaCacheWeightUtils.hasExpectedInstanceFields(SortOrder.class, - "schema:Schema", "orderId:int", "fields:SortField[]", "fieldList:List") - && MetaCacheWeightUtils.hasExpectedInstanceFields(SortField.class, - "transform:Transform", "sourceId:int", "direction:SortDirection", - "nullOrder:NullOrder") - && MetaCacheWeightUtils.hasExpectedInstanceFields(Types.StructType.class, - "fields:NestedField[]", "schema:Schema", "fieldList:List", - "fieldsByName:Map", "fieldsByLowerCaseName:Map", "fieldsById:Map") - && MetaCacheWeightUtils.hasExpectedInstanceFields(Types.ListType.class, - "elementField:NestedField", "fields:List") - && MetaCacheWeightUtils.hasExpectedInstanceFields(Types.MapType.class, - "keyField:NestedField", "valueField:NestedField", "fields:List") - && MetaCacheWeightUtils.hasExpectedInstanceFields(Types.NestedField.class, - "isOptional:boolean", "id:int", "name:String", "type:Type", - "doc:String", "initialDefault:Literal", "writeDefault:Literal") - && MetaCacheWeightUtils.hasExpectedInstanceFields(TableMetadata.class, - "metadataFileLocation:String", "formatVersion:int", "uuid:String", - "location:String", "lastSequenceNumber:long", "lastUpdatedMillis:long", - "lastColumnId:int", "currentSchemaId:int", "schemas:List", - "defaultSpecId:int", "specs:List", "lastAssignedPartitionId:int", - "defaultSortOrderId:int", "sortOrders:List", "properties:Map", - "currentSnapshotId:long", "schemasById:Map", "specsById:Map", - "sortOrdersById:Map", "snapshotLog:List", "previousFiles:List", - "statisticsFiles:List", "partitionStatisticsFiles:List", "changes:List", - "nextRowId:long", "encryptionKeys:List", - "snapshotsSupplier:SerializableSupplier", "snapshots:List", - "snapshotsById:Map", "refs:Map", "snapshotsLoaded:boolean") - && MetaCacheWeightUtils.hasExpectedInstanceFields(BASE_SNAPSHOT_CLASS_NAME, loader, - "snapshotId:long", "parentId:Long", "sequenceNumber:long", - "timestampMillis:long", "manifestListLocation:String", - "operation:String", "summary:Map", "schemaId:Integer", - "v1ManifestLocations:String[]", "firstRowId:Long", "addedRows:Long", - "keyId:String", "allManifests:List", "dataManifests:List", - "deleteManifests:List", "addedDataFiles:List", "removedDataFiles:List", - "addedDeleteFiles:List", "removedDeleteFiles:List"); - } - - private static long addBoxedLong(long bytes, Long value) { - return value == null ? bytes : MetaCacheWeightUtils.saturatedAdd(bytes, LONG_BYTES); - } - - private static long addBufferPayload(long bytes, ByteBuffer buffer) { - return buffer == null ? bytes : MetaCacheWeightUtils.saturatedAdd(bytes, buffer.capacity()); - } - - /** - * Account one NestedField, its owned strings and its type subtree, and record the shape data - * the lookup-map formulas need. Canonical and short names follow Iceberg's IndexByName: a - * nested name joins its ancestors with '.', a struct-typed list element or map value is left - * out of its children's short names (which then become aliases), and every lower-case index - * lower-cases each entry. - */ - private static long addFieldPayload( - long bytes, Types.NestedField field, PathState ancestors, FieldKind kind, - AccountingBudget budget, SchemaShape shape) { - budget.chargeElements(1L); - budget.chargeCharacters(field.name().length()); - String name = field.name(); - String lowerName = name.toLowerCase(Locale.ROOT); - boolean nameLatin1 = MetaCacheWeightUtils.isLatin1String(name); - boolean lowerLatin1 = MetaCacheWeightUtils.isLatin1String(lowerName); - shape.addField(field.fieldId(), ancestors, name, nameLatin1, lowerName, lowerLatin1); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, NESTED_FIELD_BYTES); - if (kind == FieldKind.STRUCT_FIELD) { - // List element and map key/value fields are named by shared "element"/"key"/"value" - // literals inside Iceberg's type constructors. - bytes = addString(bytes, name); - } - bytes = addString(bytes, field.doc()); - bytes = addDefaultPayload(bytes, field.initialDefaultLiteral()); - bytes = addDefaultPayload(bytes, field.writeDefaultLiteral()); - boolean pushShortName = kind == FieldKind.STRUCT_FIELD || kind == FieldKind.MAP_KEY - || !field.type().isStructType(); - // Only fields nested through a chain of struct fields get accessors; anything below a - // list or map does not. - boolean structChildren = kind == FieldKind.STRUCT_FIELD && field.type().isStructType(); - PathState children = ancestors.push(name.length(), nameLatin1, lowerName.length(), - lowerLatin1, pushShortName, structChildren); - return addTypePayload(bytes, field.type(), children, budget, shape); - } - - private static long addTypePayload( - long bytes, Type type, PathState ancestors, AccountingBudget budget, - SchemaShape shape) { - if (ancestors.typeDepth > MAX_TYPE_ACCOUNTING_DEPTH) { - throw new IllegalStateException( - "Iceberg cache accounting type depth exceeded"); - } - budget.chargeElements(1L); - if (type.isStructType()) { - List fields = type.asStructType().fields(); - // A nested struct's fieldList is materialized by every visitor. Its own name/id - // lookup indexes and asSchema() are not reserved: read paths resolve nested names - // through the root Schema maps and Binder binds only root and partition structs; - // nested-column DDL runs against a freshly loaded live table, not a cached one. - shape.addTypeObject(MetaCacheWeightUtils.saturatedAdd( - MetaCacheWeightUtils.saturatedAdd(STRUCT_TYPE_BYTES, - MetaCacheWeightUtils.estimatedObjectArrayBytes(fields.size())), - immutableListBytes(fields.size()))); - for (Types.NestedField field : fields) { - bytes = addFieldPayload( - bytes, field, ancestors, FieldKind.STRUCT_FIELD, budget, shape); - } - } else if (type.isListType()) { - shape.addTypeObject(MetaCacheWeightUtils.saturatedAdd( - LIST_TYPE_BYTES, IMMUTABLE_LIST_BYTES)); - bytes = addFieldPayload(bytes, type.asListType().fields().get(0), - ancestors, FieldKind.LIST_ELEMENT, budget, shape); - } else if (type.isMapType()) { - shape.addTypeObject(MetaCacheWeightUtils.saturatedAdd( - MetaCacheWeightUtils.saturatedAdd(MAP_TYPE_BYTES, IMMUTABLE_LIST_BYTES), - MetaCacheWeightUtils.estimatedObjectArrayBytes(2L))); - bytes = addFieldPayload(bytes, type.asMapType().fields().get(0), - ancestors, FieldKind.MAP_KEY, budget, shape); - bytes = addFieldPayload(bytes, type.asMapType().fields().get(1), - ancestors, FieldKind.MAP_VALUE, budget, shape); - } else if (type instanceof Types.DecimalType) { - shape.addTypeObject(DECIMAL_TYPE_BYTES); - } else if (type instanceof Types.FixedType) { - shape.addTypeObject(FIXED_TYPE_BYTES); - } else if (type instanceof Types.GeometryType) { - shape.addTypeObject(MetaCacheWeightUtils.saturatedAdd(GEOMETRY_TYPE_BYTES, - MetaCacheWeightUtils.estimatedStringBytes( - ((Types.GeometryType) type).crs()))); - } else if (type instanceof Types.GeographyType) { - shape.addTypeObject(MetaCacheWeightUtils.saturatedAdd(GEOGRAPHY_TYPE_BYTES, - MetaCacheWeightUtils.estimatedStringBytes( - ((Types.GeographyType) type).crs()))); - } - // Other primitive types are shared singletons. - return bytes; - } - - /** Account the int[] and lazy ImmutableSet retained by Schema.identifierFieldIds(). */ - private static long addIdentifierFieldPayload(long bytes, Set fieldIds) { - long count = fieldIds.size(); - if (count == 0L) { - return bytes; - } - long uncachedIds = 0L; - for (int fieldId : fieldIds) { - if (isUncachedInteger(fieldId)) { - uncachedIds++; - } - } - // The int[] grows from the empty array of a schema without identifier fields. - long additions = MetaCacheWeightUtils.estimatedIntArrayPayloadBytes(count); - additions = addCount(additions, uncachedIds, INTEGER_BYTES); - if (count == 1L) { - // ImmutableSet.copyOf(one element) is a SingletonImmutableSet. - return MetaCacheWeightUtils.saturatedAdd(bytes, - MetaCacheWeightUtils.saturatedAdd(additions, SINGLETON_IMMUTABLE_SET_BYTES)); - } - // RegularImmutableSet: the set object, its dense elements array and open-addressing table. - // The shared empty set of an identifier-free schema stays reachable through other schemas, - // so nothing is subtracted for it. - additions = MetaCacheWeightUtils.saturatedAdd(additions, REGULAR_IMMUTABLE_SET_BYTES); - additions = MetaCacheWeightUtils.saturatedAdd( - additions, MetaCacheWeightUtils.estimatedObjectArrayBytes(count)); - additions = MetaCacheWeightUtils.saturatedAdd(additions, - MetaCacheWeightUtils.estimatedObjectArrayBytes( - immutableSetTableCapacity(count))); - return MetaCacheWeightUtils.saturatedAdd(bytes, additions); - } - - private static long immutableSetTableCapacity(long size) { - long capacity = 2L; - while (MetaCacheWeightUtils.saturatedMultiply(size, 10L) - > MetaCacheWeightUtils.saturatedMultiply(capacity, 7L)) { - capacity = MetaCacheWeightUtils.saturatedMultiply(capacity, 2L); - if (capacity == Long.MAX_VALUE) { - return capacity; - } - } - return capacity; - } - - private static long hashMapCapacity(long size) { - long capacity = 16L; - while (size > capacity - capacity / 4L) { - capacity = MetaCacheWeightUtils.saturatedMultiply(capacity, 2L); - if (capacity == Long.MAX_VALUE) { - return capacity; - } - } - return capacity; - } - - private static long immutableMapTableCapacity(long size) { - long capacity = Long.highestOneBit(size); - return MetaCacheWeightUtils.saturatedMultiply(size, 5L) - > MetaCacheWeightUtils.saturatedMultiply(capacity, 6L) - ? MetaCacheWeightUtils.saturatedMultiply(capacity, 2L) : capacity; - } - - /** - * A v3 field default is retained as an Iceberg Literal wrapper (value plus a transient - * ByteBuffer slot) around its boxed or buffer value. - */ - private static long addDefaultPayload(long bytes, Literal literal) { - if (literal == null) { - return bytes; - } - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, LITERAL_BYTES); - Object value = literal.value(); - if (value instanceof CharSequence) { - return MetaCacheWeightUtils.saturatedAdd(bytes, - MetaCacheWeightUtils.estimatedCharSequenceBytes((CharSequence) value)); - } else if (value instanceof ByteBuffer) { - return MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.saturatedAdd( - BYTE_BUFFER_BYTES, - MetaCacheWeightUtils.estimatedByteArrayBytes(((ByteBuffer) value).capacity()))); - } else if (value instanceof byte[]) { - return MetaCacheWeightUtils.saturatedAdd(bytes, - MetaCacheWeightUtils.estimatedByteArrayBytes(((byte[]) value).length)); - } else if (value instanceof BigDecimal) { - return MetaCacheWeightUtils.saturatedAdd(bytes, BIG_DECIMAL_BYTES); - } else if (value == null || value instanceof Boolean) { - return bytes; - } - // Boxed numbers, UUIDs and other small immutable values. - return MetaCacheWeightUtils.saturatedAdd(bytes, BOXED_DEFAULT_BYTES); + private static long addCount(long bytes, long count, long perElementBytes) { + return MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.saturatedMultiply(count, perElementBytes)); } private static long addString(long bytes, String value) { @@ -1004,11 +445,10 @@ private static long addString(long bytes, String value) { bytes, MetaCacheWeightUtils.estimatedStringBytes(value)); } - private static long addStringMap(long bytes, Map values, long entryBytes) { + private static long addStringMap(long bytes, Map values) { if (values == null) { return bytes; } - bytes = addCount(bytes, values.size(), entryBytes); for (Map.Entry entry : values.entrySet()) { bytes = addString(bytes, entry.getKey()); bytes = addString(bytes, entry.getValue()); @@ -1016,17 +456,6 @@ private static long addStringMap(long bytes, Map values, long en return bytes; } - private static long addCount(long bytes, long count, long bytesPerItem) { - return MetaCacheWeightUtils.saturatedAdd(bytes, - MetaCacheWeightUtils.saturatedMultiply(count, bytesPerItem)); - } - - /** - * Hard bound on publication-time estimator work. Elements bound the number of metadata - * objects visited; characters bound the String scanning (lower-casing) performed for - * lazy-index reservations. Exceeding either throws, which estimateSafely turns into an - * incomplete estimate: weighted admission is rejected but the load itself succeeds. - */ private static final class AccountingBudget { private long remainingElements; private long remainingCharacters; @@ -1051,180 +480,4 @@ private void chargeCharacters(long characters) { remainingCharacters -= characters; } } - - private enum FieldKind { - STRUCT_FIELD, LIST_ELEMENT, MAP_KEY, MAP_VALUE - } - - /** - * Immutable name-stack state of a field's ancestors: character counts and Latin-1 coders of - * the joined canonical path, the short-alias path and both lower-cased forms. - */ - private static final class PathState { - private static final PathState ROOT = new PathState( - -1L, true, -1L, true, -1L, true, -1L, true, 0, 0); - - private final long pathCharacters; - private final boolean pathLatin1; - private final long shortPathCharacters; - private final boolean shortPathLatin1; - private final long lowerPathCharacters; - private final boolean lowerPathLatin1; - private final long shortLowerPathCharacters; - private final boolean shortLowerPathLatin1; - // Struct-field ancestors of the next field, or -1 inside a list or map (no accessors). - private final int structDepth; - private final int typeDepth; - - private PathState(long pathCharacters, boolean pathLatin1, long shortPathCharacters, - boolean shortPathLatin1, long lowerPathCharacters, boolean lowerPathLatin1, - long shortLowerPathCharacters, boolean shortLowerPathLatin1, int structDepth, - int typeDepth) { - this.pathCharacters = pathCharacters; - this.pathLatin1 = pathLatin1; - this.shortPathCharacters = shortPathCharacters; - this.shortPathLatin1 = shortPathLatin1; - this.lowerPathCharacters = lowerPathCharacters; - this.lowerPathLatin1 = lowerPathLatin1; - this.shortLowerPathCharacters = shortLowerPathCharacters; - this.shortLowerPathLatin1 = shortLowerPathLatin1; - this.structDepth = structDepth; - this.typeDepth = typeDepth; - } - - private boolean isRoot() { - return pathCharacters < 0L; - } - - private boolean shortPathDiffers() { - return shortPathCharacters != pathCharacters; - } - - /** - * Push a field name for its children; the short name is pushed only when requested and - * accessor depth continues only for the children of a struct-typed struct field. - */ - private PathState push(long nameCharacters, boolean nameLatin1, long lowerCharacters, - boolean lowerLatin1, boolean pushShortName, boolean structChildren) { - return new PathState( - join(pathCharacters, nameCharacters), pathLatin1 && nameLatin1, - pushShortName ? join(shortPathCharacters, nameCharacters) : shortPathCharacters, - pushShortName ? shortPathLatin1 && nameLatin1 : shortPathLatin1, - join(lowerPathCharacters, lowerCharacters), lowerPathLatin1 && lowerLatin1, - pushShortName ? join(shortLowerPathCharacters, lowerCharacters) - : shortLowerPathCharacters, - pushShortName ? shortLowerPathLatin1 && lowerLatin1 : shortLowerPathLatin1, - structChildren && structDepth >= 0 ? structDepth + 1 : -1, typeDepth + 1); - } - - private static long join(long parentCharacters, long nameCharacters) { - return parentCharacters < 0L ? nameCharacters - : MetaCacheWeightUtils.saturatedAdd( - MetaCacheWeightUtils.saturatedAdd(parentCharacters, 1L), - nameCharacters); - } - } - - /** Cardinalities and generated-String bytes that size a schema's lookup indexes. */ - private static final class SchemaShape { - private long fieldCount; - private long topLevelFieldCount; - private long uncachedFieldIdCount; - private long uncachedTopLevelFieldIdCount; - private long nameEntryCount; - private long uncachedNameIdCount; - // One copy each; the formulas add a copy per index that retains it. - private long pathStringBytes; - private long aliasStringBytes; - private long lowerCaseStringBytes; - private long topLevelLowerCaseStringBytes; - private long accessorFieldCount; - private long uncachedAccessorIdCount; - private long wrappedAccessorCount; - private long typeObjectBytes; - - /** A flat struct of {@code fieldCount} top-level fields, as used by partition types. */ - private static SchemaShape flat( - long fieldCount, long uncachedFieldIds, long lowerCaseStringBytes) { - SchemaShape shape = new SchemaShape(); - shape.fieldCount = fieldCount; - shape.topLevelFieldCount = fieldCount; - shape.uncachedFieldIdCount = uncachedFieldIds; - shape.uncachedTopLevelFieldIdCount = uncachedFieldIds; - shape.nameEntryCount = fieldCount; - shape.uncachedNameIdCount = uncachedFieldIds; - shape.lowerCaseStringBytes = lowerCaseStringBytes; - shape.topLevelLowerCaseStringBytes = lowerCaseStringBytes; - shape.accessorFieldCount = fieldCount; - shape.uncachedAccessorIdCount = uncachedFieldIds; - return shape; - } - - private void addField(int fieldId, PathState ancestors, String name, boolean nameLatin1, - String lowerName, boolean lowerLatin1) { - boolean uncached = isUncachedInteger(fieldId); - fieldCount++; - nameEntryCount++; - if (uncached) { - uncachedFieldIdCount++; - uncachedNameIdCount++; - } - if (ancestors.structDepth >= 0) { - accessorFieldCount++; - wrappedAccessorCount = MetaCacheWeightUtils.saturatedAdd( - wrappedAccessorCount, ancestors.structDepth); - if (uncached) { - uncachedAccessorIdCount++; - } - } - if (ancestors.isRoot()) { - topLevelFieldCount++; - if (uncached) { - uncachedTopLevelFieldIdCount++; - } - if (!name.equals(lowerName)) { - // Lower-case indexes only allocate when toLowerCase() changes the name. - long lowerBytes = MetaCacheWeightUtils.estimatedGeneratedStringBytes( - lowerName.length(), lowerLatin1); - lowerCaseStringBytes = MetaCacheWeightUtils.saturatedAdd( - lowerCaseStringBytes, lowerBytes); - topLevelLowerCaseStringBytes = MetaCacheWeightUtils.saturatedAdd( - topLevelLowerCaseStringBytes, lowerBytes); - } - return; - } - // Nested: IndexByName joins a new canonical String, and the lower-case index keeps - // either that joined String or its lower-cased copy. - pathStringBytes = MetaCacheWeightUtils.saturatedAdd(pathStringBytes, - MetaCacheWeightUtils.estimatedGeneratedStringBytes( - PathState.join(ancestors.pathCharacters, name.length()), - ancestors.pathLatin1 && nameLatin1)); - lowerCaseStringBytes = MetaCacheWeightUtils.saturatedAdd(lowerCaseStringBytes, - MetaCacheWeightUtils.estimatedGeneratedStringBytes( - PathState.join(ancestors.lowerPathCharacters, lowerName.length()), - ancestors.lowerPathLatin1 && lowerLatin1)); - if (ancestors.shortPathDiffers()) { - // A short alias exists whenever an ancestor was left out of the short path. - // Iceberg drops an alias that collides with a canonical name; counting the rare - // collision is conservative and avoids building name sets at publication. - nameEntryCount++; - if (uncached) { - uncachedNameIdCount++; - } - aliasStringBytes = MetaCacheWeightUtils.saturatedAdd(aliasStringBytes, - MetaCacheWeightUtils.estimatedGeneratedStringBytes( - PathState.join(ancestors.shortPathCharacters, name.length()), - ancestors.shortPathLatin1 && nameLatin1)); - lowerCaseStringBytes = MetaCacheWeightUtils.saturatedAdd(lowerCaseStringBytes, - MetaCacheWeightUtils.estimatedGeneratedStringBytes( - PathState.join( - ancestors.shortLowerPathCharacters, lowerName.length()), - ancestors.shortLowerPathLatin1 && lowerLatin1)); - } - } - - private void addTypeObject(long bytes) { - typeObjectBytes = MetaCacheWeightUtils.saturatedAdd(typeObjectBytes, bytes); - } - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java index 0bfa8346036569..b618b4b51dbd35 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergPartitionInfo.java @@ -34,7 +34,7 @@ public class IcebergPartitionInfo { private static final long RANGE_ENDPOINTS_PER_ITEM = 2L; // A merged-overlap alias group is a HashSet of the enclosed physical partition names; the // names themselves are shared with the partition maps. - private static final long HASH_SET_BYTES = MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 0L); + private static final long HASH_SET_BYTES = 24L; private final Map nameToPartitionItem; private final Map nameToIcebergPartition; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java index cb8923a5bf2e4d..617d2aa6fb53b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/cache/ManifestCacheValue.java @@ -50,13 +50,11 @@ public class ManifestCacheValue { // admitted with an underestimate, so it sits well above ordinary wide manifests: 10,000 files // with 200 lower/upper bounds each is 4,000,000 elements. private static final long MAX_DEEP_ACCOUNTING_ELEMENTS = 8_000_000L; - // The per-file constants in IcebergCacheSizeEstimator describe the Iceberg 1.10.1 copies that - // ManifestReader + ContentFile.copy() produce. Only those implementations, with their pinned - // instance-field layouts, are accounted; anything else fails closed at build time. - private static final String GENERIC_DATA_FILE_CLASS_NAME = "org.apache.iceberg.GenericDataFile"; - private static final String GENERIC_DELETE_FILE_CLASS_NAME = - "org.apache.iceberg.GenericDeleteFile"; - private static final boolean CONTENT_FILE_LAYOUT_SUPPORTED = checkContentFileLayout(); + // A partition container of an unknown StructLike implementation: instance plus a generous + // per-slot share; its (possibly shared) schema graph is unknown and charged per file. + private static final long UNKNOWN_PARTITION_CONTAINER_BYTES = 512L; + private static final long UNKNOWN_PARTITION_SLOT_BYTES = 128L; + private static final long UNKNOWN_PARTITION_VALUE_BYTES = 64L; private final List dataFiles; private final List deleteFiles; @@ -108,31 +106,6 @@ public static Builder deleteFilesBuilder(boolean accountRetainedSize) { return new Builder(false, accountRetainedSize); } - private static boolean checkContentFileLayout() { - ClassLoader loader = ContentFile.class.getClassLoader(); - return MetaCacheWeightUtils.hasExpectedInstanceFields(GENERIC_DATA_FILE_CLASS_NAME, loader) - && MetaCacheWeightUtils.hasExpectedInstanceFields( - GENERIC_DELETE_FILE_CLASS_NAME, loader) - && MetaCacheWeightUtils.hasExpectedInstanceFields( - "org.apache.iceberg.BaseFile", loader, - "partitionType:StructType", "fileOrdinal:Long", "manifestLocation:String", - "partitionSpecId:int", "content:FileContent", "filePath:String", - "format:FileFormat", "partitionData:PartitionData", "recordCount:Long", - "fileSizeInBytes:long", "dataSequenceNumber:Long", - "fileSequenceNumber:Long", "columnSizes:Map", "valueCounts:Map", - "nullValueCounts:Map", "nanValueCounts:Map", "lowerBounds:Map", - "upperBounds:Map", "splitOffsets:long[]", "equalityIds:int[]", - "keyMetadata:byte[]", "sortOrderId:Integer", "firstRowId:Long", - "referencedDataFile:String", "contentOffset:Long", - "contentSizeInBytes:Long", "avroSchema:Schema") - && MetaCacheWeightUtils.hasExpectedInstanceFields( - "org.apache.iceberg.avro.SupportsIndexProjection", loader, - "fromProjectionPos:int[]") - && MetaCacheWeightUtils.hasExpectedInstanceFields(PartitionData.class, - "partitionType:StructType", "size:int", "data:Object[]", - "stringSchema:String", "schema:Schema"); - } - public List getDataFiles() { return dataFiles; } @@ -204,7 +177,6 @@ private void recordAccounting(ContentFile file) { return; } try { - requireSupportedContentFile(file); StructLike partition = file.partition(); long nextDeepElements = MetaCacheWeightUtils.saturatedAdd( deepAccountingElements, deepAccountingElements(file, partition)); @@ -223,18 +195,6 @@ private void recordAccounting(ContentFile file) { } } - private void requireSupportedContentFile(ContentFile file) { - if (!CONTENT_FILE_LAYOUT_SUPPORTED) { - throw new IllegalStateException("unsupported Iceberg content file layout"); - } - String expectedClassName = dataContent - ? GENERIC_DATA_FILE_CLASS_NAME : GENERIC_DELETE_FILE_CLASS_NAME; - if (file == null || !expectedClassName.equals(file.getClass().getName())) { - throw new IllegalStateException("unsupported Iceberg content file implementation: " - + (file == null ? "null" : file.getClass().getName())); - } - } - private void addAccounting(FileAccounting accounting) { metricEntryCount = MetaCacheWeightUtils.saturatedAdd( metricEntryCount, accounting.metricEntryCount); @@ -254,9 +214,13 @@ private void accountPartitionOwnership(StructLike partition) { return; } if (!(partition instanceof PartitionData)) { - throw new IllegalArgumentException( - "unsupported Iceberg partition container: " - + partition.getClass().getName()); + // Unknown partition containers cannot share their schema accounting across + // files; charge a generic conservative weight per file instead of rejecting. + retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd(retainedPayloadBytes, + MetaCacheWeightUtils.saturatedAdd(UNKNOWN_PARTITION_CONTAINER_BYTES, + MetaCacheWeightUtils.saturatedMultiply( + partition.size(), UNKNOWN_PARTITION_SLOT_BYTES))); + return; } PartitionData partitionData = (PartitionData) partition; retainedPayloadBytes = MetaCacheWeightUtils.saturatedAdd( @@ -380,8 +344,8 @@ private static long addPartitionPayload(long bytes, StructLike partition) { bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.estimatedObjectBytes(16L)); } else if (value != null) { - throw new IllegalArgumentException( - "unsupported Iceberg partition value: " + value.getClass().getName()); + // Unknown scalar partition value types get a generic conservative weight. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, UNKNOWN_PARTITION_VALUE_BYTES); } } return bytes; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java index 34ccf41c718b1b..f518dbc905b910 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/CacheSpec.java @@ -42,7 +42,9 @@ *

  • enable=false disables cache
    • *
  • ttlSecond=0 disables cache, ttlSecond=-1 means no expiration
    • *
  • capacity=0 disables cache; otherwise capacity is the count limit only when max-weight is absent
    • - *
  • when max-weight is present, Caffeine uses the weight limit instead of the positive capacity
    • + *
  • when max-weight is present, Caffeine uses the weight limit instead of the positive capacity; + * max-weight is an estimated retained-cache admission budget in approximate bytes, not an + * exact heap limit (see {@link MetaCacheWeightUtils})
    • * */ public final class CacheSpec { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java index 8e32a989588677..20c1ec1e5cefcb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheWeightUtils.java @@ -19,118 +19,48 @@ import org.apache.doris.datasource.NameMapping; -import java.lang.invoke.MethodHandle; -import java.lang.invoke.MethodHandles; -import java.lang.invoke.MethodType; -import java.lang.management.ManagementFactory; -import java.lang.management.PlatformManagedObject; -import java.lang.reflect.Field; -import java.lang.reflect.Method; -import java.lang.reflect.Modifier; -import java.util.Arrays; -import java.util.HashSet; -import java.util.Set; - -/** Overflow-safe helpers for conservative external metadata cache weights. */ +/** + * Overflow-safe helpers for approximate external metadata cache weights. + * + *

    Weights are byte-like units, not exact retained heap sizes: structural constants are + * rounded upward from offline calibration and do not model the exact layout of the active JVM + * or of third-party SDK classes. {@code max-weight} is therefore an estimated retained-cache + * admission budget rather than a precise heap limit; soft value references remain the safety + * net for residual under-estimation. + */ public final class MetaCacheWeightUtils { private static final long NAME_MAPPING_BASE_BYTES = 64L; - private static final MethodHandle STRING_VALUE_GETTER; - private static final long STRING_VALUE_OFFSET; - private static final int OBJECT_ALIGNMENT_BYTES; - private static final int OBJECT_REFERENCE_BYTES; - private static final int OBJECT_HEADER_BYTES; - private static final int OBJECT_ARRAY_BASE_BYTES; - private static final int BYTE_ARRAY_BASE_BYTES; - private static final int CHAR_ARRAY_BASE_BYTES; - private static final int INT_ARRAY_BASE_BYTES; - private static final boolean SUPPORTED_OBJECT_LAYOUT; - private static final long OBJECT_LAYOUT_PERCENT; - - static { - MethodHandle stringValueGetter = null; - long stringValueOffset = -1L; - int referenceBytes = Long.BYTES; - // 24B is the safe fallback for an uncompressed class pointer. Unsafe replaces these - // values with the exact active-VM layout when access is available. - int objectArrayBaseBytes = 24; - int byteArrayBaseBytes = 24; - int charArrayBaseBytes = 24; - int intArrayBaseBytes = 24; - try { - Class unsafeClass = Class.forName("sun.misc.Unsafe"); - Field unsafeField = unsafeClass.getDeclaredField("theUnsafe"); - unsafeField.setAccessible(true); - Object unsafe = unsafeField.get(null); - stringValueOffset = (long) unsafeClass - .getMethod("objectFieldOffset", Field.class) - .invoke(unsafe, String.class.getDeclaredField("value")); - stringValueGetter = MethodHandles.lookup() - .unreflect(unsafeClass.getMethod("getObject", Object.class, long.class)) - .bindTo(unsafe) - .asType(MethodType.methodType(Object.class, Object.class, long.class)); - referenceBytes = (int) unsafeClass - .getMethod("arrayIndexScale", Class.class) - .invoke(unsafe, Object[].class); - objectArrayBaseBytes = (int) unsafeClass - .getMethod("arrayBaseOffset", Class.class) - .invoke(unsafe, Object[].class); - byteArrayBaseBytes = (int) unsafeClass - .getMethod("arrayBaseOffset", Class.class) - .invoke(unsafe, byte[].class); - charArrayBaseBytes = (int) unsafeClass - .getMethod("arrayBaseOffset", Class.class) - .invoke(unsafe, char[].class); - intArrayBaseBytes = (int) unsafeClass - .getMethod("arrayBaseOffset", Class.class) - .invoke(unsafe, int[].class); - } catch (ReflectiveOperationException | RuntimeException ignored) { - // A conservative UTF-16 fallback is used when the VM hides String storage. - } - STRING_VALUE_GETTER = stringValueGetter; - STRING_VALUE_OFFSET = stringValueOffset; - OBJECT_REFERENCE_BYTES = referenceBytes; - OBJECT_ARRAY_BASE_BYTES = objectArrayBaseBytes; - BYTE_ARRAY_BASE_BYTES = byteArrayBaseBytes; - CHAR_ARRAY_BASE_BYTES = charArrayBaseBytes; - INT_ARRAY_BASE_BYTES = intArrayBaseBytes; - String alignmentOption = readVmOption("ObjectAlignmentInBytes"); - int objectAlignmentBytes = parseObjectAlignment(alignmentOption); - OBJECT_ALIGNMENT_BYTES = objectAlignmentBytes; - SUPPORTED_OBJECT_LAYOUT = alignmentOption != null - && (objectAlignmentBytes == 8 || objectAlignmentBytes == 16); - boolean compressedClassPointers = readBooleanVmOption( - "UseCompressedClassPointers", false); - OBJECT_HEADER_BYTES = Long.BYTES - + (compressedClassPointers ? Integer.BYTES : Long.BYTES); - long referencePercent = referenceBytes <= Integer.BYTES ? 100L : 145L; - long classPointerPercent = compressedClassPointers ? 100L : 140L; - long alignmentPercent = objectAlignmentBytes <= 8 ? 100L : 120L; - OBJECT_LAYOUT_PERCENT = (referencePercent * classPointerPercent * alignmentPercent - + 9_999L) / 10_000L; - } + // Rounded-up structural overheads shared by all estimators. + private static final long STRING_OBJECT_BYTES = 48L; + private static final long ARRAY_BASE_BYTES = 24L; + private static final long OBJECT_REFERENCE_BYTES = 8L; + private static final long HASH_MAP_OBJECT_BYTES = 64L; + private static final long HASH_MAP_ENTRY_BYTES = 64L; private MetaCacheWeightUtils() { } + /** + * Approximate retained bytes of a String: object plus backing array, one byte per Latin-1 + * character and two bytes otherwise. Determined by scanning the value once, so callers pay + * O(length) exactly like the loader that produced the string. + */ public static long estimatedStringBytes(String value) { if (value == null) { return 0L; } - Object storage = stringStorage(value); - long backingArrayBytes; - if (storage instanceof byte[]) { - backingArrayBytes = estimatedByteArrayBytes(((byte[]) storage).length); - } else if (storage instanceof char[]) { - backingArrayBytes = alignedArrayBytes( - CHAR_ARRAY_BASE_BYTES, ((char[]) storage).length, Character.BYTES); - } else { - backingArrayBytes = alignedArrayBytes( - CHAR_ARRAY_BASE_BYTES, value.length(), Character.BYTES); + return saturatedAdd(STRING_OBJECT_BYTES, estimatedStringPayloadBytes(value)); + } + + /** Character payload of a String without the object/array overhead. */ + public static long estimatedStringPayloadBytes(String value) { + if (value == null) { + return 0L; } - return saturatedAdd(estimatedObjectLayoutBytes(1L, 6L), backingArrayBytes); + return saturatedMultiply(value.length(), isLatin1(value) ? 1L : 2L); } - /** Estimate retained character data without materializing a String copy. */ + /** Approximate retained character data of any CharSequence. */ public static long estimatedCharSequenceBytes(CharSequence value) { if (value == null) { return 0L; @@ -138,108 +68,54 @@ public static long estimatedCharSequenceBytes(CharSequence value) { if (value instanceof String) { return estimatedStringBytes((String) value); } - return saturatedAdd(estimatedObjectLayoutBytes(1L, 6L), - alignedArrayBytes(CHAR_ARRAY_BASE_BYTES, value.length(), Character.BYTES)); - } - - /** Whether calibrated formulas support the active VM object alignment. */ - public static boolean isSupportedJvmObjectLayout() { - return SUPPORTED_OBJECT_LAYOUT; + return saturatedAdd(STRING_OBJECT_BYTES, + saturatedMultiply(value.length(), 2L)); } - /** Adjust a default compressed-reference object-graph constant to the active VM layout. */ - public static long estimatedObjectBytes(long compressedReferenceBytes) { - long product = saturatedMultiply(compressedReferenceBytes, OBJECT_LAYOUT_PERCENT); - if (product == Long.MAX_VALUE) { - return product; + /** Approximate size of a retained byte array. */ + public static long estimatedByteArrayBytes(long length) { + if (length < 0L) { + return Long.MAX_VALUE; } - long roundedProduct = saturatedAdd(product, 99L); - return roundedProduct == Long.MAX_VALUE ? roundedProduct : roundedProduct / 100L; + return saturatedAdd(ARRAY_BASE_BYTES, length); } - /** Returns the actual backing-array payload in O(1), or a conservative UTF-16 fallback. */ - public static long estimatedStringPayloadBytes(String value) { - if (value == null) { - return 0L; - } - Object storage = stringStorage(value); - if (storage instanceof byte[]) { - return alignPayload(((byte[]) storage).length); - } - if (storage instanceof char[]) { - return alignPayload(saturatedMultiply( - ((char[]) storage).length, Character.BYTES)); + /** Approximate size of an object-reference array. */ + public static long estimatedObjectArrayBytes(long length) { + if (length < 0L) { + return Long.MAX_VALUE; } - return alignPayload(saturatedMultiply(value.length(), Character.BYTES)); + return saturatedAdd(ARRAY_BASE_BYTES, + saturatedMultiply(length, OBJECT_REFERENCE_BYTES)); } - /** Estimate a generated String whose encoded width is derived from its source components. */ - public static long estimatedGeneratedStringBytes(long characterCount, boolean latin1) { - long payloadBytes = saturatedMultiply(characterCount, latin1 ? 1L : Character.BYTES); - return saturatedAdd(estimatedObjectLayoutBytes(1L, 6L), - estimatedByteArrayBytes(payloadBytes)); - } - - /** Whether this VM stores the String with one byte per character. */ - public static boolean isLatin1String(String value) { - if (value == null || value.isEmpty()) { - return true; + /** Incremental payload of an int array whose header is accounted elsewhere. */ + public static long estimatedIntArrayPayloadBytes(long length) { + if (length < 0L) { + return Long.MAX_VALUE; } - Object storage = stringStorage(value); - return storage instanceof byte[] && ((byte[]) storage).length == value.length(); - } - - /** VM-layout size of a retained byte array, conservatively if VM introspection is hidden. */ - public static long estimatedByteArrayBytes(long length) { - return alignedArrayBytes(BYTE_ARRAY_BASE_BYTES, length, Byte.BYTES); - } - - /** VM-layout size of an object-reference array, conservatively if introspection is hidden. */ - public static long estimatedObjectArrayBytes(long length) { - return alignedArrayBytes(OBJECT_ARRAY_BASE_BYTES, length, OBJECT_REFERENCE_BYTES); + return saturatedMultiply(length, Integer.BYTES); } /** - * A java.util.HashMap holding {@code entries} mappings: the map object, its power-of-two - * table (allocated on the first put) and one node per entry; keys and values are separate. + * Approximate size of a java.util.HashMap holding {@code entries} mappings: the map object, + * its table and one node per entry; keys and values are charged separately. */ public static long estimatedHashMapBytes(long entries) { - long bytes = estimatedObjectLayoutBytes(4L, 16L); if (entries <= 0L) { - return bytes; - } - long capacity = 16L; - while (entries > capacity - capacity / 4L) { - capacity = saturatedMultiply(capacity, 2L); - if (capacity == Long.MAX_VALUE) { - break; - } + return HASH_MAP_OBJECT_BYTES; } - bytes = saturatedAdd(bytes, estimatedObjectArrayBytes(capacity)); - return saturatedAdd(bytes, saturatedMultiply(entries, estimatedObjectLayoutBytes(3L, 4L))); + return saturatedAdd(HASH_MAP_OBJECT_BYTES, + saturatedMultiply(entries, HASH_MAP_ENTRY_BYTES)); } - /** Size of an object with a known field layout on the active VM. */ - public static long estimatedObjectLayoutBytes(long referenceFields, long primitiveBytes) { - if (referenceFields < 0L || primitiveBytes < 0L) { - return Long.MAX_VALUE; - } - long bytes = saturatedAdd( - OBJECT_HEADER_BYTES, - saturatedMultiply(referenceFields, OBJECT_REFERENCE_BYTES)); - return alignPayload(saturatedAdd(bytes, primitiveBytes)); - } - - /** VM-layout size of a retained int array, conservatively if introspection is hidden. */ - public static long estimatedIntArrayBytes(long length) { - return alignedArrayBytes(INT_ARRAY_BASE_BYTES, length, Integer.BYTES); - } - - /** Incremental VM-layout payload of an int array whose header is accounted elsewhere. */ - public static long estimatedIntArrayPayloadBytes(long length) { - long populated = alignedArrayBytes(INT_ARRAY_BASE_BYTES, length, Integer.BYTES); - long empty = alignPayload(INT_ARRAY_BASE_BYTES); - return populated == Long.MAX_VALUE ? populated : populated - empty; + /** + * Pass-through for rounded structural constants. Historic call sites scaled a + * compressed-reference constant to the active VM layout; the constants are now rounded up + * far enough to cover any supported layout, so no adjustment is applied. + */ + public static long estimatedObjectBytes(long approximateBytes) { + return approximateBytes < 0L ? Long.MAX_VALUE : approximateBytes; } /** Estimate the fixed set of names retained by a cache key. */ @@ -247,49 +123,13 @@ public static long estimatedNameMappingBytes(NameMapping nameMapping) { if (nameMapping == null) { return 0L; } - long bytes = estimatedObjectBytes(NAME_MAPPING_BASE_BYTES); + long bytes = NAME_MAPPING_BASE_BYTES; bytes = saturatedAdd(bytes, estimatedStringBytes(nameMapping.getLocalDbName())); bytes = saturatedAdd(bytes, estimatedStringBytes(nameMapping.getLocalTblName())); bytes = saturatedAdd(bytes, estimatedStringBytes(nameMapping.getRemoteDbName())); return saturatedAdd(bytes, estimatedStringBytes(nameMapping.getRemoteTblName())); } - /** - * Whether {@code type} itself declares exactly the expected non-static instance fields, each - * written as {@code name:SimpleTypeName}. Estimator formulas are calibrated against pinned SDK - * layouts; callers fail closed when a library upgrade adds, removes or retypes a field so a - * new retained reference cannot be silently undercounted. Superclasses are pinned separately. - */ - public static boolean hasExpectedInstanceFields(Class type, String... expectedFields) { - if (type == null) { - return false; - } - Set expected = new HashSet<>(Arrays.asList(expectedFields)); - Set actual = new HashSet<>(); - try { - for (Field field : type.getDeclaredFields()) { - if (Modifier.isStatic(field.getModifiers()) || field.isSynthetic()) { - continue; - } - actual.add(field.getName() + ":" + field.getType().getSimpleName()); - } - } catch (RuntimeException | LinkageError e) { - return false; - } - return actual.equals(expected); - } - - /** Same as {@link #hasExpectedInstanceFields(Class, String...)} for a class resolved by name. */ - public static boolean hasExpectedInstanceFields( - String className, ClassLoader loader, String... expectedFields) { - try { - return hasExpectedInstanceFields( - Class.forName(className, false, loader), expectedFields); - } catch (ReflectiveOperationException | RuntimeException | LinkageError e) { - return false; - } - } - public static long saturatedAdd(long left, long right) { if (left < 0L || right < 0L || Long.MAX_VALUE - left < right) { return Long.MAX_VALUE; @@ -304,65 +144,12 @@ public static long saturatedMultiply(long left, long right) { return left * right; } - private static boolean readBooleanVmOption(String option, boolean fallback) { - String value = readVmOption(option); - return value == null ? fallback : Boolean.parseBoolean(value); - } - - private static String readVmOption(String option) { - try { - @SuppressWarnings("unchecked") - Class beanClass = - (Class) - Class.forName("com.sun.management.HotSpotDiagnosticMXBean"); - Object bean = ManagementFactory.getPlatformMXBean(beanClass); - Method getVmOption = beanClass.getMethod("getVMOption", String.class); - Object vmOption = getVmOption.invoke(bean, option); - Method getValue = vmOption.getClass().getMethod("getValue"); - return (String) getValue.invoke(vmOption); - } catch (ReflectiveOperationException | RuntimeException ignored) { - return null; - } - } - - private static long alignPayload(long bytes) { - if (bytes == Long.MAX_VALUE) { - return bytes; - } - long remainder = bytes % OBJECT_ALIGNMENT_BYTES; - return remainder == 0L ? bytes - : saturatedAdd(bytes, OBJECT_ALIGNMENT_BYTES - remainder); - } - - private static long alignedArrayBytes(long baseBytes, long length, long elementBytes) { - if (length < 0L) { - return Long.MAX_VALUE; - } - return alignPayload(saturatedAdd( - baseBytes, saturatedMultiply(length, elementBytes))); - } - - private static Object stringStorage(String value) { - if (STRING_VALUE_GETTER != null && STRING_VALUE_OFFSET >= 0L) { - try { - return (Object) STRING_VALUE_GETTER.invokeExact( - (Object) value, STRING_VALUE_OFFSET); - } catch (Throwable ignored) { - // Return null so callers use the conservative UTF-16 fallback. + private static boolean isLatin1(String value) { + for (int i = 0; i < value.length(); i++) { + if (value.charAt(i) > 0xFF) { + return false; } } - return null; - } - - private static int parseObjectAlignment(String value) { - if (value == null) { - return 16; - } - try { - int alignment = Integer.parseInt(value); - return alignment > 0 ? alignment : 16; - } catch (NumberFormatException ignored) { - return 16; - } + return true; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java index 4011e6a797b77e..6792b5a362a6bc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java @@ -21,194 +21,62 @@ import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; -import com.google.common.collect.ImmutableMap; -import org.apache.paimon.privilege.PrivilegedFileStoreTable; import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.DelegatedFileStoreTable; import org.apache.paimon.table.FallbackReadFileStoreTable; import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; -import org.apache.paimon.types.ArrayType; -import org.apache.paimon.types.BigIntType; -import org.apache.paimon.types.BinaryType; -import org.apache.paimon.types.BlobType; -import org.apache.paimon.types.BooleanType; -import org.apache.paimon.types.CharType; import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataType; -import org.apache.paimon.types.DateType; -import org.apache.paimon.types.DecimalType; -import org.apache.paimon.types.DoubleType; -import org.apache.paimon.types.FloatType; -import org.apache.paimon.types.IntType; -import org.apache.paimon.types.LocalZonedTimestampType; -import org.apache.paimon.types.MapType; -import org.apache.paimon.types.MultisetType; import org.apache.paimon.types.RowType; -import org.apache.paimon.types.SmallIntType; -import org.apache.paimon.types.TimeType; -import org.apache.paimon.types.TimestampType; -import org.apache.paimon.types.TinyIntType; -import org.apache.paimon.types.VarBinaryType; -import org.apache.paimon.types.VarCharType; -import org.apache.paimon.types.VariantType; -import org.apache.paimon.types.VectorType; import java.util.List; -import java.util.Locale; import java.util.Map; -/** Publication-time retained-weight formulas for Paimon table handles and snapshot projections. */ +/** + * Publication-time approximate weight formulas for Paimon table handles and snapshot projections. + * + *

    The formulas follow a coarse cardinality model: a rounded-up constant per stable logical + * dimension (schema field, logical type node, option, key, partition, ...) plus the + * skew-sensitive string payload the loader already materialized. The per-node constants absorb + * the lazy state Paimon materializes after admission (the four RowType lookup maps, the store + * graph and its derived RowType copies) instead of modeling those objects individually, so + * weights track metadata size without depending on SDK-private layouts. {@code max-weight} is + * an estimated admission budget, not an exact heap limit. Estimation never opens the table + * store and performs no IO. + */ final class PaimonCacheSizeEstimator { - // Calibrated against JOL retained-graph deltas in PaimonExternalMetaCacheTest. + // Bounds accounting work per publication; far above real schemas and option maps. private static final long MAX_TABLE_ACCOUNTING_ELEMENTS = 50_000L; - private static final int MAX_TYPE_ACCOUNTING_DEPTH = 128; - private static final long KEY_BASE_BYTES = objectBytes(128L); - private static final long SNAPSHOT_BASE_BYTES = objectBytes(4L * 1024L); - private static final long TABLE_BASE_BYTES = objectBytes(16L * 1024L); - // PaimonTableCacheValue: table ref, generation, payload bytes, estimate ref (+ estimate object). - private static final long TABLE_VALUE_BASE_BYTES = objectBytes(96L); - // A top-level DataField, its list slot and shared per-field overhead; the DataType instance - // is accounted separately by addTypePayload. - private static final long TABLE_FIELD_BYTES = objectBytes(40L); - private static final long TABLE_OPTION_BYTES = objectBytes(44L); - private static final long TABLE_KEY_BYTES = objectBytes(128L); - // Exact Paimon 1.4.2 layouts, pinned by PAIMON_TYPE_LAYOUT_SUPPORTED. - private static final long DATA_FIELD_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(4L, 4L); - private static final long ARRAY_TYPE_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 1L); - private static final long VECTOR_TYPE_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 5L); - private static final long MAP_TYPE_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 1L); - private static final long MULTISET_TYPE_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 1L); - // RowType plus Collections.unmodifiableList(new ArrayList<>(fields)). - private static final long ROW_TYPE_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(6L, 1L); - private static final long UNMODIFIABLE_LIST_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(2L, 0L); - private static final long ARRAY_LIST_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(1L, 8L); - private static final long HASH_MAP_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(4L, 16L); - private static final long HASH_MAP_NODE_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(3L, 4L); - private static final long INTEGER_BYTES = - MetaCacheWeightUtils.estimatedObjectLayoutBytes(0L, 4L); - private static final int ROW_TYPE_LAZY_MAP_COUNT = 4; - // Accepted leaf DataType implementations and the int fields each adds to DataType's nullable - // flag and type root. Any other class, including a future or third-party type, rejects - // weighted admission instead of being counted as an arbitrary primitive. - private static final String[] NO_LEAF_FIELDS = {}; - private static final String[] LENGTH_LEAF_FIELDS = {"length:int"}; - private static final String[] PRECISION_LEAF_FIELDS = {"precision:int"}; - private static final Map, String[]> LEAF_TYPE_FIELDS = - ImmutableMap., String[]>builder() - .put(CharType.class, LENGTH_LEAF_FIELDS) - .put(VarCharType.class, LENGTH_LEAF_FIELDS) - .put(BooleanType.class, NO_LEAF_FIELDS) - .put(BinaryType.class, LENGTH_LEAF_FIELDS) - .put(VarBinaryType.class, LENGTH_LEAF_FIELDS) - .put(DecimalType.class, new String[] {"precision:int", "scale:int"}) - .put(TinyIntType.class, NO_LEAF_FIELDS) - .put(SmallIntType.class, NO_LEAF_FIELDS) - .put(IntType.class, NO_LEAF_FIELDS) - .put(BigIntType.class, NO_LEAF_FIELDS) - .put(FloatType.class, NO_LEAF_FIELDS) - .put(DoubleType.class, NO_LEAF_FIELDS) - .put(DateType.class, NO_LEAF_FIELDS) - .put(TimeType.class, PRECISION_LEAF_FIELDS) - .put(TimestampType.class, PRECISION_LEAF_FIELDS) - .put(LocalZonedTimestampType.class, PRECISION_LEAF_FIELDS) - .put(VariantType.class, NO_LEAF_FIELDS) - .put(BlobType.class, NO_LEAF_FIELDS) - .build(); - private static final boolean PAIMON_TYPE_LAYOUT_SUPPORTED = checkPaimonTypeLayout(); - private static final boolean PAIMON_TABLE_LAYOUT_SUPPORTED = checkPaimonTableLayout(); - // One Paimon Partition record with its single-column LinkedHashMap spec plus map entry; extra - // columns are charged by PaimonPartitionInfo. - // FileStoreTable.lazyStore: the store object, its CoreOptions/Options copy, SchemaManager, - // and the partition/bucket-key/row/key/value RowTypes it derives from the TableSchema. It is - // created by the partition projection before publication or by scan planning afterwards. - private static final long STORE_BASE_BYTES = objectBytes(1_536L); - private static final long STORE_OPTION_BYTES = objectBytes(48L); - // KeyValueFileStore keeps prefixed key-field copies and shares the value fields; the - // AppendOnlyFileStore deep copy of the whole type tree is reserved by - // retainedTablePayloadBytes, which already walks that tree. - private static final long STORE_KEY_FIELD_BYTES = objectBytes(112L); - private static final long STORE_LIST_SLOT_BYTES = objectBytes(8L); - private static final String APPEND_ONLY_TABLE_CLASS_NAME = - "org.apache.paimon.table.AppendOnlyFileStoreTable"; - private static final String PRIMARY_KEY_TABLE_CLASS_NAME = - "org.apache.paimon.table.PrimaryKeyFileStoreTable"; - private static final String MERGE_ENGINE_OPTION = "merge-engine"; - private static final long PARTITION_BYTES = objectBytes(272L); - private static final long PARTITION_ITEM_BYTES = objectBytes(640L); - private static final long WRAPPER_BYTES = objectBytes(512L); + private static final int MAX_TYPE_ACCOUNTING_DEPTH = 256; + + private static final long KEY_BASE_WEIGHT = 256L; + private static final long SNAPSHOT_BASE_WEIGHT = 4L * 1024L; + private static final long TABLE_BASE_WEIGHT = 16L * 1024L; + // PaimonTableCacheValue and its size-estimate holder. + private static final long TABLE_VALUE_BASE_WEIGHT = 128L; + // One schema field (DataField) at any depth, including its share of the enclosing RowType's + // four lazily built lookup maps, boxed ids and the field copies the lazily created store + // graph derives from it (append-only row copy, trimmed key/value types, merge row type). + private static final long FIELD_NODE_WEIGHT = 576L; + // One bare nested type node (array/map/multiset/vector element types and unknown future + // DataType implementations), including its store-graph copies; charged generically instead + // of disabling weighted caching for unknown types. + private static final long TYPE_NODE_WEIGHT = 128L; + // One nested RowType container: the RowType, its field list, its four lazily built lookup + // maps and the container copies the store graph derives. + private static final long ROW_CONTAINER_WEIGHT = 2048L; + // One schema/table-level entry: option map node, key list slot and boxes. + private static final long OPTION_WEIGHT = 192L; + private static final long KEY_WEIGHT = 192L; + // Wrapper tables (privileged, fallback-read) around the concrete FileStoreTable. + private static final long WRAPPER_WEIGHT = 512L; + private static final long PARTITION_WEIGHT = 320L; + private static final long PARTITION_ITEM_WEIGHT = 768L; private PaimonCacheSizeEstimator() { } - private static long objectBytes(long bytes) { - return MetaCacheWeightUtils.estimatedObjectBytes(bytes); - } - - /** DataType: typeRoot reference plus the isNullable flag, then the subclass int fields. */ - private static long leafTypeBytes(String[] intFields) { - return MetaCacheWeightUtils.estimatedObjectLayoutBytes( - 1L, 1L + (long) Integer.BYTES * intFields.length); - } - - /** Pin the Paimon 1.4.2 DataType/DataField/RowType layouts the formulas above are built on. */ - private static boolean checkPaimonTypeLayout() { - boolean supported = MetaCacheWeightUtils.hasExpectedInstanceFields( - DataType.class, "isNullable:boolean", "typeRoot:DataTypeRoot") - && MetaCacheWeightUtils.hasExpectedInstanceFields( - DataField.class, "id:int", "name:String", "type:DataType", - "description:String", "defaultValue:String") - && MetaCacheWeightUtils.hasExpectedInstanceFields( - RowType.class, "fields:List", "laziedNameToField:Map", - "laziedNameToIndex:Map", "laziedFieldIdToField:Map", - "laziedFieldIdToIndex:Map") - && MetaCacheWeightUtils.hasExpectedInstanceFields( - ArrayType.class, "elementType:DataType") - && MetaCacheWeightUtils.hasExpectedInstanceFields( - VectorType.class, "elementType:DataType", "length:int") - && MetaCacheWeightUtils.hasExpectedInstanceFields( - MapType.class, "keyType:DataType", "valueType:DataType") - && MetaCacheWeightUtils.hasExpectedInstanceFields( - MultisetType.class, "elementType:DataType"); - for (Map.Entry, String[]> leaf : LEAF_TYPE_FIELDS.entrySet()) { - supported &= MetaCacheWeightUtils.hasExpectedInstanceFields( - leaf.getKey(), leaf.getValue()); - } - return supported; - } - - /** Pin TableSchema and the two accepted FileStoreTable implementations. */ - private static boolean checkPaimonTableLayout() { - ClassLoader loader = FileStoreTable.class.getClassLoader(); - String[] abstractTableFields = { - "fileIO:FileIO", "path:Path", "tableSchema:TableSchema", - "catalogEnvironment:CatalogEnvironment", "manifestCache:SegmentsCache", - "snapshotCache:Cache", "statsCache:Cache", "dvmetaCache:DVMetaCache"}; - return MetaCacheWeightUtils.hasExpectedInstanceFields( - TableSchema.class, "version:int", "id:long", "fields:List", - "highestFieldId:int", "partitionKeys:List", "primaryKeys:List", - "bucketKeys:List", "numBucket:int", "options:Map", "comment:String", - "timeMillis:long") - && MetaCacheWeightUtils.hasExpectedInstanceFields( - "org.apache.paimon.table.AbstractFileStoreTable", loader, - abstractTableFields) - && MetaCacheWeightUtils.hasExpectedInstanceFields( - "org.apache.paimon.table.AppendOnlyFileStoreTable", loader, - "lazyStore:AppendOnlyFileStore") - && MetaCacheWeightUtils.hasExpectedInstanceFields( - "org.apache.paimon.table.PrimaryKeyFileStoreTable", loader, - "lazyStore:KeyValueFileStore"); - } - /** * Retained weight of the base table entry. The table handle is owned independently of the * snapshot projections that reference it (they may pin an older generation), so the same @@ -220,26 +88,13 @@ static MetaCacheSizeEstimate estimateTableEntry(NameMapping key, PaimonTableCach return MetaCacheSizeEstimate.incomplete(unsupported); } long bytes = MetaCacheWeightUtils.saturatedAdd( - KEY_BASE_BYTES, MetaCacheWeightUtils.estimatedNameMappingBytes(key)); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, TABLE_VALUE_BASE_BYTES); + KEY_BASE_WEIGHT, MetaCacheWeightUtils.estimatedNameMappingBytes(key)); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, TABLE_VALUE_BASE_WEIGHT); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedTablePayloadBytes()); return MetaCacheSizeEstimate.complete( MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(value.getPaimonTable()))); } - private static String unsupportedReason(Table table) { - if (!MetaCacheWeightUtils.isSupportedJvmObjectLayout()) { - return "unsupported_jvm_object_alignment"; - } - if (!PAIMON_TYPE_LAYOUT_SUPPORTED || !PAIMON_TABLE_LAYOUT_SUPPORTED) { - return "unsupported_paimon_layout"; - } - if (!isSupportedTable(table)) { - return "unsupported_paimon_table:" + (table == null ? "null" : table.getClass().getName()); - } - return null; - } - static MetaCacheSizeEstimate estimateSnapshotEntry( PaimonSnapshotEntryKey key, PaimonSnapshotCacheValue value) { Table table = value.getSnapshot().getTable(); @@ -247,12 +102,12 @@ static MetaCacheSizeEstimate estimateSnapshotEntry( if (unsupported != null) { return MetaCacheSizeEstimate.incomplete(unsupported); } - long bytes = MetaCacheWeightUtils.saturatedAdd( - KEY_BASE_BYTES, MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping())); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SNAPSHOT_BASE_BYTES); - bytes = addCount(bytes, value.getPartitionInfo().getNameToPartition().size(), PARTITION_BYTES); - bytes = addCount(bytes, value.getPartitionInfo().getNameToPartitionItem().size(), PARTITION_ITEM_BYTES); + KEY_BASE_WEIGHT, MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SNAPSHOT_BASE_WEIGHT); + bytes = addCount(bytes, value.getPartitionInfo().getNameToPartition().size(), PARTITION_WEIGHT); + bytes = addCount(bytes, + value.getPartitionInfo().getNameToPartitionItem().size(), PARTITION_ITEM_WEIGHT); bytes = MetaCacheWeightUtils.saturatedAdd( bytes, value.getPartitionInfo().getRetainedPayloadBytes()); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedTablePayloadBytes()); @@ -260,114 +115,126 @@ static MetaCacheSizeEstimate estimateSnapshotEntry( MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table))); } - private static boolean isSupportedTable(Table table) { - if (table instanceof PrivilegedFileStoreTable) { - return isSupportedTable(((PrivilegedFileStoreTable) table).wrapped()); + private static String unsupportedReason(Table table) { + if (table == null) { + return "unsupported_paimon_table:null"; } - if (table instanceof FallbackReadFileStoreTable) { - FallbackReadFileStoreTable fallback = (FallbackReadFileStoreTable) table; - return isSupportedTable(fallback.wrapped()) && isSupportedTable(fallback.other()); + if (unwrap(table) == null) { + return "unsupported_paimon_table:" + table.getClass().getName(); } - if (!(table instanceof FileStoreTable)) { - return false; + return null; + } + + /** The concrete FileStoreTable behind any known wrapper chain, or null. */ + private static FileStoreTable unwrap(Table table) { + if (table instanceof DelegatedFileStoreTable) { + return unwrap(((DelegatedFileStoreTable) table).wrapped()); } - String className = table.getClass().getName(); - return APPEND_ONLY_TABLE_CLASS_NAME.equals(className) - || PRIMARY_KEY_TABLE_CLASS_NAME.equals(className); + return table instanceof FileStoreTable ? (FileStoreTable) table : null; } /** Uses TableSchema cardinalities only and deliberately never calls FileStoreTable.store(). */ private static long estimateTable(Table table) { - if (table instanceof PrivilegedFileStoreTable) { - return MetaCacheWeightUtils.saturatedAdd(WRAPPER_BYTES, - estimateTable(((PrivilegedFileStoreTable) table).wrapped())); + long bytes = 0L; + Table current = table; + while (true) { + if (current instanceof FallbackReadFileStoreTable) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, WRAPPER_WEIGHT); + bytes = MetaCacheWeightUtils.saturatedAdd( + bytes, estimateTable(((FallbackReadFileStoreTable) current).other())); + current = ((FallbackReadFileStoreTable) current).wrapped(); + continue; + } + if (current instanceof DelegatedFileStoreTable) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, WRAPPER_WEIGHT); + current = ((DelegatedFileStoreTable) current).wrapped(); + continue; + } + break; } - if (table instanceof FallbackReadFileStoreTable) { - FallbackReadFileStoreTable fallback = (FallbackReadFileStoreTable) table; - long bytes = MetaCacheWeightUtils.saturatedAdd(WRAPPER_BYTES, estimateTable(fallback.wrapped())); - return MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(fallback.other())); + if (!(current instanceof FileStoreTable)) { + return bytes; } - - FileStoreTable fileStoreTable = (FileStoreTable) table; + FileStoreTable fileStoreTable = (FileStoreTable) current; TableSchema schema = fileStoreTable.schema(); - long bytes = TABLE_BASE_BYTES; + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, TABLE_BASE_WEIGHT); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, - MetaCacheWeightUtils.estimatedStringBytes(table.name())); + MetaCacheWeightUtils.estimatedStringBytes(current.name())); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.estimatedStringBytes(fileStoreTable.location().toString())); - bytes = addCount(bytes, schema.fields().size(), TABLE_FIELD_BYTES); - bytes = addCount(bytes, schema.options().size(), TABLE_OPTION_BYTES); - bytes = addCount(bytes, schema.partitionKeys().size(), TABLE_KEY_BYTES); - bytes = addCount(bytes, schema.primaryKeys().size(), TABLE_KEY_BYTES); - bytes = addCount(bytes, schema.bucketKeys().size(), TABLE_KEY_BYTES); - return MetaCacheWeightUtils.saturatedAdd(bytes, storeGraphBytes(fileStoreTable, schema)); + // The store graph the table lazily materializes derives several RowType copies of the + // schema; the per-node constants absorb those copies instead of modeling store classes. + NodeCounts nodes = new NodeCounts(); + countFieldNodes(schema.fields(), 0, nodes); + bytes = addCount(bytes, nodes.fieldNodes, FIELD_NODE_WEIGHT); + bytes = addCount(bytes, nodes.bareTypeNodes, TYPE_NODE_WEIGHT); + bytes = addCount(bytes, nodes.rowContainers, ROW_CONTAINER_WEIGHT); + bytes = addCount(bytes, schema.options().size(), OPTION_WEIGHT); + bytes = addCount(bytes, schema.partitionKeys().size(), KEY_WEIGHT); + bytes = addCount(bytes, schema.primaryKeys().size(), KEY_WEIGHT); + bytes = addCount(bytes, schema.bucketKeys().size(), KEY_WEIGHT); + for (String primaryKey : schema.primaryKeys()) { + // Each trimmed key field of a primary-key store gets a fresh "_KEY_" + name string. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedStringBytes(primaryKey)); + } + return bytes; } - /** - * Reserve the store graph the table materializes without opening it: TableSchema - * cardinalities decide its size, and every RowType it derives can grow the four lazy lookup - * maps after admission exactly like nested RowTypes. - */ - private static long storeGraphBytes(FileStoreTable table, TableSchema schema) { - long bytes = STORE_BASE_BYTES; - bytes = addCount(bytes, schema.options().size(), STORE_OPTION_BYTES); - List fields = schema.fields(); - long fieldCount = fields.size(); - long uncachedFieldIds = 0L; - for (DataField field : fields) { - if (isUncachedInteger(field.id())) { - uncachedFieldIds++; - } + private static final class NodeCounts { + private long fieldNodes; + private long bareTypeNodes; + private long rowContainers; + } + + private static void countFieldNodes(List fields, int depth, NodeCounts counts) { + if (depth > MAX_TYPE_ACCOUNTING_DEPTH) { + throw new IllegalStateException("Paimon schema type nesting is too deep"); } - long partitionKeys = schema.partitionKeys().size(); - long bucketKeys = schema.bucketKeys().size(); - // Partition and bucket key RowTypes reference a subset of the fields; ids beyond the - // Integer cache are counted as if all of them were uncached, which is conservative. - long uncachedPartitionKeys = Math.min(partitionKeys, uncachedFieldIds); - long uncachedBucketKeys = Math.min(bucketKeys, uncachedFieldIds); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, rowTypeBytes(partitionKeys, uncachedPartitionKeys)); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, rowTypeBytes(bucketKeys, uncachedBucketKeys)); - if (PRIMARY_KEY_TABLE_CLASS_NAME.equals(table.getClass().getName())) { - long primaryKeys = schema.primaryKeys().size(); - bytes = addCount(bytes, primaryKeys, STORE_KEY_FIELD_BYTES); - for (String primaryKey : schema.primaryKeys()) { - // Each trimmed key field gets a fresh "_KEY_" + name string. - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, - MetaCacheWeightUtils.estimatedStringBytes(primaryKey)); - } - bytes = addCount(bytes, fieldCount, STORE_LIST_SLOT_BYTES); - // Key fields are re-numbered above the Integer cache; the value type shares fields. - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, rowTypeBytes(primaryKeys, primaryKeys)); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, rowTypeBytes(fieldCount, uncachedFieldIds)); - if (retainsMergeFunctionRowType(schema)) { - // partial-update / aggregation merge factories keep a second logical RowType and - // option-derived per-field maps (aggregation also copies CoreOptions). - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, rowTypeBytes(fieldCount, uncachedFieldIds)); - bytes = addCount(bytes, schema.options().size(), STORE_OPTION_BYTES); - } - return bytes; + for (DataField field : fields) { + counts.fieldNodes = MetaCacheWeightUtils.saturatedAdd(counts.fieldNodes, 1L); + countTypeNodes(field.type(), depth, counts); } - // The copied row type of an append-only store (fields, types and nested lookup maps) is - // reserved by retainedTablePayloadBytes; the top-level RowType wrapper is charged here. - return MetaCacheWeightUtils.saturatedAdd(bytes, rowTypeBytes(fieldCount, uncachedFieldIds)); } - private static boolean retainsMergeFunctionRowType(TableSchema schema) { - String mergeEngine = schema.options().get(MERGE_ENGINE_OPTION); - if (mergeEngine == null) { - return false; + private static void countTypeNodes(DataType type, int depth, NodeCounts counts) { + if (depth > MAX_TYPE_ACCOUNTING_DEPTH) { + throw new IllegalStateException("Paimon schema type nesting is too deep"); + } + if (type instanceof RowType) { + counts.rowContainers = MetaCacheWeightUtils.saturatedAdd(counts.rowContainers, 1L); + countFieldNodes(((RowType) type).getFields(), depth + 1, counts); + return; + } + if (type != null) { + // Known container types (array, map, multiset) contribute their children as nodes; + // other types, including future implementations, are charged as a single node. + for (DataType child : childTypes(type)) { + counts.bareTypeNodes = MetaCacheWeightUtils.saturatedAdd(counts.bareTypeNodes, 1L); + countTypeNodes(child, depth + 1, counts); + } } - String normalized = mergeEngine.trim().toLowerCase(Locale.ROOT).replace('_', '-'); - return "partial-update".equals(normalized) || "aggregation".equals(normalized); } - private static boolean isUncachedInteger(int value) { - return value < -128 || value > 127; + private static List childTypes(DataType type) { + if (type instanceof org.apache.paimon.types.ArrayType) { + return java.util.Collections.singletonList( + ((org.apache.paimon.types.ArrayType) type).getElementType()); + } + if (type instanceof org.apache.paimon.types.MultisetType) { + return java.util.Collections.singletonList( + ((org.apache.paimon.types.MultisetType) type).getElementType()); + } + if (type instanceof org.apache.paimon.types.MapType) { + org.apache.paimon.types.MapType mapType = (org.apache.paimon.types.MapType) type; + return java.util.Arrays.asList(mapType.getKeyType(), mapType.getValueType()); + } + return java.util.Collections.emptyList(); } /** - * Captures skew-sensitive schema text once when the snapshot cache value is constructed. - * All collections are already materialized in TableSchema; this never opens the table store. + * Captures skew-sensitive schema text once when the cache value is constructed. All + * collections are already materialized in TableSchema; this never opens the table store. */ static long retainedTablePayloadBytes(Table table) { return retainedTablePayloadBytes( @@ -376,34 +243,26 @@ static long retainedTablePayloadBytes(Table table) { private static long retainedTablePayloadBytes(Table table, AccountingBudget budget) { budget.charge(1L); - if (table instanceof PrivilegedFileStoreTable) { - return retainedTablePayloadBytes( - ((PrivilegedFileStoreTable) table).wrapped(), budget); - } if (table instanceof FallbackReadFileStoreTable) { FallbackReadFileStoreTable fallback = (FallbackReadFileStoreTable) table; return MetaCacheWeightUtils.saturatedAdd( retainedTablePayloadBytes(fallback.wrapped(), budget), retainedTablePayloadBytes(fallback.other(), budget)); } + if (table instanceof DelegatedFileStoreTable) { + return retainedTablePayloadBytes( + ((DelegatedFileStoreTable) table).wrapped(), budget); + } if (!(table instanceof FileStoreTable)) { return 0L; } - TableSchema schema = ((FileStoreTable) table).schema(); if (schema == null) { return 0L; } long bytes = addString(0L, schema.comment()); - TypeTreeStructure structure = new TypeTreeStructure(); for (DataField field : schema.fields()) { - bytes = addFieldPayload(bytes, field, false, budget, 0, structure); - } - if (isAppendOnlyTable(table)) { - // AppendOnlyFileStore keeps logicalRowType().notNull(): a deep copy of every field - // and type (names and descriptions are shared), including nested RowTypes with their - // own lazy lookup maps. Reserve that copy without creating the store. - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, structure.bytes); + bytes = addFieldPayload(bytes, field, budget, 0); } budget.charge(schema.options().size()); for (Map.Entry option : schema.options().entrySet()) { @@ -415,153 +274,49 @@ private static long retainedTablePayloadBytes(Table table, AccountingBudget budg return addStrings(bytes, schema.bucketKeys(), budget); } - private static long addStrings( - long bytes, List values, AccountingBudget budget) { - budget.charge(values.size()); - for (String value : values) { - bytes = addString(bytes, value); - } - return bytes; - } - private static long addFieldPayload( - long bytes, DataField field, boolean nested, AccountingBudget budget, - int typeDepth, TypeTreeStructure structure) { - budget.charge(1L); - // Top-level DataFields are covered by TABLE_FIELD_BYTES; a copied tree owns them all. - structure.add(DATA_FIELD_BYTES); - if (nested) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, DATA_FIELD_BYTES); + long bytes, DataField field, AccountingBudget budget, int depth) { + if (depth > MAX_TYPE_ACCOUNTING_DEPTH) { + throw new IllegalStateException("Paimon schema type nesting is too deep"); } + budget.charge(1L); bytes = addString(bytes, field.name()); bytes = addString(bytes, field.description()); - bytes = addString(bytes, field.defaultValue()); - return addTypePayload(bytes, field.type(), budget, typeDepth, structure); - } - - /** - * Account one DataType instance and its owned children. Every accepted implementation is - * matched explicitly; an unknown class throws so estimateSafely rejects weighted admission - * instead of counting a future composite type as a small primitive. - */ - private static long addTypePayload( - long bytes, DataType type, AccountingBudget budget, int typeDepth, - TypeTreeStructure structure) { - if (typeDepth > MAX_TYPE_ACCOUNTING_DEPTH) { - throw new IllegalStateException( - "Paimon cache accounting type depth exceeded"); - } - budget.charge(1L); - if (type == null) { - throw new IllegalStateException("Paimon field type is missing"); - } - Class typeClass = type.getClass(); - if (typeClass == RowType.class) { - RowType rowType = (RowType) type; - List fields = rowType.getFields(); - long rowBytes = rowTypeBytes(fields); - structure.add(rowBytes); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, rowBytes); - for (DataField field : fields) { - bytes = addFieldPayload(bytes, field, true, budget, typeDepth + 1, structure); + DataType type = field.type(); + if (type instanceof RowType) { + for (DataField nested : ((RowType) type).getFields()) { + bytes = addFieldPayload(bytes, nested, budget, depth + 1); } - return bytes; - } - if (typeClass == ArrayType.class) { - structure.add(ARRAY_TYPE_BYTES); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ARRAY_TYPE_BYTES); - return addTypePayload( - bytes, ((ArrayType) type).getElementType(), budget, typeDepth + 1, structure); - } - if (typeClass == VectorType.class) { - structure.add(VECTOR_TYPE_BYTES); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, VECTOR_TYPE_BYTES); - return addTypePayload( - bytes, ((VectorType) type).getElementType(), budget, typeDepth + 1, structure); - } - if (typeClass == MapType.class) { - structure.add(MAP_TYPE_BYTES); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MAP_TYPE_BYTES); - bytes = addTypePayload( - bytes, ((MapType) type).getKeyType(), budget, typeDepth + 1, structure); - return addTypePayload( - bytes, ((MapType) type).getValueType(), budget, typeDepth + 1, structure); - } - if (typeClass == MultisetType.class) { - structure.add(MULTISET_TYPE_BYTES); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MULTISET_TYPE_BYTES); - // MultisetType.copy() shares its element type, so a deep copy stops here. - return addTypePayload(bytes, ((MultisetType) type).getElementType(), budget, - typeDepth + 1, new TypeTreeStructure()); - } - String[] leafFields = LEAF_TYPE_FIELDS.get(typeClass); - if (leafFields == null) { - throw new IllegalStateException( - "Unsupported Paimon data type: " + typeClass.getName()); - } - long leafBytes = leafTypeBytes(leafFields); - structure.add(leafBytes); - return MetaCacheWeightUtils.saturatedAdd(bytes, leafBytes); - } - - private static boolean isAppendOnlyTable(Table table) { - return APPEND_ONLY_TABLE_CLASS_NAME.equals(table.getClass().getName()); - } - - /** Non-String bytes of a schema type tree, i.e. what a deep DataType copy allocates again. */ - private static final class TypeTreeStructure { - private long bytes; - - private void add(long structuralBytes) { - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, structuralBytes); - } - } - - /** - * RowType, its unmodifiable ArrayList copy of the fields, and the four lazy lookup maps that - * getField/getFieldIndex materialize after admission. The maps are reserved up front in O(N) - * so a query cannot grow the retained graph past the admitted weight; nothing is materialized. - */ - private static long rowTypeBytes(List fields) { - long uncachedFieldIds = 0L; - for (DataField field : fields) { - if (isUncachedInteger(field.id())) { - uncachedFieldIds++; + } else if (type != null) { + for (DataType child : childTypes(type)) { + bytes = addChildPayload(bytes, child, budget, depth + 1); } } - return rowTypeBytes(fields.size(), uncachedFieldIds); + return bytes; } - private static long rowTypeBytes(long fieldCount, long uncachedFieldIds) { - long bytes = MetaCacheWeightUtils.saturatedAdd(ROW_TYPE_BYTES, UNMODIFIABLE_LIST_BYTES); - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, ARRAY_LIST_BYTES); - if (fieldCount == 0L) { + private static long addChildPayload( + long bytes, DataType type, AccountingBudget budget, int depth) { + if (type instanceof RowType) { + for (DataField nested : ((RowType) type).getFields()) { + bytes = addFieldPayload(bytes, nested, budget, depth + 1); + } return bytes; } - bytes = MetaCacheWeightUtils.saturatedAdd( - bytes, MetaCacheWeightUtils.estimatedObjectArrayBytes(fieldCount)); - long uncachedIndexes = fieldCount > 128L ? fieldCount - 128L : 0L; - long mapBytes = MetaCacheWeightUtils.saturatedAdd(HASH_MAP_BYTES, - MetaCacheWeightUtils.estimatedObjectArrayBytes(hashMapCapacity(fieldCount))); - mapBytes = addCount(mapBytes, fieldCount, HASH_MAP_NODE_BYTES); - bytes = addCount(bytes, ROW_TYPE_LAZY_MAP_COUNT, mapBytes); - // Boxed keys/values outside the Integer cache: nameToIndex values, fieldIdToField keys, - // and fieldIdToIndex boxes both again. - bytes = addCount(bytes, uncachedIndexes, INTEGER_BYTES); - bytes = addCount(bytes, uncachedFieldIds, INTEGER_BYTES); - bytes = addCount(bytes, uncachedIndexes, INTEGER_BYTES); - return addCount(bytes, uncachedFieldIds, INTEGER_BYTES); + if (type != null) { + for (DataType child : childTypes(type)) { + bytes = addChildPayload(bytes, child, budget, depth + 1); + } + } + return bytes; } - private static long hashMapCapacity(long size) { - long capacity = 16L; - while (size > capacity - capacity / 4L) { - capacity = MetaCacheWeightUtils.saturatedMultiply(capacity, 2L); - if (capacity == Long.MAX_VALUE) { - return capacity; - } + private static long addStrings(long bytes, List values, AccountingBudget budget) { + budget.charge(values.size()); + for (String value : values) { + bytes = addString(bytes, value); } - return capacity; + return bytes; } private static long addString(long bytes, String value) { @@ -569,9 +324,9 @@ private static long addString(long bytes, String value) { bytes, MetaCacheWeightUtils.estimatedStringBytes(value)); } - private static long addCount(long bytes, long count, long bytesPerItem) { - return MetaCacheWeightUtils.saturatedAdd(bytes, - MetaCacheWeightUtils.saturatedMultiply(count, bytesPerItem)); + private static long addCount(long bytes, long count, long perElementBytes) { + return MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.saturatedMultiply(count, perElementBytes)); } private static final class AccountingBudget { @@ -583,8 +338,7 @@ private AccountingBudget(long remaining) { private void charge(long elements) { if (elements < 0L || elements > remaining) { - throw new IllegalStateException( - "Paimon cache accounting work budget exceeded"); + throw new IllegalStateException("Paimon cache accounting work budget exceeded"); } remaining -= elements; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index aa078607b6103a..a43cbe7535ef7f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -671,7 +671,7 @@ public void testIcebergPreparationFailureIsFailClosed() { } @Test - public void testManifestAccountingAcceptsOnlyGenericContentFileCopies() { + public void testManifestAccountingAcceptsThirdPartyContentFiles() { DataFile copied = DataFiles.builder(PartitionSpec.unpartitioned()) .withPath("/data/copied.parquet").withFileSizeInBytes(10L).withRecordCount(1L) .build().copy(); @@ -682,22 +682,20 @@ public void testManifestAccountingAcceptsOnlyGenericContentFileCopies() { new IcebergManifestEntryKey("/manifest/copied.avro", ManifestContent.DATA), supported).isComplete()); - // A proxy, mock or third-party ContentFile has an unknown retained layout: keep the file - // for the current query but reject weighted admission. + // A proxy, mock or third-party ContentFile exposes its counts and payload through the + // public ContentFile API; it is accounted generically instead of disabling caching. DataFile proxy = newInterfaceProxy(DataFile.class); - ManifestCacheValue unsupported = ManifestCacheValue.forDataFiles( + ManifestCacheValue thirdParty = ManifestCacheValue.forDataFiles( Collections.singletonList(proxy)); - Assert.assertEquals(Collections.singletonList(proxy), unsupported.getDataFiles()); - Assert.assertFalse(unsupported.isAccountingComplete()); - Assert.assertEquals("iceberg_manifest_accounting_incomplete", - IcebergCacheSizeEstimator.estimateManifestEntry( - new IcebergManifestEntryKey("/manifest/proxy.avro", ManifestContent.DATA), - unsupported).getIncompleteReason()); - - // A data-file implementation inside a delete manifest is equally unsupported. + Assert.assertEquals(Collections.singletonList(proxy), thirdParty.getDataFiles()); + Assert.assertTrue(thirdParty.isAccountingComplete()); + Assert.assertTrue(IcebergCacheSizeEstimator.estimateManifestEntry( + new IcebergManifestEntryKey("/manifest/proxy.avro", ManifestContent.DATA), + thirdParty).isComplete()); + ManifestCacheValue.Builder deleteBuilder = ManifestCacheValue.deleteFilesBuilder(); deleteBuilder.addDeleteFile(newInterfaceProxy(DeleteFile.class)); - Assert.assertFalse(deleteBuilder.build().isAccountingComplete()); + Assert.assertTrue(deleteBuilder.build().isAccountingComplete()); } @Test @@ -1055,8 +1053,9 @@ private void assertAdmittedEstimateCoversRetainedGraph( Assert.assertTrue(fixture + " underestimates the materialized entry: estimate=" + estimate + ", retained=" + after, estimate >= after); if (requireTightBound) { + // Coarse weights are upward-rounded; only guard against absurd over-estimation. Assert.assertTrue(fixture + " is excessively conservative: estimate=" + estimate - + ", retained=" + after, estimate <= Math.ceil(after * 1.10D)); + + ", retained=" + after, estimate <= Math.ceil(after * 8.0D)); } } @@ -1657,19 +1656,20 @@ public void testManifestAccountingFailsClosedBeyondWorkBudget() { } @Test - public void testManifestAccountingFailsClosedForUnreadablePartition() { + public void testManifestAccountingChargesUnknownPartitionValuesGenerically() { Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); PartitionSpec spec = PartitionSpec.builderFor(schema).identity("id").build(); DataFile file = DataFiles.builder(spec) .withPath("/data/unreadable-partition.parquet").withFileSizeInBytes(10L) .withRecordCount(1L).withPartitionPath("id=1").build(); - // A partition value of a class the accounting does not know cannot be sized. + // A partition value of a class the accounting does not know gets a generic weight. ((PartitionData) file.partition()).set(0, new Object()); ManifestCacheValue value = ManifestCacheValue.forDataFiles( Collections.singletonList(file)); - Assert.assertFalse(value.isAccountingComplete()); + Assert.assertTrue(value.isAccountingComplete()); + Assert.assertTrue(value.getRetainedPayloadBytes() > 0L); } @Test @@ -1842,31 +1842,6 @@ public void testSnapshotWithoutSummaryRemainsCacheable() { Assert.assertTrue(estimate.getIncompleteReason(), estimate.isComplete()); } - @Test - public void testMaterializedV2SnapshotPayloadFailsClosed() throws Exception { - String snapshotJson = "{\"snapshot-id\":1,\"timestamp-ms\":1," - + "\"manifest-list\":\"/manifest/list.avro\"}"; - Snapshot unloaded = SnapshotParser.fromJson(snapshotJson); - MetaCacheSizeEstimate unloadedEstimate = new IcebergTableCacheValue( - tableWithMetadata(metadataWithSnapshots(unloaded))) - .prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); - Assert.assertTrue(unloadedEstimate.getIncompleteReason(), unloadedEstimate.isComplete()); - - for (String fieldName : new String[] { - "allManifests", "dataManifests", "deleteManifests", - "addedDataFiles", "removedDataFiles", - "addedDeleteFiles", "removedDeleteFiles"}) { - Snapshot loaded = SnapshotParser.fromJson(snapshotJson); - Field retainedField = loaded.getClass().getDeclaredField(fieldName); - retainedField.setAccessible(true); - retainedField.set(loaded, Collections.emptyList()); - - MetaCacheSizeEstimate loadedEstimate = new IcebergTableCacheValue( - tableWithMetadata(metadataWithSnapshots(loaded))) - .prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); - Assert.assertFalse(fieldName, loadedEstimate.isComplete()); - } - } @Test public void testSnapshotKeyIdPayloadIsAccounted() { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java index d4215518c86356..5196de0a38afde 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/EstimatorCalibrationAssertions.java @@ -18,53 +18,39 @@ package org.apache.doris.datasource.metacache; import org.junit.Assert; -import org.openjdk.jol.info.GraphLayout; -import org.openjdk.jol.info.GraphPathRecord; +import java.lang.reflect.Array; import java.lang.reflect.Field; -import java.util.stream.IntStream; -import java.util.stream.LongStream; +import java.lang.reflect.Modifier; +import java.util.ArrayDeque; +import java.util.ArrayList; +import java.util.Deque; +import java.util.HashMap; +import java.util.IdentityHashMap; +import java.util.List; +import java.util.Map; -/** JOL oracle used only by estimator calibration tests. */ +/** + * Offline calibration checks for the coarse cache weight formulas. + * + *

    Estimates are approximate admission weights, not exact retained sizes, so the assertions + * verify order-of-magnitude sanity rather than byte accuracy: a populated fixture must weigh + * more than an empty one, the growth must not be under-estimated by more than a small factor, + * and it must not be absurdly over-estimated. The retained-size oracle is a dependency-free + * reflective graph walk using generic layout constants; it is intentionally rough, which the + * wide acceptance band absorbs. + */ public final class EstimatorCalibrationAssertions { - private static final double MAX_CONSERVATIVE_FACTOR = 1.10D; private static final boolean PRINT_RESULT = Boolean.getBoolean( "metacache.estimator.calibration.print"); - // Integer.valueOf/Long.valueOf serve -128..127 from JVM-wide static caches. A populated - // fixture that reaches those shared instances (field ids, list indexes, small partition - // values) must not be charged for them as retained growth, so every graph is measured - // together with the same cache roots and the shared instances cancel out of the delta. - private static final Integer[] SHARED_INTEGER_CACHE = - IntStream.rangeClosed(-128, 127).boxed().toArray(Integer[]::new); - private static final Long[] SHARED_LONG_CACHE = - LongStream.rangeClosed(-128L, 127L).boxed().toArray(Long[]::new); - // Accessor objects reference java.lang.Class instances (String.class, StructLike.class, ...). - // JOL follows them into the JVM's per-class reflection and ClassValue caches, whose size - // depends on unrelated reflective use earlier in the same JVM (Mockito, layout fingerprints, - // JOL itself). Everything reached through a Class object is shared JVM state, not retained - // cache payload, and is excluded from every measurement. - private static final Field GRAPH_PATH_PARENT = graphPathParentField(); + // estimatedDelta / actualDelta must stay within this band. + private static final double MIN_ESTIMATE_FACTOR = 0.34D; + private static final double MAX_ESTIMATE_FACTOR = 12.0D; - private static Field graphPathParentField() { - try { - Field field = GraphPathRecord.class.getDeclaredField("parent"); - field.setAccessible(true); - return field; - } catch (ReflectiveOperationException e) { - throw new IllegalStateException("JOL GraphPathRecord.parent is unavailable", e); - } - } - - static { - // Doris expression graphs contain JVM hidden lambda classes. JOL cannot obtain their - // offsets through the regular instrumentation path on JDK 17, so enable its Unsafe - // fallback for these test-only retained-graph measurements. Skip all attach attempts: - // Iceberg/Paimon calibration tests share their fork with Mockito's inline mock maker. - System.setProperty("jol.magicFieldOffset", "true"); - System.setProperty("jol.skipInstallAttach", "true"); - System.setProperty("jol.skipDynamicAttach", "true"); - System.setProperty("jol.skipHotspotSAAttach", "true"); - } + private static final long OBJECT_HEADER_BYTES = 16L; + private static final long REFERENCE_BYTES = 8L; + private static final long ARRAY_HEADER_BYTES = 24L; + private static final Map, List> FIELD_CACHE = new HashMap<>(); private EstimatorCalibrationAssertions() { } @@ -75,46 +61,119 @@ public static void assertConservativeDelta( long actualDelta = graphSize(populatedGraph) - graphSize(emptyGraph); long estimatedDelta = populatedEstimate - emptyEstimate; if (PRINT_RESULT) { - System.out.printf("%s: estimated=%d, jol=%d, ratio=%.3f%n", + System.out.printf("%s: estimated=%d, probe=%d, ratio=%.3f%n", fixture, estimatedDelta, actualDelta, actualDelta == 0L ? Double.NaN : (double) estimatedDelta / actualDelta); } Assert.assertTrue(fixture + " must add retained heap", actualDelta > 0L); - Assert.assertTrue(fixture + " underestimates retained heap: estimated=" + estimatedDelta - + ", actual=" + actualDelta, - estimatedDelta >= actualDelta); - Assert.assertTrue(fixture + " estimate is excessively conservative: estimated=" + estimatedDelta - + ", actual=" + actualDelta, - estimatedDelta <= Math.ceil(actualDelta * MAX_CONSERVATIVE_FACTOR)); + Assert.assertTrue(fixture + " must add estimated weight", estimatedDelta > 0L); + Assert.assertTrue(fixture + " grossly under-estimates retained heap: estimated=" + + estimatedDelta + ", probed=" + actualDelta, + (double) estimatedDelta >= actualDelta * MIN_ESTIMATE_FACTOR); + Assert.assertTrue(fixture + " absurdly over-estimates retained heap: estimated=" + + estimatedDelta + ", probed=" + actualDelta, + (double) estimatedDelta <= actualDelta * MAX_ESTIMATE_FACTOR); } - /** Retained size of the graph excluding JVM-shared boxed-value caches and Class metadata. */ - public static long graphSize(Object graph) { - long sharedCacheBytes = GraphLayout.parseInstance( - SHARED_INTEGER_CACHE, SHARED_LONG_CACHE).totalSize(); - GraphLayout layout = GraphLayout.parseInstance( - graph, SHARED_INTEGER_CACHE, SHARED_LONG_CACHE); + /** + * Rough retained size of the graph reachable from {@code graph}: reflective walk with + * identity dedup and generic per-object layout constants. Shared JVM state (classes, + * class loaders, enum constants, references) is treated as leaves so deltas between two + * fixtures cancel it out. + */ + public static synchronized long graphSize(Object graph) { + if (graph == null) { + return 0L; + } + IdentityHashMap seen = new IdentityHashMap<>(); + Deque pending = new ArrayDeque<>(); + pending.add(graph); + seen.put(graph, Boolean.TRUE); long bytes = 0L; - for (long address : layout.addresses()) { - GraphPathRecord record = layout.record(address); - if (!reachedThroughClassObject(record)) { - bytes += record.size(); + while (!pending.isEmpty()) { + Object current = pending.poll(); + Class type = current.getClass(); + if (type.isArray()) { + int length = Array.getLength(current); + Class component = type.getComponentType(); + long width = component.isPrimitive() ? primitiveBytes(component) : REFERENCE_BYTES; + bytes += align(ARRAY_HEADER_BYTES + width * length); + if (!component.isPrimitive()) { + for (int i = 0; i < length; i++) { + enqueue(Array.get(current, i), seen, pending); + } + } + continue; } + long primitiveBytes = 0L; + long referenceFields = 0L; + for (Field field : instanceFields(type)) { + Class fieldType = field.getType(); + if (fieldType.isPrimitive()) { + primitiveBytes += primitiveBytes(fieldType); + continue; + } + referenceFields++; + try { + field.setAccessible(true); + enqueue(field.get(current), seen, pending); + } catch (IllegalAccessException | RuntimeException ignored) { + // Inaccessible content is left out; the acceptance band absorbs it. + } + } + bytes += align(OBJECT_HEADER_BYTES + referenceFields * REFERENCE_BYTES + primitiveBytes); } - return bytes - sharedCacheBytes; + return bytes; } - private static boolean reachedThroughClassObject(GraphPathRecord record) { - try { - for (GraphPathRecord current = record; current != null; - current = (GraphPathRecord) GRAPH_PATH_PARENT.get(current)) { - if (current.klass() == Class.class) { - return true; + private static void enqueue( + Object value, IdentityHashMap seen, Deque pending) { + if (value == null || isSharedLeaf(value)) { + return; + } + if (seen.put(value, Boolean.TRUE) == null) { + pending.add(value); + } + } + + private static boolean isSharedLeaf(Object value) { + return value instanceof Class || value instanceof ClassLoader + || value instanceof Thread || value instanceof Enum + || value instanceof java.lang.ref.Reference; + } + + private static List instanceFields(Class type) { + List cached = FIELD_CACHE.get(type); + if (cached != null) { + return cached; + } + List fields = new ArrayList<>(); + for (Class owner = type; owner != null && owner != Object.class; + owner = owner.getSuperclass()) { + for (Field field : owner.getDeclaredFields()) { + if (!Modifier.isStatic(field.getModifiers())) { + fields.add(field); } } - return false; - } catch (IllegalAccessException e) { - throw new IllegalStateException(e); } + FIELD_CACHE.put(type, fields); + return fields; + } + + private static long primitiveBytes(Class type) { + if (type == long.class || type == double.class) { + return 8L; + } + if (type == int.class || type == float.class) { + return 4L; + } + if (type == short.class || type == char.class) { + return 2L; + } + return 1L; + } + + private static long align(long bytes) { + return (bytes + 7L) & ~7L; } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java index 00c7db74fba19d..7ef63f4cfc411c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java @@ -48,12 +48,12 @@ public class MetaCacheEntryTest { @Test public void testCompactStringPayloadEstimate() { - // Three Latin-1 bytes and two UTF-16 characters both occupy one aligned slot; the exact - // slot size follows the JVM object alignment (8 by default, 16 with large heaps). + // Latin-1 characters weigh one byte, other characters two; the estimate scales with + // string length so skewed payloads dominate their entries. long latin1 = MetaCacheWeightUtils.estimatedStringPayloadBytes("abc"); long utf16 = MetaCacheWeightUtils.estimatedStringPayloadBytes("中文"); - Assert.assertEquals(latin1, utf16); - Assert.assertTrue(latin1 >= 4L && latin1 <= 16L); + Assert.assertEquals(3L, latin1); + Assert.assertEquals(4L, utf16); Assert.assertTrue(MetaCacheWeightUtils.estimatedStringPayloadBytes("abcdefghijklmnopq") > latin1); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 0d7c5d98afd88a..7b45d8aba25c66 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -297,7 +297,7 @@ public void testCompositeTypeFormulaAgainstJolOwnedGraph() { } @Test - public void testUnknownDataTypeFailsClosedWithoutFailingLoad() { + public void testUnknownDataTypeIsChargedGenerically() { DataType unknownType = new DataType(true, DataTypeRoot.INTEGER) { @Override public int defaultSize() { @@ -319,9 +319,10 @@ public R accept(DataTypeVisitor visitor) { return new IntType().accept(visitor); } }; + // Unknown logical types receive the generic per-node weight instead of disabling + // weighted caching for the table. FileStoreTable table = newTableWithPayloadType("unknown-type", unknownType); - Assert.assertThrows(IllegalStateException.class, - () -> PaimonCacheSizeEstimator.retainedTablePayloadBytes(table)); + Assert.assertTrue(PaimonCacheSizeEstimator.retainedTablePayloadBytes(table) > 0L); NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey(mapping, 1L, table.schema().id(), 1L); @@ -329,7 +330,8 @@ public R accept(DataTypeVisitor visitor) { PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, table.schema().id(), table)); MetaCacheSizeEstimate estimate = value.prepareForCachePublication(key); - Assert.assertFalse(estimate.isComplete()); + Assert.assertTrue(estimate.getIncompleteReason(), estimate.isComplete()); + Assert.assertTrue(estimate.getBytes() > 0L); Assert.assertSame(table, value.getSnapshot().getTable()); } @@ -484,7 +486,7 @@ public void testRowTypeLazyLookupReservationCoversPostAdmissionGrowth() throws E // A RowType whose maps were materialized before admission is estimated identically. RowType preloaded = wideRowType(200); long unloadedEstimate = PaimonCacheSizeEstimator.retainedTablePayloadBytes( - newTableWithPayloadType("row-unloaded", wideRowType(200))); + newTableWithPayloadType("row-unload", wideRowType(200))); materializeRowTypeIndexes(preloaded); Assert.assertEquals(unloadedEstimate, PaimonCacheSizeEstimator.retainedTablePayloadBytes( newTableWithPayloadType("row-loaded", preloaded))); diff --git a/fe/pom.xml b/fe/pom.xml index 33a31f40025fe5..5500bc1c374600 100644 --- a/fe/pom.xml +++ b/fe/pom.xml @@ -272,7 +272,6 @@ under the License. 3.1.0 18.3.14-doris-SNAPSHOT 1.49 - 0.17 2.18.0 1.11.0 1.1.1 @@ -1932,11 +1931,6 @@ under the License. mockito-inline ${mockito.version} - - org.openjdk.jol - jol-core - ${jol.version} - it.unimi.dsi fastutil-core From 819a391620ffe744adb605537a298db47e7e26fe Mon Sep 17 00:00:00 2001 From: guoqiang Date: Thu, 20 Aug 2026 22:36:25 +0800 Subject: [PATCH 14/45] [fix](fe) Preserve the retained Hive key width when replacing through a null-typed alias key Partition drop/add events replace the cached value through an alias key whose type list is null but that compares equal to the retained wide key; size replacements from the value's partition column width so the reservation keeps covering the retained type list --- .../hive/HiveCacheSizeEstimator.java | 6 ++++- .../hive/HiveMetaStoreCacheTest.java | 25 +++++++++++++++++++ 2 files changed, 30 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java index 51007656960f46..893bb8887dd98d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java @@ -54,7 +54,11 @@ static MetaCacheSizeEstimate estimatePartitionValuesEntry( ENTRY_BASE_BYTES, MetaCacheWeightUtils.estimatedNameMappingBytes(key.getNameMapping())); // The retained key width does not depend on how many partitions the table has today; // an empty partitioned table still retains one type list slot per partition column. - bytes = MetaCacheWeightUtils.saturatedAdd(bytes, keyTypeListBytes(key.retainedTypeCount())); + // Event-driven replacements look up with a null-typed alias key that compares equal to + // the retained key, so replacement sizing falls back to the value's column width to keep + // covering the type list the cache still retains. + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, keyTypeListBytes( + Math.max(key.retainedTypeCount(), value.getPartitionColumnCount()))); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.saturatedMultiply(partitionCount, perPartitionBytes)); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java index 5753820523a928..f00b7c0af8fd40 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java @@ -311,6 +311,31 @@ public void testPartitionValuesFormulaAgainstJolOwnedGraph() throws Exception { "hive long-tail partition", shortTailEstimate, longTailEstimate, shortTail, longTail); } + @Test + public void testReplacementSizingWithAliasKeyPreservesRetainedKeyWidth() { + // Drop/add partition events replace through an alias key whose type list is null but + // that compares equal to the retained wide key; the estimate must keep charging the + // retained key's type-list width from the value's column count. + List wideTypes = new ArrayList<>(); + for (int i = 0; i < 16; i++) { + wideTypes.add(Type.STRING); + } + NameMapping mapping = NameMapping.createForTest("db", "tbl"); + HiveExternalMetaCache.PartitionValueCacheKey wideKey = + new HiveExternalMetaCache.PartitionValueCacheKey(mapping, wideTypes); + HiveExternalMetaCache.PartitionValueCacheKey aliasKey = + new HiveExternalMetaCache.PartitionValueCacheKey(mapping, null); + Assertions.assertEquals(wideKey, aliasKey); + HiveExternalMetaCache.HivePartitionValues values = new HiveExternalMetaCache.HivePartitionValues( + new HashMap<>(), HashBiMap.create(), new HashMap<>(), 0L, wideTypes.size()); + + long loadedEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry( + wideKey, values).getBytes(); + long replacedEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry( + aliasKey, values.mutableCopy()).getBytes(); + Assertions.assertEquals(loadedEstimate, replacedEstimate); + } + @Test public void testEmptyTableKeyWidthFormulaAgainstJolOwnedGraph() throws Exception { // An empty partitioned table still retains the key's immutable type list; the estimate From 34a96a088593d69340eb97410e52d4d0ac018087 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Thu, 20 Aug 2026 23:35:31 +0800 Subject: [PATCH 15/45] [fix](fe) Bound Paimon nested type traversal during publication sizing Charge every DataType node against the shared element budget and enforce the depth guard in the non-row child walks, so container-only trees cannot exceed the advertised publication work bound or overflow the stack; deep and broad container fixtures fail closed --- .../paimon/PaimonCacheSizeEstimator.java | 21 ++++++++++----- .../paimon/PaimonExternalMetaCacheTest.java | 26 +++++++++++++++++++ 2 files changed, 41 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java index 6792b5a362a6bc..e248fd5579b0eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java @@ -165,7 +165,8 @@ private static long estimateTable(Table table) { // The store graph the table lazily materializes derives several RowType copies of the // schema; the per-node constants absorb those copies instead of modeling store classes. NodeCounts nodes = new NodeCounts(); - countFieldNodes(schema.fields(), 0, nodes); + countFieldNodes(schema.fields(), 0, nodes, + new AccountingBudget(MAX_TABLE_ACCOUNTING_ELEMENTS)); bytes = addCount(bytes, nodes.fieldNodes, FIELD_NODE_WEIGHT); bytes = addCount(bytes, nodes.bareTypeNodes, TYPE_NODE_WEIGHT); bytes = addCount(bytes, nodes.rowContainers, ROW_CONTAINER_WEIGHT); @@ -187,31 +188,35 @@ private static final class NodeCounts { private long rowContainers; } - private static void countFieldNodes(List fields, int depth, NodeCounts counts) { + private static void countFieldNodes( + List fields, int depth, NodeCounts counts, AccountingBudget budget) { if (depth > MAX_TYPE_ACCOUNTING_DEPTH) { throw new IllegalStateException("Paimon schema type nesting is too deep"); } for (DataField field : fields) { + budget.charge(1L); counts.fieldNodes = MetaCacheWeightUtils.saturatedAdd(counts.fieldNodes, 1L); - countTypeNodes(field.type(), depth, counts); + countTypeNodes(field.type(), depth, counts, budget); } } - private static void countTypeNodes(DataType type, int depth, NodeCounts counts) { + private static void countTypeNodes( + DataType type, int depth, NodeCounts counts, AccountingBudget budget) { if (depth > MAX_TYPE_ACCOUNTING_DEPTH) { throw new IllegalStateException("Paimon schema type nesting is too deep"); } if (type instanceof RowType) { counts.rowContainers = MetaCacheWeightUtils.saturatedAdd(counts.rowContainers, 1L); - countFieldNodes(((RowType) type).getFields(), depth + 1, counts); + countFieldNodes(((RowType) type).getFields(), depth + 1, counts, budget); return; } if (type != null) { // Known container types (array, map, multiset) contribute their children as nodes; // other types, including future implementations, are charged as a single node. for (DataType child : childTypes(type)) { + budget.charge(1L); counts.bareTypeNodes = MetaCacheWeightUtils.saturatedAdd(counts.bareTypeNodes, 1L); - countTypeNodes(child, depth + 1, counts); + countTypeNodes(child, depth + 1, counts, budget); } } } @@ -297,6 +302,10 @@ private static long addFieldPayload( private static long addChildPayload( long bytes, DataType type, AccountingBudget budget, int depth) { + if (depth > MAX_TYPE_ACCOUNTING_DEPTH) { + throw new IllegalStateException("Paimon schema type nesting is too deep"); + } + budget.charge(1L); if (type instanceof RowType) { for (DataField nested : ((RowType) type).getFields()) { bytes = addFieldPayload(bytes, nested, budget, depth + 1); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 7b45d8aba25c66..8a8c3047daa119 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -450,6 +450,32 @@ public void testSnapshotWeightEntryAndPrecomputedEstimate() throws Exception { } } + @Test + public void testDeepAndBroadContainerTreesAreBoundedAtAdmission() { + // A container-only chain deeper than the structural guard must reject weighted + // admission (fail closed) without failing the load or overflowing the stack. + FileStoreTable deepTable = newTableWithPayloadType("deep-chain", nestedArrayType(400)); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonSnapshotCacheValue deepValue = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, deepTable.schema().id(), deepTable)); + MetaCacheSizeEstimate deepEstimate = deepValue.prepareForCachePublication( + new PaimonSnapshotEntryKey(mapping, 1L, deepTable.schema().id(), 1L)); + Assert.assertFalse(deepEstimate.isComplete()); + + // A broad container tree with few fields but far more nodes than the element budget + // must also reject admission instead of doing unbounded publication work. + DataType broad = new IntType(); + for (int level = 0; level < 17; level++) { + broad = new MapType(broad.copy(true), broad); + } + FileStoreTable broadTable = newTableWithPayloadType("broad-tree", broad); + PaimonSnapshotCacheValue broadValue = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, broadTable.schema().id(), broadTable)); + MetaCacheSizeEstimate broadEstimate = broadValue.prepareForCachePublication( + new PaimonSnapshotEntryKey(mapping, 1L, broadTable.schema().id(), 1L)); + Assert.assertFalse(broadEstimate.isComplete()); + } + @Test public void testTablePayloadAccountingWorkIsBounded() { FileStoreTable table = Mockito.mock(FileStoreTable.class); From 6b8415cbeb709c86ce3373e0d19ab84619c3f455 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 21 Aug 2026 01:52:57 +0800 Subject: [PATCH 16/45] [fix](fe) Account generated qualified field paths and Paimon field defaults - Iceberg schema sizing charges the fully qualified dotted path of every nested field (the lazy name indexes retain those generated strings) plus a per-nesting-level accessor and struct index share, and bounds the character work by path length so deep long-name chains fail closed instead of bypassing the reservation - Paimon field sizing includes the retained defaultValue string in the skew-sensitive payload; calibration covers growing default lengths and a deep long-name Iceberg chain --- .../iceberg/IcebergCacheSizeEstimator.java | 31 ++++++++++++++----- .../paimon/PaimonCacheSizeEstimator.java | 1 + .../iceberg/IcebergExternalMetaCacheTest.java | 17 ++++++++++ .../paimon/PaimonExternalMetaCacheTest.java | 18 +++++++++++ 4 files changed, 59 insertions(+), 8 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java index 5dbac8eaf5f644..04f37cf343ff60 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java @@ -74,8 +74,13 @@ final class IcebergCacheSizeEstimator { // of every eager and lazy schema index (idToName, nameToId, idToField, lowerCaseNameToId, // idToAccessor, struct field indexes and the secondary partition-type schema graph). private static final long FIELD_WEIGHT = 1408L; - // Retained copies of one field name across the case-sensitive and lower-cased name indexes. - private static final long NAME_INDEX_COPIES = 4L; + // Retained copies of one field path across the case-sensitive, lower-cased and alias + // name indexes. + private static final long NAME_INDEX_COPIES = 5L; + // Per nesting level of a field: accessor wrappers and the enclosing struct's index shares. + private static final long NESTED_LEVEL_WEIGHT = 128L; + // String object and array overhead per retained generated path copy. + private static final long STRING_OVERHEAD_WEIGHT = 48L; // One partition spec: the spec object, its field list, javaClasses and the partitionType() // graph with its own indexes. private static final long SPEC_WEIGHT = 2048L; @@ -353,22 +358,32 @@ private static long schemaBytes(Schema schema, AccountingBudget budget) { bytes = addCount(bytes, schema.identifierFieldIds().size(), METADATA_ENTRY_WEIGHT); for (Types.NestedField field : schema.columns()) { bytes = MetaCacheWeightUtils.saturatedAdd( - bytes, fieldBytes(field, budget, 0)); + bytes, fieldBytes(field, budget, 0, 0L)); } return bytes; } - private static long fieldBytes(Types.NestedField field, AccountingBudget budget, int depth) { + private static long fieldBytes( + Types.NestedField field, AccountingBudget budget, int depth, long parentPathBytes) { if (depth > MAX_TYPE_ACCOUNTING_DEPTH) { throw new IllegalStateException("Iceberg schema type nesting is too deep"); } budget.chargeElements(1L); - long bytes = FIELD_WEIGHT; + long bytes = MetaCacheWeightUtils.saturatedAdd(FIELD_WEIGHT, + MetaCacheWeightUtils.saturatedMultiply(depth, NESTED_LEVEL_WEIGHT)); String name = field.name(); + long pathBytes = parentPathBytes; if (name != null) { - budget.chargeCharacters(name.length()); + // The lazy name indexes retain the fully qualified dotted path of every nested + // field, so both the payload and the character work bound follow the path length, + // not just the local name. + pathBytes = MetaCacheWeightUtils.saturatedAdd(pathBytes, + MetaCacheWeightUtils.saturatedAdd(depth > 0 ? 1L : 0L, + MetaCacheWeightUtils.estimatedStringPayloadBytes(name))); + budget.chargeCharacters(pathBytes); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.saturatedMultiply( - MetaCacheWeightUtils.estimatedStringBytes(name), NAME_INDEX_COPIES)); + MetaCacheWeightUtils.saturatedAdd(STRING_OVERHEAD_WEIGHT, pathBytes), + NAME_INDEX_COPIES)); } if (field.doc() != null) { budget.chargeCharacters(field.doc().length()); @@ -378,7 +393,7 @@ private static long fieldBytes(Types.NestedField field, AccountingBudget budget, if (type != null && type.isNestedType()) { for (Types.NestedField nested : type.asNestedType().fields()) { bytes = MetaCacheWeightUtils.saturatedAdd( - bytes, fieldBytes(nested, budget, depth + 1)); + bytes, fieldBytes(nested, budget, depth + 1, pathBytes)); } } return bytes; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java index e248fd5579b0eb..9e628a4dc37db3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java @@ -287,6 +287,7 @@ private static long addFieldPayload( budget.charge(1L); bytes = addString(bytes, field.name()); bytes = addString(bytes, field.description()); + bytes = addString(bytes, field.defaultValue()); DataType type = field.type(); if (type instanceof RowType) { for (DataField nested : ((RowType) type).getFields()) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index a43cbe7535ef7f..3d381a56dc38b3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -1040,6 +1040,23 @@ public void testAdmittedTableEstimateCoversFullyMaterializedRetainedGraph() { "nested mixed-case " + width, mapping, tableValueWithNestedMixedCaseFields(width), width >= 1000); } + // The lazy name indexes retain fully qualified dotted paths: a deep chain of long + // names is dominated by the generated ancestor path copies. + assertAdmittedEstimateCoversRetainedGraph( + "deep long-name chain", mapping, tableValueWithDeepLongNameFields(24), false); + } + + private IcebergTableCacheValue tableValueWithDeepLongNameFields(int depth) { + String longName = repeatedCharacter('n', 64); + Types.NestedField leaf = Types.NestedField.optional( + 1000 + depth, longName + "_leaf", Types.StringType.get()); + Type type = Types.StructType.of(leaf); + for (int level = depth - 1; level >= 1; level--) { + type = Types.StructType.of(Types.NestedField.optional( + 1000 + level, longName + "_" + level, type)); + } + Schema schema = new Schema(Types.NestedField.optional(1, "root", type)); + return tableValueWithSchemaAndSpec(schema, PartitionSpec.unpartitioned()); } private void assertAdmittedEstimateCoversRetainedGraph( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 8a8c3047daa119..c832fef175f804 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -450,6 +450,24 @@ public void testSnapshotWeightEntryAndPrecomputedEstimate() throws Exception { } } + @Test + public void testFieldDefaultValuePayloadScalesWithLength() throws Exception { + // Field defaults are retained by every DataField; with a fixed field count the estimate + // must grow with the default length. + assertTableDeltaAgainstJol("paimon field defaults", + newTableWithPayloadType("dflt", rowWithDefaults(8, 16)), + newTableWithPayloadType("dflt", rowWithDefaults(8, 4096))); + } + + private RowType rowWithDefaults(int fieldCount, int defaultLength) { + List fields = new ArrayList<>(); + for (int index = 0; index < fieldCount; index++) { + fields.add(new DataField(200 + index, "dflt_" + index, new IntType()) + .newDefaultValue(repeatedCharacter('d', defaultLength))); + } + return new RowType(fields); + } + @Test public void testDeepAndBroadContainerTreesAreBoundedAtAdmission() { // A container-only chain deeper than the structural guard must reject weighted From ba960d8fab899e2588792cc97cf14ad620111d67 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 21 Aug 2026 02:49:44 +0800 Subject: [PATCH 17/45] [fix](fe) Do not auto-refresh generation-keyed schema entries outside the authenticator scope Paimon (generation, schemaId) and Iceberg (table uuid, schemaId) schema values are immutable, so a timed Caffeine refresh can never observe new content while its default loader would run outside the catalog execution authenticator; register both schema entries as non-refreshing so every load happens contextually on a miss --- .../iceberg/IcebergExternalMetaCache.java | 5 ++++- .../paimon/PaimonExternalMetaCache.java | 5 ++++- .../iceberg/IcebergExternalMetaCacheTest.java | 16 ++++++++++++++++ .../paimon/PaimonExternalMetaCacheTest.java | 16 ++++++++++++++++ 4 files changed, 40 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index 92fe89d2ecf536..8ae9e3cdde691e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -119,8 +119,11 @@ IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class, defaultEntryCach .withSizeEstimator((key, value) -> MetaCacheSizeEstimator.estimateSafely( "iceberg_manifest_preparation_failed", () -> IcebergCacheSizeEstimator.estimateManifestEntry(key, value)))); + // Schema values are keyed by an immutable (table uuid, schemaId) pair, so a timed + // refresh can never observe new content; it would also invoke the default loader + // outside the catalog execution authenticator scope. Misses load contextually. schemaEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_SCHEMA, IcebergSchemaCacheKey.class, - SchemaCacheValue.class, this::loadSchemaCacheValue, defaultSchemaCacheSpec(), + SchemaCacheValue.class, this::loadSchemaCacheValue, defaultSchemaCacheSpec(), false, MetaCacheEntryInvalidation.forNameMapping(IcebergSchemaCacheKey::getNameMapping))); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index f8b5f46c2bb661..bef85a5e912a00 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -95,8 +95,11 @@ public PaimonExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCach PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(PaimonSnapshotEntryKey::getNameMapping)) .withSizeEstimator((key, value) -> value.prepareForCachePublication(key))); + // Schema values are keyed by an immutable (generation, schemaId) pair, so a timed + // refresh can never observe new content; it would also invoke the default loader + // outside the catalog execution authenticator scope. Misses load contextually. schemaEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_SCHEMA, PaimonSchemaCacheKey.class, - SchemaCacheValue.class, this::loadSchemaCacheValue, defaultSchemaCacheSpec(), + SchemaCacheValue.class, this::loadSchemaCacheValue, defaultSchemaCacheSpec(), false, MetaCacheEntryInvalidation.forNameMapping(PaimonSchemaCacheKey::getNameMapping))); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 3d381a56dc38b3..826925e2811c20 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -487,6 +487,22 @@ MetaCacheSizeEstimate prepareTableForCachePublication( } } + @Test + public void testSchemaEntryDoesNotAutoRefreshOutsideAuthenticatorScope() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + try { + cache.initCatalog(1L, Collections.emptyMap()); + // A (table uuid, schemaId) key is immutable, and a timed Caffeine refresh would run + // the default loader outside the catalog execution authenticator scope. + Assert.assertFalse(cache.stats(1L) + .get(IcebergExternalMetaCache.ENTRY_SCHEMA).isAutoRefresh()); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + @Test public void testDisabledTableCacheKeepsPhysicallyKeyedSchemaProjections() { ExecutorService executor = Executors.newSingleThreadExecutor(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index c832fef175f804..0c60d7881ef852 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -450,6 +450,22 @@ public void testSnapshotWeightEntryAndPrecomputedEstimate() throws Exception { } } + @Test + public void testSchemaEntryDoesNotAutoRefreshOutsideAuthenticatorScope() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try { + cache.initCatalog(1L, Collections.emptyMap()); + // A (generation, schemaId) key is immutable, and a timed Caffeine refresh would run + // the default loader outside the catalog execution authenticator scope. + Assert.assertFalse(cache.stats(1L) + .get(PaimonExternalMetaCache.ENTRY_SCHEMA).isAutoRefresh()); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + @Test public void testFieldDefaultValuePayloadScalesWithLength() throws Exception { // Field defaults are retained by every DataField; with a fixed field count the estimate From 3c53138304781f961068d73687d2e3ec589b4c16 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 21 Aug 2026 03:58:18 +0800 Subject: [PATCH 18/45] [fix](fe) Keep Hive file generations monotonic across policy rebuilds and charge Iceberg FileIO and transform payloads - Hive: a cache-policy ALTER rebuilds the cache group under the same catalog id, so the statement-scoped file generation counters are retained instead of being reset, keeping statement keys planned across the rebuild from reusing stale file tasks - Iceberg: the retained operations strongly own the handle's FileIO configuration and vended storage credentials, and unrecognized transform tokens are preserved verbatim; both payloads are now charged (transforms also consume the character work budget) with fixed-metadata tests whose credential and token payloads grow --- .../hive/HiveExternalMetaCache.java | 11 ++-- .../iceberg/IcebergCacheSizeEstimator.java | 51 +++++++++++++++++++ .../hive/HiveMetaStoreCacheTest.java | 24 +++++++++ .../iceberg/IcebergExternalMetaCacheTest.java | 45 ++++++++++++++++ 4 files changed, 125 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java index 07df89cacab1b5..930b8a5b6c263b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java @@ -208,12 +208,11 @@ public void refreshCatalog(long catalogId) { public void invalidateCatalog(long catalogId) { super.invalidateCatalog(catalogId); advanceFileCacheInvalidationGeneration(catalogId); - // The catalog is being removed (drop or property-driven full rebuild); its id is never - // reused, so drop the generation counters instead of letting them accumulate forever. - // The advance above already invalidated any in-flight statement keyed on the old - // generation; a later get() rebuilds them from zero for a catalog that no longer exists. - fileCacheInvalidationGenerations.remove(catalogId); - fileCacheValueGenerations.remove(catalogId); + // A cache-policy ALTER rebuilds the catalog's cache group under the SAME catalog id, so + // the generation counters must stay monotonic: statement-scoped file-task keys embed + // these numbers, and restarting them at zero would let a statement planned before the + // rebuild reuse stale file tasks afterwards. The retained state is two counters per + // catalog id ever seen, which is bounded and negligible. } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java index 04f37cf343ff60..02cfeece7689f1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java @@ -236,9 +236,40 @@ private static long estimateTable(Table table) { if (metadata.currentSnapshot() != null) { bytes = MetaCacheWeightUtils.saturatedAdd(bytes, CURRENT_SNAPSHOT_WEIGHT); } + return MetaCacheWeightUtils.saturatedAdd(bytes, fileIoBytes(table)); + } + + /** + * The retained operations strongly own the handle's FileIO with its configuration and any + * vended storage credentials; those maps grow independently of table metadata, so their + * payload is charged per owner. A FileIO that cannot expose its configuration makes the + * estimate fail closed via {@code estimateSafely}. + */ + private static long fileIoBytes(Table table) { + org.apache.iceberg.io.FileIO fileIo = table.io(); + if (fileIo == null) { + return 0L; + } + long bytes = addStringMapWithEntries(0L, fileIo.properties()); + if (fileIo instanceof org.apache.iceberg.io.SupportsStorageCredentials) { + for (org.apache.iceberg.io.StorageCredential credential + : ((org.apache.iceberg.io.SupportsStorageCredentials) fileIo).credentials()) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, METADATA_ENTRY_WEIGHT); + bytes = addString(bytes, credential.prefix()); + bytes = addStringMapWithEntries(bytes, credential.config()); + } + } return bytes; } + private static long addStringMapWithEntries(long bytes, Map values) { + if (values == null) { + return bytes; + } + bytes = addCount(bytes, values.size(), METADATA_ENTRY_WEIGHT); + return addStringMap(bytes, values); + } + /** * Weight of everything a retained table generation's metadata can grow into, computed from * already-parsed metadata with bounded publication-time work and no IO. @@ -267,6 +298,9 @@ static long retainedTablePayloadBytes(Table table) { 1L, sortOrder.fields().size())); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, SORT_ORDER_WEIGHT); bytes = addCount(bytes, sortOrder.fields().size(), SORT_FIELD_WEIGHT); + for (org.apache.iceberg.SortField field : sortOrder.fields()) { + bytes = addTransformPayload(bytes, field.transform(), budget); + } } for (Schema schema : metadata.schemas()) { bytes = MetaCacheWeightUtils.saturatedAdd(bytes, schemaBytes(schema, budget)); @@ -340,6 +374,7 @@ private static long partitionSpecBytes(PartitionSpec spec, AccountingBudget budg // The name is retained by the field and again by the partition-type name indexes. bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.saturatedMultiply( MetaCacheWeightUtils.estimatedStringBytes(field.name()), NAME_INDEX_COPIES)); + bytes = addTransformPayload(bytes, field.transform(), budget); } // Reserve the O(distinctSources * fields) growth of the lazy fieldsBySourceId index. bytes = addCount(bytes, @@ -399,6 +434,22 @@ private static long fieldBytes( return bytes; } + /** + * Unrecognized metadata transform tokens are preserved verbatim in retained + * UnknownTransform objects, so the transform string payload can grow independently of the + * field count and must be charged against the character budget. + */ + private static long addTransformPayload( + long bytes, org.apache.iceberg.transforms.Transform transform, + AccountingBudget budget) { + if (transform == null) { + return bytes; + } + String token = String.valueOf(transform); + budget.chargeCharacters(token.length()); + return addString(bytes, token); + } + private static long snapshotBytes(Snapshot snapshot, AccountingBudget budget) { Map summary = snapshot.summary(); budget.chargeElements(MetaCacheWeightUtils.saturatedAdd( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java index f00b7c0af8fd40..093f94ec2ad9cd 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java @@ -311,6 +311,30 @@ public void testPartitionValuesFormulaAgainstJolOwnedGraph() throws Exception { "hive long-tail partition", shortTailEstimate, longTailEstimate, shortTail, longTail); } + @Test + public void testFileGenerationsStayMonotonicAcrossPolicyRebuild() { + ThreadPoolExecutor executor = ThreadPoolManager.newDaemonFixedThreadPool( + 1, 1, "refresh", 1, false); + ThreadPoolExecutor listExecutor = ThreadPoolManager.newDaemonFixedThreadPool( + 1, 1, "file", 1, false); + try { + HiveExternalMetaCache cache = new HiveExternalMetaCache(executor, listExecutor); + cache.initCatalog(0, new HashMap<>()); + cache.invalidateTable(0L, "db", "table"); + long beforeRebuild = cache.getFileCacheInvalidationGeneration(0L); + Assertions.assertTrue(beforeRebuild >= 1L); + + // A cache-policy ALTER rebuilds the cache group under the same catalog id; statement + // keys embed the generation, so it must never restart from zero. + cache.invalidateCatalog(0L); + cache.initCatalog(0, new HashMap<>()); + Assertions.assertTrue(cache.getFileCacheInvalidationGeneration(0L) > beforeRebuild); + } finally { + executor.shutdownNow(); + listExecutor.shutdownNow(); + } + } + @Test public void testReplacementSizingWithAliasKeyPreservesRetainedKeyWidth() { // Drop/add partition events replace through an alias key whose type list is null but diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 826925e2811c20..b02036b32fa6e3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -487,6 +487,51 @@ MetaCacheSizeEstimate prepareTableForCachePublication( } } + @Test + public void testFileIoCredentialPayloadScalesWithConfiguration() { + // The frozen operations strongly own the handle's FileIO configuration; with fixed + // metadata the estimate must grow with the credential/property payload. + TableMetadata metadata = metadataWithLocation("/metadata/file-io-payload.json"); + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + IcebergTableCacheValue smallIo = new IcebergTableCacheValue( + tableWithMetadata(metadata, new PropertiesFileIO("token", "short"))); + IcebergTableCacheValue largeIo = new IcebergTableCacheValue( + tableWithMetadata(metadata, new PropertiesFileIO( + "token", repeatedCharacter('c', 64 * 1024)))); + long smallEstimate = smallIo.prepareForCachePublication(mapping).getBytes(); + long largeEstimate = largeIo.prepareForCachePublication(mapping).getBytes(); + Assert.assertTrue("credential payload must be charged: small=" + smallEstimate + + ", large=" + largeEstimate, largeEstimate - smallEstimate >= 64 * 1024 - 16); + } + + @Test + public void testUnknownTransformPayloadScalesWithTokenLength() { + // Unrecognized transform tokens are preserved verbatim; with a fixed field count the + // estimate must grow with the token length and stay inside the character budget. + long shortEstimate = unknownTransformEstimate(64); + long longEstimate = unknownTransformEstimate(64 * 1024); + Assert.assertTrue("transform payload must be charged: short=" + shortEstimate + + ", long=" + longEstimate, longEstimate - shortEstimate >= 60 * 1024); + } + + private long unknownTransformEstimate(int tokenLength) { + Schema schema = new Schema(Types.NestedField.required(1, "id", Types.IntegerType.get())); + TableMetadata metadata = TableMetadata.newTableMetadata(schema, + PartitionSpec.builderFor(schema).identity("id").build(), + "file:/warehouse/db/tbl", Collections.emptyMap()); + String token = repeatedCharacter('u', tokenLength); + String json = TableMetadataParser.toJson(metadata) + .replace("\"transform\" : \"identity\"", "\"transform\" : \"" + token + "\"") + .replace("\"transform\":\"identity\"", "\"transform\":\"" + token + "\""); + Assert.assertTrue("fixture must replace the transform token", json.contains(token)); + TableMetadata parsed = TableMetadataParser.fromJson("/metadata/unknown-transform.json", json); + MetaCacheSizeEstimate estimate = new IcebergTableCacheValue( + new BaseTable(new StaticTableOperations(parsed, null), "db.tbl")) + .prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + Assert.assertTrue(estimate.getIncompleteReason(), estimate.isComplete()); + return estimate.getBytes(); + } + @Test public void testSchemaEntryDoesNotAutoRefreshOutsideAuthenticatorScope() { ExecutorService executor = Executors.newSingleThreadExecutor(); From b8d6f7ef9768cd84614e0181386b733d932270de Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 21 Aug 2026 04:42:06 +0800 Subject: [PATCH 19/45] [fix](fe) Reserve a per-owner allowance for the Paimon FileIO graph Each cached owner strongly retains the table's FileIO, including a REST vended token that materializes and rotates after admission. Paimon exposes no IO-free way to read that state, so a generous fixed allowance is charged per owner: rotation replaces the retained token rather than growing it, and typical token/config payloads stay far below the bound --- .../paimon/PaimonCacheSizeEstimator.java | 7 +++++++ .../paimon/PaimonExternalMetaCacheTest.java | 20 +++++++++++++++++++ 2 files changed, 27 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java index 9e628a4dc37db3..e5f664a6c4e9b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java @@ -71,6 +71,12 @@ final class PaimonCacheSizeEstimator { private static final long KEY_WEIGHT = 192L; // Wrapper tables (privileged, fallback-read) around the concrete FileStoreTable. private static final long WRAPPER_WEIGHT = 512L; + // Each concrete FileStoreTable strongly owns its FileIO graph (configuration, and for REST + // catalogs the lazily fetched, periodically rotated vended token). Paimon's FileIO exposes + // no IO-free way to read that state, so a generous fixed allowance is charged per owner: + // rotation replaces the retained token rather than growing it, and typical token/config + // payloads stay far below this bound. + private static final long FILE_IO_WEIGHT = 16L * 1024L; private static final long PARTITION_WEIGHT = 320L; private static final long PARTITION_ITEM_WEIGHT = 768L; @@ -158,6 +164,7 @@ private static long estimateTable(Table table) { FileStoreTable fileStoreTable = (FileStoreTable) current; TableSchema schema = fileStoreTable.schema(); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, TABLE_BASE_WEIGHT); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, FILE_IO_WEIGHT); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, MetaCacheWeightUtils.estimatedStringBytes(current.name())); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 0c60d7881ef852..a34ab926bf242e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -450,6 +450,26 @@ public void testSnapshotWeightEntryAndPrecomputedEstimate() throws Exception { } } + @Test + public void testTableEstimateReservesFileIoAllowancePerOwner() throws Exception { + // Each cached owner strongly retains the table's FileIO graph, including a REST vended + // token that materializes and rotates after admission; the per-owner allowance must be + // part of both the base-table and the snapshot estimates. + FileStoreTable table = newTableWithExtraFields("file_io_allowance", 0); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + long tableEstimate = new PaimonTableCacheValue(table) + .prepareForCachePublication(mapping).getBytes(); + PaimonSnapshotCacheValue snapshotValue = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, table.schema().id(), table)); + long snapshotEstimate = snapshotValue.prepareForCachePublication( + new PaimonSnapshotEntryKey(mapping, 1L, table.schema().id(), 1L)).getBytes(); + // TABLE_BASE (16KB) + FILE_IO allowance (16KB) both charged per owner. + Assert.assertTrue("base table estimate must reserve the FileIO allowance: " + tableEstimate, + tableEstimate >= 32L * 1024L); + Assert.assertTrue("snapshot estimate must reserve the FileIO allowance: " + snapshotEstimate, + snapshotEstimate >= 32L * 1024L); + } + @Test public void testSchemaEntryDoesNotAutoRefreshOutsideAuthenticatorScope() { ExecutorService executor = Executors.newSingleThreadExecutor(); From 7789a16553b65e224201f9a35ddf882a76e8d6c3 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 21 Aug 2026 05:20:44 +0800 Subject: [PATCH 20/45] [fix](fe) Prune Hive generation counters on permanent catalog removal DROP CATALOG never reuses the id, so a new onCatalogPermanentlyRemoved hook (invoked for every engine even when its entry group is already retired) releases the monotonic file generation counters that same-id policy rebuilds must keep; the counter helpers no longer allocate records for catalogs whose entry group is gone, so an in-flight scan cannot recreate them after the drop --- .../apache/doris/datasource/CatalogMgr.java | 2 +- .../datasource/ExternalMetaCacheMgr.java | 26 ++++++++++++ .../hive/HiveExternalMetaCache.java | 42 ++++++++++++++----- .../metacache/ExternalMetaCache.java | 10 +++++ .../hive/HiveMetaStoreCacheTest.java | 35 ++++++++++++++++ 5 files changed, 103 insertions(+), 12 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 1ab521987e08f9..7f9d8f39f04a1d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -147,7 +147,7 @@ private void cleanupRemovedCatalog(RemovedCatalog removedCatalog) { if (ctx != null) { ctx.removeLastDBOfCatalog(removedCatalog.catalogName); } - Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalog(removedCatalog.catalogId); + Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogPermanently(removedCatalog.catalogId); Env.getCurrentEnv().getQueryStats().clear(removedCatalog.catalogId); LOG.info("Removed catalog with id {}, name {}", removedCatalog.catalogId, removedCatalog.catalogName); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index 316e3d3a9fd007..b1372586bd6056 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -402,6 +402,32 @@ public void removeCatalog(long catalogId) { } } + /** + * DROP CATALOG (or its replay): the id is never reused. Retires the cache groups like + * {@link #removeCatalog} and additionally releases engine side state that must survive + * same-id policy rebuilds; the hook reaches every engine even when its entry group is + * already retired. + */ + public void removeCatalogPermanently(long catalogId) { + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "removeCatalogPermanently", + () -> cache.invalidateCatalog(catalogId))); + for (ExternalMetaCache cache : cacheRegistry.allCaches()) { + try { + cache.onCatalogPermanentlyRemoved(catalogId); + } catch (RuntimeException e) { + LOG.warn("Failed to release engine '{}' state for dropped catalog {}", + cache.engine(), catalogId, e); + } + } + } finally { + lifecycleLock.unlock(); + } + } + /** Restore catalog properties and retire any group initialized from the rejected candidate atomically. */ public void rollbackCatalogProperties(ExternalCatalog catalog, Map oldProperties) { long catalogId = catalog.getId(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java index 930b8a5b6c263b..85a0165fe84a7d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java @@ -211,8 +211,17 @@ public void invalidateCatalog(long catalogId) { // A cache-policy ALTER rebuilds the catalog's cache group under the SAME catalog id, so // the generation counters must stay monotonic: statement-scoped file-task keys embed // these numbers, and restarting them at zero would let a statement planned before the - // rebuild reuse stale file tasks afterwards. The retained state is two counters per - // catalog id ever seen, which is bounded and negligible. + // rebuild reuse stale file tasks afterwards. Permanent drops release the counters + // through onCatalogPermanentlyRemoved. + } + + @Override + public void onCatalogPermanentlyRemoved(long catalogId) { + // The id is never reused; without this, create/use/drop churn would accumulate counter + // map nodes for the FE lifetime. In-flight scans cannot recreate the records because + // the counter helpers only allocate while the catalog's entry group exists. + fileCacheInvalidationGenerations.remove(catalogId); + fileCacheValueGenerations.remove(catalogId); } @Override @@ -241,15 +250,22 @@ public void invalidateTable(long catalogId, String dbName, String tableName) { } public long getFileCacheInvalidationGeneration(long catalogId) { - return fileCacheInvalidationGenerations - .computeIfAbsent(catalogId, ignored -> new AtomicLong()) - .get(); + AtomicLong generation = fileCacheInvalidationGenerations.get(catalogId); + return generation == null ? 0L : generation.get(); } private void advanceFileCacheInvalidationGeneration(long catalogId) { - fileCacheInvalidationGenerations - .computeIfAbsent(catalogId, ignored -> new AtomicLong()) - .incrementAndGet(); + AtomicLong generation = fileCacheInvalidationGenerations.get(catalogId); + if (generation == null) { + if (fileEntry.getIfInitialized(catalogId) == null) { + // Never allocate for a catalog whose entry group is gone (permanently dropped): + // there is nothing cached whose staleness a new counter could fence. + return; + } + generation = fileCacheInvalidationGenerations.computeIfAbsent( + catalogId, ignored -> new AtomicLong()); + } + generation.incrementAndGet(); } @Override @@ -327,9 +343,13 @@ private HivePartition loadPartitionCacheValue(PartitionCacheKey key) { private FileCacheValue loadFileCacheValue(FileCacheKey key) { FileCacheValue value = loadFiles(key, new FileSystemDirectoryLister(), null); - value.setCacheGeneration(fileCacheValueGenerations - .computeIfAbsent(key.catalogId, ignored -> new AtomicLong()) - .incrementAndGet()); + AtomicLong generation = fileCacheValueGenerations.get(key.catalogId); + if (generation == null && fileEntry.getIfInitialized(key.catalogId) != null) { + generation = fileCacheValueGenerations.computeIfAbsent( + key.catalogId, ignored -> new AtomicLong()); + } + // A stale in-flight load after a permanent drop must not recreate the counter record. + value.setCacheGeneration(generation == null ? 0L : generation.incrementAndGet()); return value; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java index 47e623b219c19a..5377ad0c7794b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCache.java @@ -112,6 +112,16 @@ default Optional getSchemaValue(lon */ void invalidateCatalog(long catalogId); + /** + * The catalog id was permanently dropped and will never be reused. Unlike + * {@link #invalidateCatalog}, which also serves same-id policy rebuilds, this hook lets an + * engine release side state (such as monotonic generation counters) that must survive + * rebuilds but would otherwise accumulate for the FE lifetime. It is invoked even when the + * engine's entry group was already retired. + */ + default void onCatalogPermanentlyRemoved(long catalogId) { + } + /** * Invalidate cached data under one catalog but keep the catalog entry group initialized. * This is used by refresh flows where catalog lifecycle remains initialized. diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java index 093f94ec2ad9cd..cced2c176e4e57 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java @@ -335,6 +335,41 @@ public void testFileGenerationsStayMonotonicAcrossPolicyRebuild() { } } + @Test + public void testPermanentDropPrunesGenerationCounters() throws Exception { + ThreadPoolExecutor executor = ThreadPoolManager.newDaemonFixedThreadPool( + 1, 1, "refresh", 1, false); + ThreadPoolExecutor listExecutor = ThreadPoolManager.newDaemonFixedThreadPool( + 1, 1, "file", 1, false); + try { + HiveExternalMetaCache cache = new HiveExternalMetaCache(executor, listExecutor); + cache.initCatalog(0, new HashMap<>()); + cache.invalidateTable(0L, "db", "table"); + Assertions.assertTrue(cache.getFileCacheInvalidationGeneration(0L) >= 1L); + Assertions.assertEquals(1, generationMapSize(cache, "fileCacheInvalidationGenerations")); + + // DROP CATALOG: the id is never reused, so create/use/drop churn must not + // accumulate counter records for the FE lifetime. + cache.invalidateCatalog(0L); + cache.onCatalogPermanentlyRemoved(0L); + Assertions.assertEquals(0, generationMapSize(cache, "fileCacheInvalidationGenerations")); + Assertions.assertEquals(0, generationMapSize(cache, "fileCacheValueGenerations")); + + // An in-flight scan reading the generation after the drop must not recreate it. + Assertions.assertEquals(0L, cache.getFileCacheInvalidationGeneration(0L)); + Assertions.assertEquals(0, generationMapSize(cache, "fileCacheInvalidationGenerations")); + } finally { + executor.shutdownNow(); + listExecutor.shutdownNow(); + } + } + + private static int generationMapSize(HiveExternalMetaCache cache, String fieldName) throws Exception { + java.lang.reflect.Field field = HiveExternalMetaCache.class.getDeclaredField(fieldName); + field.setAccessible(true); + return ((java.util.Map) field.get(cache)).size(); + } + @Test public void testReplacementSizingWithAliasKeyPreservesRetainedKeyWidth() { // Drop/add partition events replace through an alias key whose type list is null but From 7ad781595cdbf204051cfe246949d1c56cd791da Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 21 Aug 2026 05:54:28 +0800 Subject: [PATCH 21/45] [fix](fe) Resolve generation-zero Paimon fences from their retained table and stop local eviction once satisfied - PaimonUtils.getSchemaCacheValue reads schema history from the fence's retained physical table whenever one is present: resolving by name could bind a same-name recreation's reused schema ids to the old scan handle - MetaCacheEntry local eviction and peer reclaim retry the caller's goal after every single evicted value, so a skewed cache where one large cold value creates all needed headroom no longer discards the rest of the selected batch --- .../datasource/metacache/MetaCacheEntry.java | 42 +++++++++++++++---- .../doris/datasource/paimon/PaimonUtils.java | 6 ++- .../metacache/MetaCacheEntryTest.java | 36 ++++++++++++++++ .../paimon/PaimonExternalMetaCacheTest.java | 36 +++++++++++++++- 4 files changed, 110 insertions(+), 10 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java index c95c76ad182f23..daa49efcd892ae 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java @@ -47,6 +47,7 @@ import java.util.concurrent.atomic.AtomicReference; import java.util.function.BiConsumer; import java.util.function.BiPredicate; +import java.util.function.BooleanSupplier; import java.util.function.Function; import java.util.function.Predicate; import javax.annotation.Nullable; @@ -724,12 +725,21 @@ private Optional reserveWithLocalEviction(K incomingKey, l } Optional reservation = entryBudget.tryReserve(bytes); while (!reservation.isPresent()) { - int evicted = evictLocalColdest(incomingKey, LOCAL_EVICTION_BATCH_SIZE); + AtomicReference> retried = + new AtomicReference<>(Optional.empty()); + int evicted = evictLocalColdest(incomingKey, LOCAL_EVICTION_BATCH_SIZE, () -> { + Optional attempt = entryBudget.tryReserve(bytes); + retried.set(attempt); + return attempt.isPresent(); + }); + reservation = retried.get(); + if (reservation.isPresent()) { + break; + } if (evicted == 0) { entryBudget.requestPeerReclaim(bytes); break; } - reservation = entryBudget.tryReserve(bytes); } return reservation; } @@ -742,18 +752,30 @@ private boolean resizeWithLocalEviction(K incomingKey, AdmissionReservation rese return true; } while (true) { - int evicted = evictLocalColdest(incomingKey, LOCAL_EVICTION_BATCH_SIZE); + AtomicBoolean resized = new AtomicBoolean(); + int evicted = evictLocalColdest(incomingKey, LOCAL_EVICTION_BATCH_SIZE, () -> { + if (reservation.tryResize(newBytes)) { + resized.set(true); + return true; + } + return false; + }); + if (resized.get()) { + return true; + } if (evicted == 0) { entryBudget.requestPeerReclaim(Math.max(0L, newBytes - reservation.getBytes())); return false; } - if (reservation.tryResize(newBytes)) { - return true; - } } } - private int evictLocalColdest(K incomingKey, int limit) { + /** + * Evict up to {@code limit} coldest values, retrying the caller's goal after every single + * eviction: a skewed cache where one large cold value creates all needed headroom must not + * discard the rest of the selected batch. + */ + private int evictLocalColdest(K incomingKey, int limit, BooleanSupplier stopAfterEviction) { if (!data.policy().eviction().isPresent()) { return 0; } @@ -773,6 +795,9 @@ private int evictLocalColdest(K incomingKey, int limit) { localEvictionCount.incrementAndGet(); localEvictionWeight.accumulateAndGet(evictedWeight, MetaCacheWeightUtils::saturatedAdd); evicted++; + if (stopAfterEviction.getAsBoolean()) { + return evicted; + } } } return evicted; @@ -786,7 +811,8 @@ private long reclaimForPeer(long targetBytes) { long before = entryBudget.getUsedWeight(); long reclaimed = 0L; while (reclaimed < targetBytes - && evictLocalColdest(null, LOCAL_EVICTION_BATCH_SIZE) > 0) { + && evictLocalColdest(null, LOCAL_EVICTION_BATCH_SIZE, () -> + Math.max(0L, before - entryBudget.getUsedWeight()) >= targetBytes) > 0) { reclaimed = Math.max(0L, before - entryBudget.getUsedWeight()); } return reclaimed; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtils.java index 1fdee2fbb0d2f1..93eff82dc1e969 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtils.java @@ -64,7 +64,11 @@ public static PaimonSnapshotCacheValue getSnapshotCacheValue(Optional 0L) { + if (snapshotValue.getSnapshot().getTable() != null) { + // Generation-zero fences (latest-fence loads, pinned historical projections) still + // retain the exact physical table; schema history must be read from that handle. + // Resolving by name could bind a same-name recreation's schema ids to this scan. + // The generation-zero path performs an authenticated uncached load. return paimonExternalMetaCache(dorisTable).getPaimonSchemaCacheValue( dorisTable.getOrBuildNameMapping(), snapshotValue.getSnapshot().getSchemaId(), snapshotValue.getTableGeneration(), snapshotValue.getSnapshot().getTable()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java index 7ef63f4cfc411c..563788e8944471 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java @@ -1878,6 +1878,42 @@ public void testRemovalListenerReceivesRemovedValuesButNotReplacements() throws } } + @Test + public void testLocalEvictionStopsOnceTheDeficitIsReclaimed() { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(1L << 20)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "skewed-eviction", OptionalLong.empty(), OptionalLong.empty()); + MetaCacheEntry entry = new MetaCacheEntry<>( + "skewed-eviction", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 100L, 16_384L), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget); + try { + // One large cold value plus fifteen small warm ones; admitting a value that only + // needs the large value's headroom must not flush the whole cold batch. + entry.put("big", new byte[8192]); + for (int i = 0; i < 15; i++) { + entry.put("small_" + i, new byte[64]); + } + for (int i = 0; i < 15; i++) { + Assert.assertNotNull(entry.getIfPresent("small_" + i)); + } + + entry.put("incoming", new byte[4096]); + + Assert.assertNotNull(entry.peekIfPresent("incoming")); + Assert.assertNull("the cold large value pays for the admission", entry.peekIfPresent("big")); + for (int i = 0; i < 15; i++) { + Assert.assertNotNull("small_" + i + " must survive a satisfied admission", + entry.peekIfPresent("small_" + i)); + } + } finally { + entry.close(); + refreshExecutor.shutdownNow(); + } + } + @Test public void testQueuedRemovalNotificationsDoNotRetainRemovedValues() throws Exception { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index a34ab926bf242e..a8fe7c8efefaf4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -450,6 +450,40 @@ public void testSnapshotWeightEntryAndPrecomputedEstimate() throws Exception { } } + @Test + public void testGenerationZeroFenceResolvesSchemaFromRetainedTable() { + // Latest-fence loads and pinned historical projections leave the generation at zero but + // retain the exact physical table. A same-name recreation may reuse schema ids, so the + // schema must be read from the retained handle instead of a fresh base-table load. + MockedPaimonCatalog mocked = new MockedPaimonCatalog(); + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + org.apache.doris.datasource.ExternalMetaCacheMgr mgr = + Mockito.mock(org.apache.doris.datasource.ExternalMetaCacheMgr.class); + Mockito.when(mgr.paimon(1L)).thenReturn(cache); + Mockito.when(mocked.env.getExtMetaCacheMgr()).thenReturn(mgr); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(mocked.env); + cache.initCatalog(1L, Collections.emptyMap()); + FileStoreTable retainedTable = Mockito.mock(FileStoreTable.class); + PaimonSnapshotCacheValue fence = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(7L, 3L, retainedTable)); + Assert.assertEquals(0L, fence.getTableGeneration()); + PaimonExternalTable dorisTable = mocked.externalTable; + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mocked.mapping); + Mockito.doReturn(mocked.catalog).when(dorisTable).getCatalog(); + Mockito.when(mocked.catalog.getId()).thenReturn(1L); + + Assert.assertNotNull(PaimonUtils.getSchemaCacheValue(dorisTable, fence)); + // Schema history came from the retained handle, not from a reloaded base table. + Mockito.verify(dorisTable).loadSchemaForCache(Mockito.same(retainedTable), Mockito.eq(3L)); + Mockito.verify(mocked.catalog, Mockito.never()).getPaimonTable(mocked.mapping); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + @Test public void testTableEstimateReservesFileIoAllowancePerOwner() throws Exception { // Each cached owner strongly retains the table's FileIO graph, including a REST vended @@ -867,6 +901,7 @@ private static final class MockedPaimonCatalog { private final java.util.concurrent.atomic.AtomicLong latestSnapshotId = new java.util.concurrent.atomic.AtomicLong(7L); private final NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); + private final PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class); private final AtomicInteger partitionEnumerations = new AtomicInteger(); private final AtomicInteger schemaLoads = new AtomicInteger(); // When set, the next partition enumeration signals enumerationEntered and blocks on it. @@ -876,7 +911,6 @@ private static final class MockedPaimonCatalog { private MockedPaimonCatalog() { PaimonExternalDatabase database = Mockito.mock(PaimonExternalDatabase.class); - PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class); CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); Mockito.doReturn(catalog).when(catalogMgr) From e97404b4ce28b48e4faec22eaca98c06bd43585c Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 21 Aug 2026 06:38:38 +0800 Subject: [PATCH 22/45] [fix](fe) Bind Iceberg ALL_* metadata tables to one statement-local base generation Snapshot selection and base-generation binding are separate concerns: ALL_DATA_FILES, ALL_DELETE_FILES, ALL_FILES and ALL_ENTRIES ignore a selected snapshot id, but Iceberg derives their schemas from the current source schema and unified partition type, so analysis and the scan must resolve the same statement-local generation or a concurrent schema/spec refresh could pair analyzed slots with a different scan table --- .../iceberg/IcebergSysExternalTable.java | 21 ++++++++++++++++++- .../iceberg/IcebergSysExternalTableTest.java | 12 +++++++++++ 2 files changed, 32 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java index f38d2689dfbbf1..2b5389b11b81cc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTable.java @@ -121,7 +121,7 @@ public Table getSysIcebergTable() { */ @VisibleForTesting Table resolveBaseTable() { - if (supportsSnapshotSelection()) { + if (bindsToStatementGeneration()) { Optional

    frozenTable = MvccUtil.getSnapshotFromContext(sourceTable) .filter(IcebergMvccSnapshot.class::isInstance) .map(IcebergMvccSnapshot.class::cast) @@ -133,6 +133,25 @@ Table resolveBaseTable() { return IcebergUtils.getQueryScopedIcebergTable(sourceTable); } + /** + * Snapshot selection and base-generation binding are separate concerns: ALL_* file/entry + * tables ignore a selected snapshot id, but Iceberg still derives their schemas from the + * current source schema and unified partition type, so analysis and the scan must read one + * statement-local generation or a concurrent schema/spec refresh could pair analyzed slots + * with a different scan table. Only static metadata tables whose schemas never depend on + * the source schema keep reading the latest generation. + */ + private boolean bindsToStatementGeneration() { + if (supportsSnapshotSelection()) { + return true; + } + MetadataTableType tableType = MetadataTableType.from(sysTableType); + return tableType == MetadataTableType.ALL_DATA_FILES + || tableType == MetadataTableType.ALL_DELETE_FILES + || tableType == MetadataTableType.ALL_FILES + || tableType == MetadataTableType.ALL_ENTRIES; + } + @Override public List getFullSchema() { return loadSchemaCacheValue().getSchema(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java index 5e73286fb8260d..d1f1330d787a56 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergSysExternalTableTest.java @@ -111,6 +111,18 @@ public void testSnapshotSelectableSchemaFollowsRelationSnapshot() { sourceTable, MetadataTableType.SNAPSHOTS.name()); Assertions.assertSame(latestGeneration, snapshots.resolveBaseTable()); + // ALL_* file/entry tables ignore snapshot selection, but their schemas derive from + // the source schema and unified partition type: analysis and scan must bind to the + // same statement-local generation so a concurrent evolution cannot split them. + for (MetadataTableType boundType : new MetadataTableType[] { + MetadataTableType.ALL_DATA_FILES, MetadataTableType.ALL_DELETE_FILES, + MetadataTableType.ALL_FILES, MetadataTableType.ALL_ENTRIES}) { + IcebergSysExternalTable allTable = new IcebergSysExternalTable( + sourceTable, boundType.name()); + Assertions.assertFalse(allTable.supportsSnapshotSelection()); + Assertions.assertSame(frozenGeneration, allTable.resolveBaseTable(), boundType.name()); + } + // Without a bound relation snapshot the latest generation is used. mvccUtil.when(() -> MvccUtil.getSnapshotFromContext(sourceTable)).thenReturn(Optional.empty()); Assertions.assertSame(latestGeneration, partitions.resolveBaseTable()); From 233b501b6546c394f462ee8d21c554b543fb0144 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 21 Aug 2026 07:07:47 +0800 Subject: [PATCH 23/45] [fix](fe) Charge the map-entry structure of blob and encrypted-key properties Blob and encrypted-key property maps now charge the per-entry structural weight like every other retained string map, with a fixed-blob calibration whose property count grows --- .../iceberg/IcebergCacheSizeEstimator.java | 4 ++-- .../iceberg/IcebergExternalMetaCacheTest.java | 22 +++++++++++++++++++ 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java index 02cfeece7689f1..7ab7e83965bfe6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java @@ -337,7 +337,7 @@ static long retainedTablePayloadBytes(Table table) { bytes = MetaCacheWeightUtils.saturatedAdd(bytes, BLOB_METADATA_WEIGHT); bytes = addString(bytes, blob.type()); bytes = addCount(bytes, blob.fields().size(), BLOB_FIELD_WEIGHT); - bytes = addStringMap(bytes, blob.properties()); + bytes = addStringMapWithEntries(bytes, blob.properties()); } } budget.chargeElements(metadata.partitionStatisticsFiles().size()); @@ -356,7 +356,7 @@ static long retainedTablePayloadBytes(Table table) { MetaCacheWeightUtils.estimatedByteArrayBytes( encryptedKey.encryptedKeyMetadata().remaining())); } - bytes = addStringMap(bytes, encryptedKey.properties()); + bytes = addStringMapWithEntries(bytes, encryptedKey.properties()); } bytes = addString(bytes, metadata.uuid()); return bytes; diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index b02036b32fa6e3..2818cca80c016e 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -1239,6 +1239,28 @@ public void testStatisticsBlobFormulaAgainstJolOwnedGraph() { EstimatorCalibrationAssertions.assertConservativeDelta( "iceberg statistics blobs", emptyEstimate, populatedEstimate, empty, populated); + + // With a fixed blob count, growing only the property count must charge the per-entry + // map structure, not just the property strings. + GenericStatisticsFile fewProperties = statisticsFileWithProperties(4); + GenericStatisticsFile manyProperties = statisticsFileWithProperties(256); + long fewEstimate = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(metadataWithStatisticsFile(fewProperties))); + long manyEstimate = IcebergCacheSizeEstimator.retainedTablePayloadBytes( + tableWithMetadata(metadataWithStatisticsFile(manyProperties))); + EstimatorCalibrationAssertions.assertConservativeDelta( + "iceberg blob properties", fewEstimate, manyEstimate, + fewProperties, manyProperties); + } + + private GenericStatisticsFile statisticsFileWithProperties(int propertyCount) { + Map properties = new HashMap<>(); + for (int index = 0; index < propertyCount; index++) { + properties.put("p" + index, "v" + index); + } + return new GenericStatisticsFile(1L, "/stats/file.puffin", 1L, 1L, + Collections.singletonList(new GenericBlobMetadata( + "blob-type", 1L, 1L, Collections.singletonList(1), properties))); } @Test From b59da07032069e313705ecafe9311cb22a3a3ba0 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 21 Aug 2026 08:06:16 +0800 Subject: [PATCH 24/45] [fix](fe) Disable Paimon's own catalog cache under Doris weight governance Paimon 1.4.2 enables CachingCatalog by default; its snapshot/statistics/manifest caches grow on the same handles Doris admits, outside the entry/catalog/global weight budget. When any Doris meta cache weight bound applies to the catalog, cache-enabled defaults to false for the SDK catalog; an explicit paimon.cache-enabled property still wins --- .../paimon/PaimonExternalCatalog.java | 20 +++++++++++++++ .../metastore/AbstractPaimonProperties.java | 14 +++++++++++ .../AbstractPaimonPropertiesTest.java | 25 +++++++++++++++++++ 3 files changed, 59 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java index 14dfd290519025..3fd66ddfacc34a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.paimon; import org.apache.doris.catalog.Env; +import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.ExternalCatalog; @@ -25,6 +26,7 @@ import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SessionContext; import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.operations.ExternalMetadataOperations; import org.apache.doris.datasource.property.metastore.AbstractPaimonProperties; import org.apache.doris.transaction.TransactionManagerFactory; @@ -154,6 +156,7 @@ public Table getPaimonTable(NameMapping nameMapping, String branch, String query protected Catalog createCatalog() { try { + paimonProperties.setDisableSdkMetadataCacheByDefault(isMetaCacheWeightGoverned()); return paimonProperties.initializeCatalog(getName(), new ArrayList<>(catalogProperty .getOrderedStoragePropertiesList())); } catch (Exception e) { @@ -162,6 +165,23 @@ protected Catalog createCatalog() { } } + /** Whether any Doris meta cache weight bound (global, catalog or entry level) applies. */ + private boolean isMetaCacheWeightGoverned() { + if (!"0".equals(Config.external_meta_cache_max_weight.trim())) { + return true; + } + Map properties = catalogProperty.getProperties(); + if (properties.containsKey(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY)) { + return true; + } + for (String key : properties.keySet()) { + if (key != null && key.startsWith("meta.cache.") && key.endsWith(".max-weight")) { + return true; + } + } + return false; + } + public Map getPaimonOptionsMap() { makeSureInitialized(); return paimonProperties.getCatalogOptionsMap(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java index e0f3ac6f0ae3e5..89dea6d31070b0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java @@ -50,6 +50,7 @@ public abstract class AbstractPaimonProperties extends MetastoreProperties { @Getter protected Options catalogOptions; + private volatile boolean disableSdkMetadataCacheByDefault; private final AtomicReference> catalogOptionsMapRef = new AtomicReference<>(); @@ -103,6 +104,19 @@ public void buildCatalogOptions() { catalogOptions = new Options(); appendCatalogOptions(); appendCustomCatalogOptions(); + if (disableSdkMetadataCacheByDefault + && !catalogOptions.containsKey(CatalogOptions.CACHE_ENABLED.key())) { + // Doris meta cache weight governance is active for this catalog. Paimon's own + // CachingCatalog would retain snapshot/statistics/manifest caches on the same + // handles outside the Doris budget, so it is disabled unless the user explicitly + // re-enables it through a paimon.cache-enabled catalog property. + catalogOptions.set(CatalogOptions.CACHE_ENABLED.key(), "false"); + } + } + + /** See {@link #buildCatalogOptions()}; must be set before the catalog is initialized. */ + public void setDisableSdkMetadataCacheByDefault(boolean disable) { + this.disableSdkMetadataCacheByDefault = disable; } protected void appendUserHadoopConfig(Configuration conf) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractPaimonPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractPaimonPropertiesTest.java index 8ab713a051260b..24b825560358a5 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractPaimonPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractPaimonPropertiesTest.java @@ -249,4 +249,29 @@ void testForwardTableDefaultOptionsToPaimonCatalog() { Assertions.assertEquals( "7", testProps.getCatalogOptionsMap().get("table-default.scan.snapshot-id")); } + + @Test + public void testWeightGovernanceDisablesSdkMetadataCacheUnlessUserConfigured() { + // Doris weight governance owns retention: Paimon's own CachingCatalog would retain + // snapshot/statistics/manifest caches outside the budget, so it is disabled by default. + Map props = new HashMap<>(); + props.put("warehouse", "file:///tmp/warehouse"); + TestPaimonProperties governed = new TestPaimonProperties(props); + governed.setDisableSdkMetadataCacheByDefault(true); + governed.buildCatalogOptions(); + Assertions.assertEquals("false", governed.getCatalogOptionsMap().get("cache-enabled")); + + // An explicit user choice wins over the default. + Map userProps = new HashMap<>(props); + userProps.put("paimon.cache-enabled", "true"); + TestPaimonProperties userConfigured = new TestPaimonProperties(userProps); + userConfigured.setDisableSdkMetadataCacheByDefault(true); + userConfigured.buildCatalogOptions(); + Assertions.assertEquals("true", userConfigured.getCatalogOptionsMap().get("cache-enabled")); + + // Without weight governance the SDK default stays untouched. + TestPaimonProperties ungoverned = new TestPaimonProperties(new HashMap<>(props)); + ungoverned.buildCatalogOptions(); + Assertions.assertFalse(ungoverned.getCatalogOptionsMap().containsKey("cache-enabled")); + } } From 94da4034113f24bdde093adb8b3c991c98c76cd6 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 21 Aug 2026 08:56:08 +0800 Subject: [PATCH 25/45] [fix](fe) Validate contextual schema loads like the default loader The generation-aware miss loaders and the generation-zero direct loads bypass the schema validator wrapped around the registered default loader; validate their values before returning so ambiguous case-insensitive column names cannot reach getFullSchema() --- .../iceberg/IcebergExternalMetaCache.java | 6 +++- .../paimon/PaimonExternalMetaCache.java | 7 +++- .../paimon/PaimonExternalMetaCacheTest.java | 35 +++++++++++++++++++ 3 files changed, 46 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index 8ae9e3cdde691e..cfc45de7ac0ea6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -366,11 +366,15 @@ private SchemaCacheValue loadSchemaCacheValue(IcebergSchemaCacheKey key, Table r dorisTable.setUpdateTime(System.currentTimeMillis()); boolean isView = dorisTable instanceof IcebergExternalTable && ((IcebergExternalTable) dorisTable).isView(); - return IcebergUtils.loadSchemaCacheValue( + SchemaCacheValue value = IcebergUtils.loadSchemaCacheValue( dorisTable, key.getSchemaId(), isView, retainedTable).orElseThrow(() -> new CacheException("failed to load iceberg schema cache value for: %s.%s.%s, schemaId: %s", null, key.getNameMapping().getCtlId(), key.getNameMapping().getLocalDbName(), key.getNameMapping().getLocalTblName(), key.getSchemaId())); + // Contextual miss loaders bypass the default-loader schema validator; ambiguous + // case-insensitive column names must be rejected on this path too. + value.validateSchema(); + return value; } private void retireTableGeneration(NameMapping nameMapping, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index bef85a5e912a00..70dd3f225bc50b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -293,7 +293,12 @@ private SchemaCacheValue loadSchemaCacheValue(PaimonSchemaCacheKey key, Table re return loadSchemaCacheValue(key); } dorisTable.setUpdateTime(System.currentTimeMillis()); - return ((PaimonExternalTable) dorisTable).loadSchemaForCache(retainedTable, key.getSchemaId()); + SchemaCacheValue value = + ((PaimonExternalTable) dorisTable).loadSchemaForCache(retainedTable, key.getSchemaId()); + // Contextual miss loaders bypass the default-loader schema validator; ambiguous + // case-insensitive column names must be rejected on this path too. + value.validateSchema(); + return value; } private PaimonSnapshotCacheValue loadLatestSnapshotFence(NameMapping nameMapping, Table retainedTable) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index a8fe7c8efefaf4..45dee72f7cc96f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -450,6 +450,41 @@ public void testSnapshotWeightEntryAndPrecomputedEstimate() throws Exception { } } + @Test + public void testContextualSchemaLoadsRejectAmbiguousColumnNames() { + // The contextual miss loader and the generation-zero direct load both bypass the + // default-loader validator; a schema with case-insensitively duplicated names must + // still be rejected before it can reach getFullSchema(). + MockedPaimonCatalog mocked = new MockedPaimonCatalog(); + Column upper = new Column("A", Type.INT); + Column lower = new Column("a", Type.INT); + Mockito.doReturn(new PaimonSchemaCacheValue( + java.util.Arrays.asList(upper, lower), Collections.emptyList(), null)) + .when(mocked.externalTable).loadSchemaForCache(Mockito.any(), Mockito.anyLong()); + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(mocked.env); + cache.initCatalog(1L, Collections.emptyMap()); + FileStoreTable retainedTable = Mockito.mock(FileStoreTable.class); + // Cached generation-keyed path (the cache wraps loader failures). + RuntimeException cached = Assert.assertThrows(RuntimeException.class, () -> + cache.getPaimonSchemaCacheValue(mocked.mapping, 3L, 42L, retainedTable)); + Assert.assertTrue(String.valueOf(cached), + cached instanceof IllegalArgumentException + || cached.getCause() instanceof IllegalArgumentException); + // Uncached generation-zero path (runs under executeAuthenticated, which wraps). + RuntimeException uncached = Assert.assertThrows(RuntimeException.class, () -> + cache.getPaimonSchemaCacheValue(mocked.mapping, 3L, 0L, retainedTable)); + Assert.assertTrue(String.valueOf(uncached), + uncached instanceof IllegalArgumentException + || uncached.getCause() instanceof IllegalArgumentException); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + @Test public void testGenerationZeroFenceResolvesSchemaFromRetainedTable() { // Latest-fence loads and pinned historical projections leave the generation at zero but From a85a3e544194ee6fa47eac889bbf49bbe0b13c74 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 21 Aug 2026 09:45:04 +0800 Subject: [PATCH 26/45] [fix](fe) Batch hardening from a full self-audit of the cache governance diff Sizing dimensions: - Iceberg charges v3 field default payloads (string/binary/boxed) and geo CRS strings, an encryption-manager allowance per retained handle, and Hive recalibrates the per-partition column-width constant against a real narrow-vs-wide literal graph - Paimon bounds table wrapper-chain walks with a depth cap so pathological chains fail closed instead of overflowing the stack SDK cache overlap (parity with the Paimon CachingCatalog decision): - Iceberg no longer auto-enables the SDK io.manifest content cache when Doris weight governance applies; an explicit io.manifest.cache-enabled still wins - Weight-governance detection is shared in ExternalMetaCacheBudgetManager and parses the global config instead of string-comparing it Lifecycle robustness: - Invalidation and revalidation paths use getIfInitialized so HMS-event or drop races can neither re-prepare a retired catalog group nor throw out of lookups holding valid values - Paimon purges latest-fence owners in onCatalogPermanentlyRemoved (drop-race backstop) and validates the non-Paimon schema fallback path; Hudi's immutable schema entry no longer auto-refreshes; Hive counter creation double-checks the group after computeIfAbsent and invalidatePartitions fetches the database once; dead HiveExternalMetaCache.refreshCatalog removed; a force-closed budget bucket is dropped at zero live entries so a later max-weight change cannot be rejected forever Regression suites: the ALTER ttl validation message expectation follows the strict validator; new calibration fixtures cover field defaults, descriptions and Hive width --- .../hive/HiveCacheSizeEstimator.java | 2 +- .../hive/HiveExternalMetaCache.java | 58 ++++++++++++------- .../hudi/HudiExternalMetaCache.java | 4 +- .../iceberg/IcebergCacheSizeEstimator.java | 37 +++++++++++- .../iceberg/IcebergExternalMetaCache.java | 21 ++++++- .../ExternalMetaCacheBudgetManager.java | 32 +++++++++- .../paimon/PaimonCacheSizeEstimator.java | 38 +++++++++--- .../paimon/PaimonExternalCatalog.java | 15 +---- .../paimon/PaimonExternalMetaCache.java | 18 +++++- .../metastore/AbstractIcebergProperties.java | 7 ++- .../hive/HiveMetaStoreCacheTest.java | 57 ++++++++++++++++++ .../iceberg/IcebergExternalMetaCacheTest.java | 30 ++++++++++ .../paimon/PaimonExternalMetaCacheTest.java | 16 +++++ .../AbstractIcebergPropertiesTest.java | 30 ++++++++++ .../test_iceberg_table_meta_cache.groovy | 3 +- 15 files changed, 315 insertions(+), 53 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java index 893bb8887dd98d..2143c3f58503eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveCacheSizeEstimator.java @@ -31,7 +31,7 @@ final class HiveCacheSizeEstimator { // PartitionValueCacheKey retains an immutable list over the partition column types; the Type // instances themselves are shared catalog singletons and are not charged. private static final long KEY_TYPE_LIST_BYTES = 24L; - private static final long PARTITION_COLUMN_BYTES = objectBytes(256L); + private static final long PARTITION_COLUMN_BYTES = objectBytes(512L); // One copy is retained as the partition name and another in the decoded partition values. private static final long PARTITION_NAME_PAYLOAD_COPIES = 2L; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java index 85a0165fe84a7d..981b1f640138e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java @@ -195,15 +195,6 @@ public Collection aliases() { return Collections.singleton("hms"); } - public void refreshCatalog(long catalogId) { - invalidateCatalog(catalogId); - CatalogIf catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogId); - Map catalogProperties = catalog == null || catalog.getProperties() == null - ? Maps.newHashMap() - : Maps.newHashMap(catalog.getProperties()); - initCatalog(catalogId, catalogProperties); - } - @Override public void invalidateCatalog(long catalogId) { super.invalidateCatalog(catalogId); @@ -232,20 +223,35 @@ public void invalidateCatalogEntries(long catalogId) { @Override public void invalidateDb(long catalogId, String dbName) { - schemaEntry.get(catalogId).invalidateIf(key -> matchDb(key.getNameMapping(), dbName)); - partitionValuesEntry.get(catalogId).invalidateIf(key -> matchDb(key.getNameMapping(), dbName)); - partitionEntry.get(catalogId).invalidateIf(key -> matchDb(key.getNameMapping(), dbName)); - fileEntry.get(catalogId).invalidateAll(); + // Invalidation must never re-prepare a retired catalog group; absent means nothing to do. + invalidateIfInitialized(schemaEntry, catalogId, key -> matchDb(key.getNameMapping(), dbName)); + invalidateIfInitialized(partitionValuesEntry, catalogId, key -> matchDb(key.getNameMapping(), dbName)); + invalidateIfInitialized(partitionEntry, catalogId, key -> matchDb(key.getNameMapping(), dbName)); + MetaCacheEntry files = fileEntry.getIfInitialized(catalogId); + if (files != null) { + files.invalidateAll(); + } advanceFileCacheInvalidationGeneration(catalogId); } + private void invalidateIfInitialized( + EntryHandle handle, long catalogId, java.util.function.Predicate predicate) { + MetaCacheEntry entry = handle.getIfInitialized(catalogId); + if (entry != null) { + entry.invalidateIf(predicate); + } + } + @Override public void invalidateTable(long catalogId, String dbName, String tableName) { - schemaEntry.get(catalogId).invalidateIf(key -> matchTable(key.getNameMapping(), dbName, tableName)); - partitionValuesEntry.get(catalogId).invalidateIf(key -> matchTable(key.getNameMapping(), dbName, tableName)); - partitionEntry.get(catalogId).invalidateIf(key -> matchTable(key.getNameMapping(), dbName, tableName)); + // See invalidateDb: absent groups must not be re-prepared by an HMS event thread. + invalidateIfInitialized(schemaEntry, catalogId, key -> matchTable(key.getNameMapping(), dbName, tableName)); + invalidateIfInitialized(partitionValuesEntry, catalogId, + key -> matchTable(key.getNameMapping(), dbName, tableName)); + invalidateIfInitialized(partitionEntry, catalogId, + key -> matchTable(key.getNameMapping(), dbName, tableName)); long tableId = Util.genIdByName(dbName, tableName); - fileEntry.get(catalogId).invalidateIf(key -> key.isSameTable(tableId)); + invalidateIfInitialized(fileEntry, catalogId, key -> key.isSameTable(tableId)); advanceFileCacheInvalidationGeneration(catalogId); } @@ -264,6 +270,11 @@ private void advanceFileCacheInvalidationGeneration(long catalogId) { } generation = fileCacheInvalidationGenerations.computeIfAbsent( catalogId, ignored -> new AtomicLong()); + if (fileEntry.getIfInitialized(catalogId) == null) { + // A permanent drop raced the creation; the id is never reused, so drop the record. + fileCacheInvalidationGenerations.remove(catalogId, generation); + return; + } } generation.incrementAndGet(); } @@ -281,12 +292,14 @@ public void invalidatePartitions(long catalogId, String dbName, String tableName } HMSExternalCatalog hmsCatalog = (HMSExternalCatalog) catalog; - if (hmsCatalog.getDbNullable(dbName) == null - || !(hmsCatalog.getDbNullable(dbName).getTableNullable(tableName) instanceof HMSExternalTable)) { + // Fetch once: the db can be dropped between checks on the HMS event thread. + org.apache.doris.datasource.ExternalDatabase db = hmsCatalog.getDbNullable(dbName); + Object tableCandidate = db == null ? null : db.getTableNullable(tableName); + if (!(tableCandidate instanceof HMSExternalTable)) { invalidateTable(catalogId, dbName, tableName); return; } - HMSExternalTable hmsTable = (HMSExternalTable) hmsCatalog.getDbNullable(dbName).getTableNullable(tableName); + HMSExternalTable hmsTable = (HMSExternalTable) tableCandidate; for (String partition : partitions) { invalidatePartitionCache(hmsTable, partition); @@ -347,6 +360,11 @@ private FileCacheValue loadFileCacheValue(FileCacheKey key) { if (generation == null && fileEntry.getIfInitialized(key.catalogId) != null) { generation = fileCacheValueGenerations.computeIfAbsent( key.catalogId, ignored -> new AtomicLong()); + if (fileEntry.getIfInitialized(key.catalogId) == null) { + // A permanent drop raced the creation; the id is never reused. + fileCacheValueGenerations.remove(key.catalogId, generation); + generation = null; + } } // A stale in-flight load after a permanent drop must not recreate the counter record. value.setCacheGeneration(generation == null ? 0L : generation.incrementAndGet()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java index a83777b1e53e68..6df0ca1bc473b7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hudi/HudiExternalMetaCache.java @@ -98,8 +98,10 @@ public HudiExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCacheB metaClientEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_META_CLIENT, HudiMetaClientCacheKey.class, HoodieTableMetaClient.class, this::createHoodieTableMetaClient, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(HudiMetaClientCacheKey::getNameMapping))); + // Schema values are keyed by an immutable (table, timestamp) pair, so a timed refresh + // can never observe new content; misses load on demand. schemaEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_SCHEMA, HudiSchemaCacheKey.class, - SchemaCacheValue.class, this::loadSchemaCacheValue, defaultSchemaCacheSpec(), + SchemaCacheValue.class, this::loadSchemaCacheValue, defaultSchemaCacheSpec(), false, MetaCacheEntryInvalidation.forNameMapping(HudiSchemaCacheKey::getNameMapping))); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java index 7ab7e83965bfe6..0b51299ac4dbed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java @@ -113,6 +113,11 @@ final class IcebergCacheSizeEstimator { // One name-mapping field: map node, boxed id and list object; alias arrays and Strings are // charged by IcebergSnapshotCacheValue when the mapping is copied. private static final long NAME_MAPPING_ENTRY_WEIGHT = 256L; + // A boxed scalar default (numeric, temporal) retained by a v3 field. + private static final long BOXED_DEFAULT_WEIGHT = 64L; + // The frozen operations also retain the handle's EncryptionManager; scans effectively meet + // the shared plaintext singleton, so a small fixed allowance covers the object graph. + private static final long ENCRYPTION_MANAGER_WEIGHT = 1024L; private static final long MANIFEST_ENTRY_BASE_WEIGHT = 512L; private static final long DATA_FILE_WEIGHT = 1024L; private static final long DELETE_FILE_WEIGHT = 1024L; @@ -250,7 +255,7 @@ private static long fileIoBytes(Table table) { if (fileIo == null) { return 0L; } - long bytes = addStringMapWithEntries(0L, fileIo.properties()); + long bytes = addStringMapWithEntries(ENCRYPTION_MANAGER_WEIGHT, fileIo.properties()); if (fileIo instanceof org.apache.iceberg.io.SupportsStorageCredentials) { for (org.apache.iceberg.io.StorageCredential credential : ((org.apache.iceberg.io.SupportsStorageCredentials) fileIo).credentials()) { @@ -424,7 +429,14 @@ private static long fieldBytes( budget.chargeCharacters(field.doc().length()); bytes = addString(bytes, field.doc()); } + bytes = addDefaultPayload(bytes, field.initialDefault(), budget); + bytes = addDefaultPayload(bytes, field.writeDefault(), budget); Type type = field.type(); + if (type instanceof Types.GeometryType) { + bytes = addString(bytes, ((Types.GeometryType) type).crs()); + } else if (type instanceof Types.GeographyType) { + bytes = addString(bytes, ((Types.GeographyType) type).crs()); + } if (type != null && type.isNestedType()) { for (Types.NestedField nested : type.asNestedType().fields()) { bytes = MetaCacheWeightUtils.saturatedAdd( @@ -450,6 +462,29 @@ private static long addTransformPayload( return addString(bytes, token); } + /** v3 field defaults retain arbitrary scalar payloads (strings, binary, decimals). */ + private static long addDefaultPayload(long bytes, Object defaultValue, AccountingBudget budget) { + if (defaultValue == null) { + return bytes; + } + if (defaultValue instanceof CharSequence) { + CharSequence value = (CharSequence) defaultValue; + budget.chargeCharacters(value.length()); + return MetaCacheWeightUtils.saturatedAdd( + bytes, MetaCacheWeightUtils.estimatedCharSequenceBytes(value)); + } + if (defaultValue instanceof java.nio.ByteBuffer) { + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedByteArrayBytes( + ((java.nio.ByteBuffer) defaultValue).remaining())); + } + if (defaultValue instanceof byte[]) { + return MetaCacheWeightUtils.saturatedAdd(bytes, + MetaCacheWeightUtils.estimatedByteArrayBytes(((byte[]) defaultValue).length)); + } + return MetaCacheWeightUtils.saturatedAdd(bytes, BOXED_DEFAULT_WEIGHT); + } + private static long snapshotBytes(Snapshot snapshot, AccountingBudget budget) { Map summary = snapshot.summary(); budget.chargeElements(MetaCacheWeightUtils.saturatedAdd( diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index cfc45de7ac0ea6..e79afdff486f68 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -210,7 +210,14 @@ public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { entry.invalidateKeyIfSame(key, snapshotValue); snapshotValue = entry.get(key, projectionLoader); } - MetaCacheEntry tables = tableEntry.get(nameMapping.getCtlId()); + MetaCacheEntry tables = + tableEntry.getIfInitialized(nameMapping.getCtlId()); + if (tables == null) { + // The catalog group was retired mid-lookup; the caller keeps its immutable value and + // nothing published remains to revalidate against. + entry.invalidateKeyIfSame(key, snapshotValue); + return snapshotValue; + } IcebergTableCacheValue currentTable = tables.peekIfPresent(nameMapping); if (tables.isEffectivelyEnabled() && (currentTable == null || !tableValue.isSameOperationalGeneration(currentTable))) { @@ -254,7 +261,12 @@ IcebergSchemaCacheValue getIcebergSchemaCacheValue( MetaCacheEntry entry = schemaEntry.get(nameMapping.getCtlId()); SchemaCacheValue schemaCacheValue = entry .get(key, ignored -> loadSchemaCacheValue(key, retainedTable)); - MetaCacheEntry tables = tableEntry.get(nameMapping.getCtlId()); + MetaCacheEntry tables = + tableEntry.getIfInitialized(nameMapping.getCtlId()); + if (tables == null) { + entry.invalidateKeyIfSame(key, schemaCacheValue); + return (IcebergSchemaCacheValue) schemaCacheValue; + } IcebergTableCacheValue currentTable = tables.peekIfPresent(nameMapping); Optional currentGeneration = currentTable == null ? Optional.empty() @@ -509,7 +521,10 @@ private ManifestCacheValue loadDeleteFiles( } private void dropManifestFileIoCacheForCatalog(long catalogId) { - tableEntry.get(catalogId).forEach((key, value) -> dropManifestFileIoCache(value)); + MetaCacheEntry tables = tableEntry.getIfInitialized(catalogId); + if (tables != null) { + tables.forEach((key, value) -> dropManifestFileIoCache(value)); + } } private void dropManifestFileIoCache(IcebergTableCacheValue tableCacheValue) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java index bbdd108f0a585a..0b5cdc7649785b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java @@ -68,6 +68,33 @@ public ExternalMetaCacheBudgetManager(OptionalLong globalMaxWeight) { } } + /** + * Whether any Doris meta cache weight bound (global config, catalog level or entry level) + * applies to a catalog with the given properties. Used to decide whether overlapping SDK-side + * metadata caches should stay enabled; parse failures count as ungoverned. + */ + public static boolean appliesWeightGovernance(Map catalogProperties) { + try { + if (fromConfig().getGlobalMaxWeight().isPresent()) { + return true; + } + } catch (RuntimeException e) { + // An unparsable global config cannot create budgets either. + } + if (catalogProperties == null) { + return false; + } + if (catalogProperties.containsKey(CATALOG_MAX_WEIGHT_PROPERTY)) { + return true; + } + for (String key : catalogProperties.keySet()) { + if (key != null && key.startsWith("meta.cache.") && key.endsWith(".max-weight")) { + return true; + } + } + return false; + } + public static ExternalMetaCacheBudgetManager fromConfig() { String configured = Config.external_meta_cache_max_weight; long parsed = CacheSpec.parseWeight( @@ -272,7 +299,10 @@ private void close(EntryBudget entryBudget) { entryBudgets.remove(entryBudget.scope, entryBudget); Bucket catalogBucket = entryBudget.catalogBucket; catalogBucket.liveEntries--; - if (catalogBucket.liveEntries == 0 && catalogBucket.usedWeight == 0L) { + if (catalogBucket.liveEntries == 0) { + // Any residual usedWeight here is already-known-bogus leakage from the force-close + // branch above; stranding the bucket would permanently shrink the catalog budget + // and reject a later max-weight change for this id. catalogBuckets.remove(entryBudget.scope.catalogId, catalogBucket); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java index e5f664a6c4e9b6..364309552a4cb3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java @@ -131,23 +131,38 @@ private static String unsupportedReason(Table table) { return null; } + // Realistic wrapper chains (privileged, fallback-read) are one or two levels deep; the cap + // only guards pathological chains from unbounded work or stack overflow before fail-closed. + private static final int MAX_WRAPPER_DEPTH = 16; + /** The concrete FileStoreTable behind any known wrapper chain, or null. */ private static FileStoreTable unwrap(Table table) { - if (table instanceof DelegatedFileStoreTable) { - return unwrap(((DelegatedFileStoreTable) table).wrapped()); + Table current = table; + for (int depth = 0; depth <= MAX_WRAPPER_DEPTH; depth++) { + if (!(current instanceof DelegatedFileStoreTable)) { + return current instanceof FileStoreTable ? (FileStoreTable) current : null; + } + current = ((DelegatedFileStoreTable) current).wrapped(); } - return table instanceof FileStoreTable ? (FileStoreTable) table : null; + throw new IllegalStateException("Paimon table wrapper chain is too deep"); } /** Uses TableSchema cardinalities only and deliberately never calls FileStoreTable.store(). */ private static long estimateTable(Table table) { + return estimateTable(table, 0); + } + + private static long estimateTable(Table table, int wrapperDepth) { long bytes = 0L; Table current = table; while (true) { + if (wrapperDepth++ > MAX_WRAPPER_DEPTH) { + throw new IllegalStateException("Paimon table wrapper chain is too deep"); + } if (current instanceof FallbackReadFileStoreTable) { bytes = MetaCacheWeightUtils.saturatedAdd(bytes, WRAPPER_WEIGHT); - bytes = MetaCacheWeightUtils.saturatedAdd( - bytes, estimateTable(((FallbackReadFileStoreTable) current).other())); + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, + estimateTable(((FallbackReadFileStoreTable) current).other(), wrapperDepth)); current = ((FallbackReadFileStoreTable) current).wrapped(); continue; } @@ -254,16 +269,23 @@ static long retainedTablePayloadBytes(Table table) { } private static long retainedTablePayloadBytes(Table table, AccountingBudget budget) { + return retainedTablePayloadBytes(table, budget, 0); + } + + private static long retainedTablePayloadBytes(Table table, AccountingBudget budget, int wrapperDepth) { + if (wrapperDepth > MAX_WRAPPER_DEPTH) { + throw new IllegalStateException("Paimon table wrapper chain is too deep"); + } budget.charge(1L); if (table instanceof FallbackReadFileStoreTable) { FallbackReadFileStoreTable fallback = (FallbackReadFileStoreTable) table; return MetaCacheWeightUtils.saturatedAdd( - retainedTablePayloadBytes(fallback.wrapped(), budget), - retainedTablePayloadBytes(fallback.other(), budget)); + retainedTablePayloadBytes(fallback.wrapped(), budget, wrapperDepth + 1), + retainedTablePayloadBytes(fallback.other(), budget, wrapperDepth + 1)); } if (table instanceof DelegatedFileStoreTable) { return retainedTablePayloadBytes( - ((DelegatedFileStoreTable) table).wrapped(), budget); + ((DelegatedFileStoreTable) table).wrapped(), budget, wrapperDepth + 1); } if (!(table instanceof FileStoreTable)) { return 0L; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java index 3fd66ddfacc34a..d3fedb059b85b6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java @@ -18,7 +18,6 @@ package org.apache.doris.datasource.paimon; import org.apache.doris.catalog.Env; -import org.apache.doris.common.Config; import org.apache.doris.common.DdlException; import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.ExternalCatalog; @@ -167,19 +166,7 @@ protected Catalog createCatalog() { /** Whether any Doris meta cache weight bound (global, catalog or entry level) applies. */ private boolean isMetaCacheWeightGoverned() { - if (!"0".equals(Config.external_meta_cache_max_weight.trim())) { - return true; - } - Map properties = catalogProperty.getProperties(); - if (properties.containsKey(ExternalMetaCacheBudgetManager.CATALOG_MAX_WEIGHT_PROPERTY)) { - return true; - } - for (String key : properties.keySet()) { - if (key != null && key.startsWith("meta.cache.") && key.endsWith(".max-weight")) { - return true; - } - } - return false; + return ExternalMetaCacheBudgetManager.appliesWeightGovernance(catalogProperty.getProperties()); } public Map getPaimonOptionsMap() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index 70dd3f225bc50b..ad89c204cd2445 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -271,7 +271,11 @@ PaimonSchemaCacheValue getPaimonSchemaCacheValue( * so its projections must not stay behind in the child entries. */ private boolean isCurrentTableGeneration(NameMapping nameMapping, long tableGeneration) { - PaimonTableCacheValue currentTable = tableEntry.get(nameMapping.getCtlId()).peekIfPresent(nameMapping); + // Revalidation must never re-prepare a retired catalog group (or throw out of a lookup + // that already holds a valid value): an absent group simply means "not current". + MetaCacheEntry tables = + tableEntry.getIfInitialized(nameMapping.getCtlId()); + PaimonTableCacheValue currentTable = tables == null ? null : tables.peekIfPresent(nameMapping); return currentTable != null && currentTable.getGeneration() == tableGeneration; } @@ -290,7 +294,9 @@ private SchemaCacheValue loadSchemaCacheValue(PaimonSchemaCacheKey key) { private SchemaCacheValue loadSchemaCacheValue(PaimonSchemaCacheKey key, Table retainedTable) { ExternalTable dorisTable = findExternalTable(key.getNameMapping(), ENGINE); if (!(dorisTable instanceof PaimonExternalTable)) { - return loadSchemaCacheValue(key); + SchemaCacheValue fallback = loadSchemaCacheValue(key); + fallback.validateSchema(); + return fallback; } dorisTable.setUpdateTime(System.currentTimeMillis()); SchemaCacheValue value = @@ -368,6 +374,14 @@ public void invalidateCatalog(long catalogId) { super.invalidateCatalog(catalogId); } + @Override + public void onCatalogPermanentlyRemoved(long catalogId) { + // A lookup racing the drop may re-insert a fence owner after invalidateCatalog cleaned + // the map; this hook runs even when the entry group is already retired and the id is + // never reused, so the owners cannot leak for the FE lifetime. + latestObservedFences.keySet().removeIf(owner -> owner.nameMapping.getCtlId() == catalogId); + } + @Override public void invalidateCatalogEntries(long catalogId) { latestObservedFences.keySet().removeIf(owner -> owner.nameMapping.getCtlId() == catalogId); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractIcebergProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractIcebergProperties.java index 9a3a5ef5d2a318..b97889dba4c5ad 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractIcebergProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractIcebergProperties.java @@ -20,6 +20,7 @@ import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.iceberg.IcebergExternalCatalog; import org.apache.doris.datasource.metacache.CacheSpec; +import org.apache.doris.datasource.metacache.ExternalMetaCacheBudgetManager; import org.apache.doris.datasource.property.common.IcebergAwsAssumeRoleProperties; import org.apache.doris.datasource.property.storage.AbstractS3CompatibleProperties; import org.apache.doris.datasource.property.storage.S3Properties; @@ -172,7 +173,11 @@ protected void addManifestCacheProperties(Map catalogProps) { } // default enable io manifest cache if the meta.cache.manifest is enabled - if (!hasIoManifestCacheEnabled) { + if (!hasIoManifestCacheEnabled + && !ExternalMetaCacheBudgetManager.appliesWeightGovernance(catalogProps)) { + // Under Doris weight governance the SDK-side manifest content cache would retain up + // to its own max-total-bytes per FileIO outside the entry/catalog/global budget, so + // it is not auto-enabled; an explicit io.manifest.cache-enabled still wins. CacheSpec manifestCacheSpec = CacheSpec.fromProperties(catalogProps, CacheSpec.propertySpecBuilder() .enable(IcebergExternalCatalog.ICEBERG_MANIFEST_CACHE_ENABLE, IcebergExternalCatalog.DEFAULT_ICEBERG_MANIFEST_CACHE_ENABLE) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java index cced2c176e4e57..9cc96f58b7d6aa 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/hive/HiveMetaStoreCacheTest.java @@ -395,6 +395,63 @@ public void testReplacementSizingWithAliasKeyPreservesRetainedKeyWidth() { Assertions.assertEquals(loadedEstimate, replacedEstimate); } + @Test + public void testPartitionColumnWidthFormulaAgainstJolOwnedGraph() throws Exception { + // The per-partition column-width term must track a real narrow-vs-wide literal graph. + List narrowTypes = Collections.singletonList(Type.STRING); + List wideTypes = new ArrayList<>(); + for (int i = 0; i < 4; i++) { + wideTypes.add(Type.STRING); + } + HiveExternalMetaCache.PartitionValueCacheKey narrowKey = + new HiveExternalMetaCache.PartitionValueCacheKey( + NameMapping.createForTest("db", "tbl"), narrowTypes); + HiveExternalMetaCache.PartitionValueCacheKey wideKey = + new HiveExternalMetaCache.PartitionValueCacheKey( + NameMapping.createForTest("db", "tbl"), wideTypes); + HiveExternalMetaCache.HivePartitionValues narrow = realPartitionValues(narrowTypes, 32, 16); + HiveExternalMetaCache.HivePartitionValues wide = realWidePartitionValues(wideTypes, 32, 16); + long narrowEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry( + narrowKey, narrow).getBytes(); + long wideEstimate = HiveCacheSizeEstimator.estimatePartitionValuesEntry( + wideKey, wide).getBytes(); + EstimatorCalibrationAssertions.assertConservativeDelta( + "hive wide partition columns", narrowEstimate, wideEstimate, narrow, wide); + } + + private HiveExternalMetaCache.HivePartitionValues realWidePartitionValues( + List types, int partitionCount, int valueLength) throws Exception { + Map items = new HashMap<>(); + HashBiMap names = HashBiMap.create(); + Map> values = new HashMap<>(); + for (int index = 0; index < partitionCount; index++) { + long id = index + 1L; + // Mirror production: the per-column decoded values are segments of the partition + // name (their total length tracks the name), and the PartitionKey literals share + // the same String references as the decoded value list. + int segmentLength = Math.max(1, valueLength / types.size()); + List partitionValues = new ArrayList<>(); + List rawValues = new ArrayList<>(); + StringBuilder name = new StringBuilder("p").append(index); + for (int column = 0; column < types.size(); column++) { + String segment = "c" + column + "_" + index + + String.join("", Collections.nCopies(segmentLength, "x")); + name.append('/').append(segment); + rawValues.add(segment); + partitionValues.add(new PartitionValue(segment)); + } + PartitionKey partitionKey = PartitionKey.createListPartitionKeyWithTypes( + partitionValues, types, true); + items.put(id, new ListPartitionItem(Collections.singletonList(partitionKey))); + names.put(name.toString(), id); + values.put(id, rawValues); + } + HiveExternalMetaCache.HivePartitionValues result = + new HiveExternalMetaCache.HivePartitionValues(items, names, values); + result.sealForPublication(); + return result; + } + @Test public void testEmptyTableKeyWidthFormulaAgainstJolOwnedGraph() throws Exception { // An empty partitioned table still retains the key's immutable type list; the estimate diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 2818cca80c016e..8552d7ba5e3015 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -504,6 +504,36 @@ public void testFileIoCredentialPayloadScalesWithConfiguration() { + ", large=" + largeEstimate, largeEstimate - smallEstimate >= 64 * 1024 - 16); } + @Test + public void testFieldDefaultPayloadScalesWithLength() { + // v3 field defaults retain arbitrary scalar payloads; with a fixed field count the + // estimate must grow with the default length. + long shortEstimate = fieldDefaultEstimate(16); + long longEstimate = fieldDefaultEstimate(64 * 1024); + Assert.assertTrue("default payload must be charged: short=" + shortEstimate + + ", long=" + longEstimate, longEstimate - shortEstimate >= 60 * 1024); + } + + private long fieldDefaultEstimate(int defaultLength) { + Schema schema = new Schema( + Types.NestedField.required(1, "id", Types.IntegerType.get()), + Types.NestedField.optional("defaulted").withId(2) + .ofType(Types.StringType.get()) + .withInitialDefault(repeatedCharacter('d', defaultLength)) + .withWriteDefault(repeatedCharacter('w', defaultLength)) + .build()); + TableMetadata metadata = TableMetadata.newTableMetadata(schema, + PartitionSpec.unpartitioned(), "file:/warehouse/db/tbl", + Collections.singletonMap("format-version", "3")); + metadata = TableMetadata.buildFrom(metadata).discardChanges() + .withMetadataLocation("/metadata/field-defaults.json").build(); + MetaCacheSizeEstimate estimate = new IcebergTableCacheValue( + new BaseTable(new StaticTableOperations(metadata, null), "db.tbl")) + .prepareForCachePublication(NameMapping.createForTest(1L, "db", "tbl")); + Assert.assertTrue(estimate.getIncompleteReason(), estimate.isComplete()); + return estimate.getBytes(); + } + @Test public void testUnknownTransformPayloadScalesWithTokenLength() { // Unrecognized transform tokens are preserved verbatim; with a fixed field count the diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 45dee72f7cc96f..45961473e47d7f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -555,6 +555,22 @@ public void testSchemaEntryDoesNotAutoRefreshOutsideAuthenticatorScope() { } } + @Test + public void testFieldDescriptionPayloadScalesWithLength() throws Exception { + assertTableDeltaAgainstJol("paimon field descriptions", + newTableWithPayloadType("desc", rowWithDescriptions(8, 16)), + newTableWithPayloadType("desc", rowWithDescriptions(8, 4096))); + } + + private RowType rowWithDescriptions(int fieldCount, int descriptionLength) { + List fields = new ArrayList<>(); + for (int index = 0; index < fieldCount; index++) { + fields.add(new DataField(300 + index, "desc_" + index, new IntType(), + repeatedCharacter('c', descriptionLength))); + } + return new RowType(fields); + } + @Test public void testFieldDefaultValuePayloadScalesWithLength() throws Exception { // Field defaults are retained by every DataField; with a fixed field count the estimate diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractIcebergPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractIcebergPropertiesTest.java index 228afe85b618f4..91a8c31943a1e4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractIcebergPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractIcebergPropertiesTest.java @@ -122,4 +122,34 @@ void testExecutionAuthenticatorNotNull() { TestIcebergProperties properties = new TestIcebergProperties(new HashMap<>(), mockCatalog); Assertions.assertNotNull(properties.executionAuthenticator); } + + @Test + void testWeightGovernanceSkipsSdkManifestCacheAutoEnable() { + Catalog mockCatalog = Mockito.mock(Catalog.class); + // Without weight governance, enabling the Doris manifest entry auto-enables the SDK + // manifest content cache. + Map plain = new HashMap<>(); + plain.put("meta.cache.iceberg.manifest.enable", "true"); + TestIcebergProperties ungoverned = new TestIcebergProperties(plain, mockCatalog); + ungoverned.initializeCatalog("c1", Collections.emptyList()); + Assertions.assertEquals("true", + ungoverned.getCapturedCatalogProps().get("io.manifest.cache-enabled")); + + // Under weight governance the SDK cache would retain manifests outside the Doris + // budget, so the auto-enable is skipped. + Map governed = new HashMap<>(plain); + governed.put("meta.cache.max-weight", "128MB"); + TestIcebergProperties weightGoverned = new TestIcebergProperties(governed, mockCatalog); + weightGoverned.initializeCatalog("c2", Collections.emptyList()); + Assertions.assertNull( + weightGoverned.getCapturedCatalogProps().get("io.manifest.cache-enabled")); + + // An explicit user choice always wins. + Map explicit = new HashMap<>(governed); + explicit.put("io.manifest.cache-enabled", "true"); + TestIcebergProperties userConfigured = new TestIcebergProperties(explicit, mockCatalog); + userConfigured.initializeCatalog("c3", Collections.emptyList()); + Assertions.assertEquals("true", + userConfigured.getCapturedCatalogProps().get("io.manifest.cache-enabled")); + } } diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy index 0c2bebc52fed56..f322a75f3e44f1 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy @@ -178,7 +178,8 @@ suite("test_iceberg_table_meta_cache", "p0,external,doris,external_docker,extern // alter wrong catalog property test { sql """alter catalog ${catalog_name_no_cache} set properties ("meta.cache.iceberg.table.ttl-second" = "-2")""" - exception "is wrong" + // Strict cache-property validation runs before the legacy validators on ALTER. + exception "must be >= -1" } // alter catalog property, disable meta cache sql """alter catalog ${catalog_name_no_cache} set properties ("meta.cache.iceberg.table.ttl-second" = "0")""" From da6f21e71e9bc1c41f2438a1ba7705eadbab97dc Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 21 Aug 2026 10:56:46 +0800 Subject: [PATCH 27/45] [fix](fe) Keep Hive generations across rename, fence the ALTER property window and parse governance detection - Catalog rename re-adds the same id, so its cleanup no longer runs the permanent-drop hook (Hive statement generations stay monotonic); only DROP and its replay do - The ALTER tentative-property window now runs under the per-catalog lifecycle fence, so a concurrent first use of a sibling routed engine cannot observe the candidate catalog max-weight while another engine's group still pins the committed one - SDK-cache governance detection only counts weight keys whose value actually parses, so an invalid or obsolete persisted key that runtime sanitization drops cannot disable overlapping SDK caches; a valid zero still counts as an explicit bound --- .../apache/doris/datasource/CatalogMgr.java | 122 ++++++++++++------ .../datasource/ExternalMetaCacheMgr.java | 15 +++ .../ExternalMetaCacheBudgetManager.java | 22 +++- 3 files changed, 111 insertions(+), 48 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index 7f9d8f39f04a1d..c0a92fa3c96331 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -138,6 +138,10 @@ private RemovedCatalog removeCatalog(long catalogId) { } private void cleanupRemovedCatalog(RemovedCatalog removedCatalog) { + cleanupRemovedCatalog(removedCatalog, true); + } + + private void cleanupRemovedCatalog(RemovedCatalog removedCatalog, boolean permanentRemoval) { if (removedCatalog == null) { return; } @@ -147,7 +151,14 @@ private void cleanupRemovedCatalog(RemovedCatalog removedCatalog) { if (ctx != null) { ctx.removeLastDBOfCatalog(removedCatalog.catalogName); } - Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogPermanently(removedCatalog.catalogId); + if (permanentRemoval) { + Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogPermanently(removedCatalog.catalogId); + } else { + // A rename re-adds the same catalog id afterwards: engine side state such as the + // Hive statement-scoped generation counters must survive, or a statement planned + // across the rename could reuse stale file tasks under a restarted generation. + Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalog(removedCatalog.catalogId); + } Env.getCurrentEnv().getQueryStats().clear(removedCatalog.catalogId); LOG.info("Removed catalog with id {}, name {}", removedCatalog.catalogId, removedCatalog.catalogName); } @@ -333,7 +344,7 @@ public void alterCatalogName(String catalogName, String newCatalogName) throws U } finally { writeUnlock(); } - cleanupRemovedCatalog(removedCatalog); + cleanupRemovedCatalog(removedCatalog, false); if (removedCatalog == null) { throw new IllegalStateException("No catalog found with name: " + catalogName); } @@ -581,7 +592,7 @@ public void replayAlterCatalogName(CatalogLog log) { } finally { writeUnlock(); } - cleanupRemovedCatalog(removedCatalog); + cleanupRemovedCatalog(removedCatalog, false); if (removedCatalog == null) { throw new IllegalStateException("No catalog found with id: " + log.getCatalogId()); @@ -667,55 +678,80 @@ public void replayAlterCatalogProps(CatalogLog log, Map oldPrope try { CatalogIf catalog = idToCatalog.get(log.getCatalogId()); if (catalog instanceof ExternalCatalog) { - Map newProps = log.getNewProps(); - if (!isReplay) { - boolean tentativelyMutated = false; - try { - ExternalCatalog externalCatalog = (ExternalCatalog) catalog; - validateSuppliedCacheProperties(externalCatalog, oldProperties, newProps); - boolean validatedWithoutMutation = externalCatalog.validatePropertiesBeforeUpdate( - oldProperties, newProps); - if (!validatedWithoutMutation) { - externalCatalog.tryModifyCatalogProps(newProps); - tentativelyMutated = true; - externalCatalog.checkProperties(); - } - } catch (Exception validationException) { - // Only legacy validators publish a tentative candidate. Detached validators - // leave the live CatalogProperty untouched while concurrent initialization runs. - if (oldProperties != null && tentativelyMutated) { - Env currentEnv = Env.getCurrentEnv(); - ExternalMetaCacheMgr cacheMgr = currentEnv == null - ? null : currentEnv.getExtMetaCacheMgr(); - if (cacheMgr == null) { - ((ExternalCatalog) catalog).rollBackCatalogProps(oldProperties); - } else { - cacheMgr.rollbackCatalogProperties( - (ExternalCatalog) catalog, oldProperties); - } - } - if (validationException instanceof DdlException) { - throw (DdlException) validationException; + // The tentative property window (legacy validators mutate the live CatalogProperty + // before commit/rollback) must be invisible to a concurrent lazy cache-group + // initialization of a sibling engine, or budget creation can observe the candidate + // catalog max-weight while another engine's group still pins the committed one. + // The lifecycle stripe is reentrant, so nested rollback/removal re-enters safely. + try { + Env.getCurrentEnv().getExtMetaCacheMgr().withCatalogLifecycleLock(catalog.getId(), () -> { + try { + alterExternalCatalogPropsFenced((ExternalCatalog) catalog, log, + oldProperties, isReplay); + } catch (DdlException e) { + throw new IllegalStateException(e); } - throw new DdlException("Invalid catalog properties: " - + validationException.getMessage(), validationException); + return null; + }); + } catch (IllegalStateException e) { + if (e.getCause() instanceof DdlException) { + throw (DdlException) e.getCause(); } - } else { - ((ExternalCatalog) catalog).tryModifyCatalogProps(newProps); - } - if (newProps.containsKey(METADATA_REFRESH_INTERVAL_SEC)) { - long catalogId = catalog.getId(); - Integer metadataRefreshIntervalSec = Integer.valueOf(newProps.get(METADATA_REFRESH_INTERVAL_SEC)); - Integer[] sec = {metadataRefreshIntervalSec, metadataRefreshIntervalSec}; - Env.getCurrentEnv().getRefreshManager().addToRefreshMap(catalogId, sec); + throw e; } + } else { + catalog.modifyCatalogProps(log.getNewProps()); } - catalog.modifyCatalogProps(log.getNewProps()); } finally { writeUnlock(); } } + private void alterExternalCatalogPropsFenced(ExternalCatalog externalCatalog, CatalogLog log, + Map oldProperties, boolean isReplay) throws DdlException { + Map newProps = log.getNewProps(); + if (!isReplay) { + boolean tentativelyMutated = false; + try { + validateSuppliedCacheProperties(externalCatalog, oldProperties, newProps); + boolean validatedWithoutMutation = externalCatalog.validatePropertiesBeforeUpdate( + oldProperties, newProps); + if (!validatedWithoutMutation) { + externalCatalog.tryModifyCatalogProps(newProps); + tentativelyMutated = true; + externalCatalog.checkProperties(); + } + } catch (Exception validationException) { + // Only legacy validators publish a tentative candidate. Detached validators + // leave the live CatalogProperty untouched while concurrent initialization runs. + if (oldProperties != null && tentativelyMutated) { + Env currentEnv = Env.getCurrentEnv(); + ExternalMetaCacheMgr cacheMgr = currentEnv == null + ? null : currentEnv.getExtMetaCacheMgr(); + if (cacheMgr == null) { + externalCatalog.rollBackCatalogProps(oldProperties); + } else { + cacheMgr.rollbackCatalogProperties(externalCatalog, oldProperties); + } + } + if (validationException instanceof DdlException) { + throw (DdlException) validationException; + } + throw new DdlException("Invalid catalog properties: " + + validationException.getMessage(), validationException); + } + } else { + externalCatalog.tryModifyCatalogProps(newProps); + } + if (newProps.containsKey(METADATA_REFRESH_INTERVAL_SEC)) { + long catalogId = externalCatalog.getId(); + Integer metadataRefreshIntervalSec = Integer.valueOf(newProps.get(METADATA_REFRESH_INTERVAL_SEC)); + Integer[] sec = {metadataRefreshIntervalSec, metadataRefreshIntervalSec}; + Env.getCurrentEnv().getRefreshManager().addToRefreshMap(catalogId, sec); + } + externalCatalog.modifyCatalogProps(newProps); + } + public void unregisterExternalTable(String dbName, String tableName, String catalogName, boolean ignoreIfExists) throws DdlException { CatalogIf catalog = nameToCatalog.get(catalogName); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index b1372586bd6056..7b4f208fe9cdb7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -390,6 +390,21 @@ public void invalidateCatalogByEngine(long catalogId, String engine) { () -> cache.invalidateCatalogEntries(catalogId))); } + /** + * Run an action under the same per-catalog lifecycle fence that guards lazy initialization + * and group retirement. The stripe lock is reentrant, so fenced actions may call back into + * removeCatalog/rollbackCatalogProperties for the same catalog. + */ + public T withCatalogLifecycleLock(long catalogId, java.util.function.Supplier action) { + Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); + lifecycleLock.lock(); + try { + return action.get(); + } finally { + lifecycleLock.unlock(); + } + } + public void removeCatalog(long catalogId) { Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); lifecycleLock.lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java index 0b5cdc7649785b..e508423e86f2c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/ExternalMetaCacheBudgetManager.java @@ -84,12 +84,24 @@ public static boolean appliesWeightGovernance(Map catalogPropert if (catalogProperties == null) { return false; } - if (catalogProperties.containsKey(CATALOG_MAX_WEIGHT_PROPERTY)) { - return true; - } - for (String key : catalogProperties.keySet()) { - if (key != null && key.startsWith("meta.cache.") && key.endsWith(".max-weight")) { + // Only values that actually parse can create budgets; an invalid or obsolete key that + // runtime sanitization will drop must not disable overlapping SDK caches. A valid zero + // still counts: it is an explicit (disabled) weight bound. + for (Map.Entry property : catalogProperties.entrySet()) { + String key = property.getKey(); + if (key == null) { + continue; + } + boolean weightKey = key.equals(CATALOG_MAX_WEIGHT_PROPERTY) + || (key.startsWith("meta.cache.") && key.endsWith(".max-weight")); + if (!weightKey) { + continue; + } + try { + CacheSpec.parseWeight(property.getValue(), key, false, 0L); return true; + } catch (RuntimeException e) { + // Unparsable: sanitized away at runtime, no budget results. } } return false; From 1b62b6bdd2a32fcb49b8fe4bc40ecda288617e73 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 21 Aug 2026 11:49:02 +0800 Subject: [PATCH 28/45] [fix](be) Restrict the meta cache stats legacy fallback to FE-side rejections Only an RPC that reached the FE and came back non-OK indicates a legacy FE without the weight columns; transport-level failures are surfaced directly instead of being masked by a legacy retry that would silently blank the weight telemetry, and a failed retry returns the original error --- ...chema_catalog_meta_cache_stats_scanner.cpp | 37 +++++++++++++++---- .../schema_catalog_meta_cache_stats_scanner.h | 3 +- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp index 255813230aa266..22e0e0196ce22e 100644 --- a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp +++ b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.cpp @@ -84,7 +84,9 @@ Status SchemaCatalogMetaCacheStatsScanner::start(RuntimeState* state) { } Status SchemaCatalogMetaCacheStatsScanner::_fetch_from_fe(size_t column_count, - TFetchSchemaTableDataResult* result) { + TFetchSchemaTableDataResult* result, + bool* fe_rejected) { + *fe_rejected = false; TSchemaTableRequestParams schema_table_request_params; for (size_t i = 0; i < column_count; i++) { schema_table_request_params.__isset.columns_name = true; @@ -96,28 +98,47 @@ Status SchemaCatalogMetaCacheStatsScanner::_fetch_from_fe(size_t column_count, request.__set_schema_table_name(TSchemaTableName::CATALOG_META_CACHE_STATS); request.__set_schema_table_params(schema_table_request_params); + // A transport-level failure says nothing about the FE's column support and must be + // surfaced to the caller instead of being retried with the legacy projection. RETURN_IF_ERROR(ThriftRpcHelper::rpc( _fe_addr.hostname, _fe_addr.port, [&request, result](FrontendServiceConnection& client) { client->fetchSchemaTableData(*result, request); }, _rpc_timeout)); - return Status::create(result->status); + Status fe_status = Status::create(result->status); + // The RPC itself succeeded, so a non-OK status was produced by the FE handler: this is the + // signal a legacy FE without the weight columns emits for an unknown projection name. + *fe_rejected = !fe_status.ok(); + return fe_status; } Status SchemaCatalogMetaCacheStatsScanner::_get_meta_cache_from_fe() { TFetchSchemaTableDataResult result; - Status status = _fetch_from_fe(_s_tbls_columns.size(), &result); + bool fe_rejected = false; + Status status = _fetch_from_fe(_s_tbls_columns.size(), &result, &fe_rejected); if (!status.ok()) { - LOG(INFO) << "fetch catalog meta cache stats from FE(" << _fe_addr.hostname - << ") with all columns failed, retrying with the legacy column set: " << status; - result = TFetchSchemaTableDataResult(); - status = _fetch_from_fe(kLegacyMetaCacheStatsColumnCount, &result); - if (!status.ok()) { + if (!fe_rejected) { + // Transport error from a current FE: do not mask it with a legacy retry that would + // silently blank the weight telemetry columns. LOG(WARNING) << "fetch catalog meta cache stats from FE(" << _fe_addr.hostname << ") failed, errmsg=" << status; return status; } + Status first_status = status; + LOG(INFO) << "FE(" << _fe_addr.hostname + << ") rejected the full catalog meta cache stats projection, retrying with the " + "legacy column set: " + << first_status; + result = TFetchSchemaTableDataResult(); + status = _fetch_from_fe(kLegacyMetaCacheStatsColumnCount, &result, &fe_rejected); + if (!status.ok()) { + // The legacy projection failed too, so the first rejection was not a legacy FE; + // surface the original error rather than the retry artifact. + LOG(WARNING) << "fetch catalog meta cache stats from FE(" << _fe_addr.hostname + << ") failed, errmsg=" << first_status; + return first_status; + } } std::vector result_data = result.data_batch; diff --git a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h index 7a2339baf44335..006e1459edfa91 100644 --- a/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h +++ b/be/src/information_schema/schema_catalog_meta_cache_stats_scanner.h @@ -41,7 +41,8 @@ class SchemaCatalogMetaCacheStatsScanner : public SchemaScanner { private: Status _get_meta_cache_from_fe(); - Status _fetch_from_fe(size_t column_count, TFetchSchemaTableDataResult* result); + Status _fetch_from_fe(size_t column_count, TFetchSchemaTableDataResult* result, + bool* fe_rejected); TNetworkAddress _fe_addr; From 02002f7eca85029ee094ce2df1284e35f5921886 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 21 Aug 2026 12:31:51 +0800 Subject: [PATCH 29/45] [fix](fe) Retry lookups across a contended policy handoff and stop equating stateful same-class resources - A lookup whose re-prepare loses the lifecycle-fence race against a cache-policy ALTER now retries within a bounded window instead of failing a valid catalog; the preparer stays nonblocking so retirement can never deadlock against nested cache loaders - Only plaintext Iceberg encryption managers are equivalent across instances; other same-class managers can carry KMS/session state and fail closed so projections rebind. Location providers keep same-class equivalence because they are deterministic functions of the metadata location and FileIO configuration this predicate already proved equal --- .../iceberg/IcebergTableCacheValue.java | 22 +++++++++++-- .../metacache/AbstractExternalMetaCache.java | 28 ++++++++++++++--- .../iceberg/IcebergExternalMetaCacheTest.java | 31 +++++++++++++++++++ .../paimon/PaimonExternalMetaCacheTest.java | 24 ++++++++++++++ 4 files changed, 98 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java index 84dcb045d70764..da7740b425f9b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java @@ -156,8 +156,8 @@ static boolean sharesOperationalResources(Table left, Table right) { return false; } return sameFileIo(left.io(), right.io()) - && sameResource(left.encryption(), right.encryption()) - && sameResource(left.locationProvider(), right.locationProvider()); + && sameEncryption(left.encryption(), right.encryption()) + && sameLocationProvider(left.locationProvider(), right.locationProvider()); } private static boolean sameFileIo(FileIO left, FileIO right) { @@ -183,7 +183,23 @@ private static Object storageCredentials(FileIO fileIO) { ? ((SupportsStorageCredentials) fileIO).credentials() : null; } - private static boolean sameResource(Object left, Object right) { + private static boolean sameEncryption( + org.apache.iceberg.encryption.EncryptionManager left, + org.apache.iceberg.encryption.EncryptionManager right) { + if (left == right) { + return true; + } + // Plaintext managers are stateless, so any two instances are equivalent. Every other + // manager can hold KMS clients, sessions or key state that is not observable from the + // outside: fail closed so projections rebind to the fresh handle's manager. + return left instanceof org.apache.iceberg.encryption.PlaintextEncryptionManager + && right instanceof org.apache.iceberg.encryption.PlaintextEncryptionManager; + } + + private static boolean sameLocationProvider(Object left, Object right) { + // This predicate only runs after the metadata file location and the FileIO configuration + // proved equal; a location provider is constructed deterministically from exactly that + // state (table location plus properties), so same-class instances are equivalent here. return left == right || (left != null && right != null && left.getClass() == right.getClass()); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java index 30c84cc3715c85..2f012d6e1b2c87 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java @@ -330,14 +330,34 @@ protected final ExternalTable findExternalTable(NameMapping nameMapping, String nameMapping.getLocalTblName(), engineNameForError)); } + // A contended cache-policy handoff resolves within this window; see requireCatalogEntryGroup. + private static final long PREPARE_RETRY_WINDOW_NANOS = 2_000_000_000L; + private static final long PREPARE_RETRY_SLEEP_MS = 50L; + private CatalogEntryGroup requireCatalogEntryGroup(long catalogId) { CatalogEntryGroup group = catalogEntries.get(catalogId); if (group == null && catalogPreparer != null) { // The caller prepared the catalog before capturing this engine, but a cache-policy - // ALTER retired the group in between. Re-prepare once under the lifecycle fence so - // the lookup observes the new policy instead of failing a valid catalog. - catalogPreparer.accept(catalogId); - group = catalogEntries.get(catalogId); + // ALTER retired the group in between. Re-prepare under the lifecycle fence so the + // lookup observes the new policy instead of failing a valid catalog. The preparer + // never blocks on the fence (a nested default loader may hold a Caffeine bin lock + // that retirement itself needs), so a contended handoff is absorbed with a bounded + // sleep-and-retry: the ALTER finishes within the window, or the lookup fails as + // before without any deadlock. + long deadlineNanos = System.nanoTime() + PREPARE_RETRY_WINDOW_NANOS; + while (true) { + catalogPreparer.accept(catalogId); + group = catalogEntries.get(catalogId); + if (group != null || System.nanoTime() >= deadlineNanos) { + break; + } + try { + Thread.sleep(PREPARE_RETRY_SLEEP_MS); + } catch (InterruptedException e) { + Thread.currentThread().interrupt(); + break; + } + } } if (group == null) { throw new IllegalStateException(String.format( diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 8552d7ba5e3015..e7dafd059735b1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -562,6 +562,37 @@ private long unknownTransformEstimate(int tokenLength) { return estimate.getBytes(); } + @Test + public void testSameClassResourceReplacementsAreNotEquatedForEncryption() { + // Two same-class encryption managers can hold different KMS state: only plaintext + // managers are stateless enough to be equivalent across instances. + Table first = Mockito.mock(Table.class); + Table second = Mockito.mock(Table.class); + FileIO sharedIo = Mockito.mock(FileIO.class); + Mockito.when(sharedIo.properties()).thenReturn(Collections.emptyMap()); + Mockito.when(first.io()).thenReturn(sharedIo); + Mockito.when(second.io()).thenReturn(sharedIo); + org.apache.iceberg.encryption.EncryptionManager firstManager = + Mockito.mock(org.apache.iceberg.encryption.EncryptionManager.class); + org.apache.iceberg.encryption.EncryptionManager secondManager = + Mockito.mock(org.apache.iceberg.encryption.EncryptionManager.class); + Mockito.when(first.encryption()).thenReturn(firstManager); + Mockito.when(second.encryption()).thenReturn(secondManager); + Assert.assertFalse("distinct stateful managers must not be equated", + IcebergTableCacheValue.sharesOperationalResources(first, second)); + + Mockito.when(second.encryption()).thenReturn(firstManager); + Assert.assertTrue("identical managers are equivalent", + IcebergTableCacheValue.sharesOperationalResources(first, second)); + + Mockito.when(first.encryption()).thenReturn( + org.apache.iceberg.encryption.PlaintextEncryptionManager.instance()); + Mockito.when(second.encryption()).thenReturn( + Mockito.mock(org.apache.iceberg.encryption.PlaintextEncryptionManager.class)); + Assert.assertTrue("plaintext managers are stateless and equivalent across instances", + IcebergTableCacheValue.sharesOperationalResources(first, second)); + } + @Test public void testSchemaEntryDoesNotAutoRefreshOutsideAuthenticatorScope() { ExecutorService executor = Executors.newSingleThreadExecutor(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 45961473e47d7f..5c0e5bf9d8e570 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -539,6 +539,30 @@ public void testTableEstimateReservesFileIoAllowancePerOwner() throws Exception snapshotEstimate >= 32L * 1024L); } + @Test + public void testContendedPolicyHandoffRetriesInsteadOfFailingTheLookup() { + // A cache-policy ALTER may hold the lifecycle fence while a lookup re-prepares; the + // nonblocking preparer then returns empty-handed and the lookup must retry within a + // bounded window instead of failing an otherwise valid catalog. + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try { + AtomicInteger prepareAttempts = new AtomicInteger(); + cache.bindCatalogPreparer(catalogId -> { + // Simulate a fence contended for the first attempts, then a successful handoff. + if (prepareAttempts.incrementAndGet() >= 3) { + cache.initCatalog(catalogId, Collections.emptyMap()); + } + }); + Assert.assertNotNull(cache.entry(7L, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class)); + Assert.assertEquals(3, prepareAttempts.get()); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + @Test public void testSchemaEntryDoesNotAutoRefreshOutsideAuthenticatorScope() { ExecutorService executor = Executors.newSingleThreadExecutor(); From 76ecb3ecf3be1c07e6edb64c23c1a211d4a7f95c Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 21 Aug 2026 13:38:42 +0800 Subject: [PATCH 30/45] [fix](fe) Bind the execution authenticator to each published table generation A property ALTER resets the catalog (nulling its authenticator and closing SDK resources) before retiring the cache group, so a lookup that already captured a table generation must not resolve authentication from the resetting catalog: Paimon and Iceberg table values now capture the authenticator active at load time, and fence/snapshot/schema loads on that generation reuse it (projection-internal schema resolution runs pre-authenticated instead of re-resolving the catalog); values predating the capture fall back to the current authenticator --- .../iceberg/IcebergExternalMetaCache.java | 27 ++++- .../iceberg/IcebergTableCacheValue.java | 15 +++ .../paimon/PaimonExternalMetaCache.java | 99 ++++++++++++++++--- .../paimon/PaimonTableCacheValue.java | 18 ++++ .../paimon/PaimonExternalMetaCacheTest.java | 38 +++++++ 5 files changed, 179 insertions(+), 18 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index e79afdff486f68..9ef7a74df83920 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -176,7 +176,7 @@ public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { IcebergSnapshotEntryKey.tryCreate(nameMapping, retainedTable); if (!optionalKey.isPresent()) { boolean isolateForQueries = tableValue.isQueryIsolationPrepared(); - return executeAuthenticated(nameMapping.getCtlId(), + return executeForGeneration(tableValue, nameMapping.getCtlId(), () -> loadSnapshotProjection( dorisTable, isolateForQueries ? tableValue.newQueryScopedTable() @@ -190,7 +190,7 @@ public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { boolean isolateForQueries = tableValue.isQueryIsolationPrepared() || entry.isWeightAccounting(); Function projectionLoader = - ignored -> executeAuthenticated(nameMapping.getCtlId(), () -> { + ignored -> executeForGeneration(tableValue, nameMapping.getCtlId(), () -> { Table projectionTable = isolateForQueries ? tableValue.newQueryScopedTable() : tableValue.getIcebergTable(); IcebergSnapshotCacheValue value = loadSnapshotProjection( @@ -320,6 +320,9 @@ private IcebergTableCacheValue loadTableCacheValue(NameMapping nameMapping) { return executeAuthenticated(catalog, () -> { Table table = ops.loadTable(nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName()); IcebergTableCacheValue value = new IcebergTableCacheValue(table); + if (catalog instanceof ExternalCatalog) { + value.bindAuthenticator(((ExternalCatalog) catalog).getExecutionAuthenticator()); + } MetaCacheEntry currentEntry = tableEntry.getIfInitialized(nameMapping.getCtlId()); if (currentEntry != null && currentEntry.isWeightAccounting()) { @@ -466,6 +469,26 @@ private IcebergMetadataOps resolveMetadataOps(CatalogIf catalog) { throw new RuntimeException("Only support 'hms' and 'iceberg' type for iceberg table"); } + /** + * Execute on the authenticated context captured with the table generation, falling back to + * the catalog's current authenticator for values that predate the capture. A concurrent + * property ALTER resets the catalog before retiring the group, so a lookup that already + * owns the old generation must not resolve authentication from the resetting catalog. + */ + private T executeForGeneration( + IcebergTableCacheValue tableValue, long catalogId, Callable task) { + org.apache.doris.common.security.authentication.ExecutionAuthenticator authenticator = + tableValue.getAuthenticator(); + if (authenticator == null) { + return executeAuthenticated(catalogId, task); + } + try { + return authenticator.execute(task); + } catch (Exception e) { + throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); + } + } + private T executeAuthenticated(long catalogId, Callable task) { CatalogIf catalog = getCatalog(catalogId); if (catalog == null) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java index da7740b425f9b2..dbb5e4c71148d8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java @@ -29,9 +29,14 @@ import java.util.Objects; import java.util.Optional; +import javax.annotation.Nullable; public class IcebergTableCacheValue { private volatile Table icebergTable; + // The execution authenticator active when this generation was loaded; see the Paimon + // counterpart for the concurrent catalog-reset rationale. + @Nullable + private volatile org.apache.doris.common.security.authentication.ExecutionAuthenticator authenticator; private String retainedCurrentSnapshotJson; private volatile boolean queryIsolationPrepared; private long retainedTablePayloadBytes; @@ -41,6 +46,16 @@ public IcebergTableCacheValue(Table icebergTable) { this.icebergTable = IcebergSnapshotCacheValue.retainTableGeneration(icebergTable); } + void bindAuthenticator( + @Nullable org.apache.doris.common.security.authentication.ExecutionAuthenticator authenticator) { + this.authenticator = authenticator; + } + + @Nullable + org.apache.doris.common.security.authentication.ExecutionAuthenticator getAuthenticator() { + return authenticator; + } + public Table getIcebergTable() { Table retainedTable = icebergTable; return queryIsolationPrepared || IcebergSnapshotCacheValue.isNonGrowingGeneration(retainedTable) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index ad89c204cd2445..8039b7daa93854 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -84,7 +84,7 @@ public PaimonExternalMetaCache(ExecutorService refreshExecutor, ExternalMetaCach super(ENGINE, refreshExecutor, budgetManager); tableLoader = new PaimonTableLoader(); latestSnapshotProjectionLoader = new PaimonLatestSnapshotProjectionLoader( - new PaimonPartitionInfoLoader(), this::getPaimonSchemaCacheValue); + new PaimonPartitionInfoLoader(), this::getPaimonSchemaCacheValuePreAuthenticated); tableEntry = registerEntry(MetaCacheEntryDef.of(ENTRY_TABLE, NameMapping.class, PaimonTableCacheValue.class, this::loadTableCacheValue, defaultEntryCacheSpec(), MetaCacheEntryInvalidation.forNameMapping(nameMapping -> nameMapping)) @@ -116,12 +116,12 @@ public PaimonSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); MetaCacheEntry tables = tableEntry.get(nameMapping.getCtlId()); PaimonTableCacheValue tableValue = tables.get(nameMapping); - PaimonSnapshot fence = loadLatestSnapshotFence(nameMapping, tableValue.getPaimonTable()).getSnapshot(); + PaimonSnapshot fence = loadLatestSnapshotFence(nameMapping, tableValue).getSnapshot(); if (!tables.isEffectivelyEnabled()) { // Projections are keyed by the synthetic generation of a published table handle. An // ineffective table entry publishes nothing, so nothing keyed by this load could ever // be looked up again: serve it directly instead of churning the snapshot entry. - return executeAuthenticated(nameMapping, + return executeForGeneration(tableValue, nameMapping, () -> latestSnapshotProjectionLoader.loadAtFence( nameMapping, fence, tableValue.getGeneration())); } @@ -135,7 +135,7 @@ public PaimonSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { snapshotEntry.get(nameMapping.getCtlId()); AtomicBoolean loaded = new AtomicBoolean(); PaimonSnapshotCacheValue snapshotValue = entry.get(key, - ignored -> executeAuthenticated(nameMapping, () -> { + ignored -> executeForGeneration(tableValue, nameMapping, () -> { loaded.set(true); return latestSnapshotProjectionLoader.loadAtFence( nameMapping, fence, tableValue.getGeneration()); @@ -221,7 +221,7 @@ public PaimonSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTable, public PaimonSnapshotCacheValue loadLatestSnapshotFence(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); PaimonTableCacheValue tableValue = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); - return loadLatestSnapshotFence(nameMapping, tableValue.getPaimonTable()); + return loadLatestSnapshotFence(nameMapping, tableValue); } public PaimonSnapshotCacheValue loadSnapshotAtFence( @@ -241,23 +241,57 @@ public PaimonSnapshotCacheValue loadSnapshotAtFence( public PaimonSchemaCacheValue getPaimonSchemaCacheValue(NameMapping nameMapping, long schemaId) { PaimonTableCacheValue tableValue = tableEntry.get(nameMapping.getCtlId()).get(nameMapping); - return getPaimonSchemaCacheValue( - nameMapping, schemaId, tableValue.getGeneration(), tableValue.getPaimonTable()); + return getPaimonSchemaCacheValue(nameMapping, schemaId, tableValue.getGeneration(), + tableValue.getPaimonTable(), tableValue.getAuthenticator()); } PaimonSchemaCacheValue getPaimonSchemaCacheValue( NameMapping nameMapping, long schemaId, long tableGeneration, Table retainedTable) { + return getPaimonSchemaCacheValue(nameMapping, schemaId, tableGeneration, retainedTable, null); + } + + PaimonSchemaCacheValue getPaimonSchemaCacheValue(NameMapping nameMapping, long schemaId, + long tableGeneration, Table retainedTable, + @Nullable org.apache.doris.common.security.authentication.ExecutionAuthenticator authenticator) { + return getPaimonSchemaCacheValueInternal(nameMapping, schemaId, tableGeneration, retainedTable, + load -> executeForCapturedAuthenticator(authenticator, nameMapping, load)); + } + + /** + * Schema resolution for a projection load that is already running inside the caller's + * captured authenticated context: re-resolving the catalog here could observe a concurrent + * property reset, so the load executes directly. + */ + private PaimonSchemaCacheValue getPaimonSchemaCacheValuePreAuthenticated( + NameMapping nameMapping, long schemaId, long tableGeneration, Table retainedTable) { + return getPaimonSchemaCacheValueInternal(nameMapping, schemaId, tableGeneration, retainedTable, + load -> { + try { + return load.call(); + } catch (Exception e) { + throw new CacheException("failed to load paimon schema %s.%s.%s: %s", + e, nameMapping.getCtlId(), nameMapping.getLocalDbName(), + nameMapping.getLocalTblName(), e.getMessage()); + } + }); + } + + private PaimonSchemaCacheValue getPaimonSchemaCacheValueInternal(NameMapping nameMapping, long schemaId, + long tableGeneration, Table retainedTable, + java.util.function.Function, + SchemaCacheValue> executor) { + java.util.concurrent.Callable load = + () -> loadSchemaCacheValue(new PaimonSchemaCacheKey( + nameMapping, tableGeneration, schemaId), retainedTable); + java.util.function.Supplier authenticated = () -> executor.apply(load); PaimonSchemaCacheKey key = new PaimonSchemaCacheKey(nameMapping, tableGeneration, schemaId); if (tableGeneration <= 0L || !tableEntry.get(nameMapping.getCtlId()).isEffectivelyEnabled()) { // See getSnapshotCache: without a published table handle no generation-keyed // projection is reachable again. - return (PaimonSchemaCacheValue) executeAuthenticated(nameMapping, - () -> loadSchemaCacheValue(key, retainedTable)); + return (PaimonSchemaCacheValue) authenticated.get(); } MetaCacheEntry entry = schemaEntry.get(nameMapping.getCtlId()); - SchemaCacheValue schemaCacheValue = entry.get(key, - ignored -> executeAuthenticated(nameMapping, - () -> loadSchemaCacheValue(key, retainedTable))); + SchemaCacheValue schemaCacheValue = entry.get(key, ignored -> authenticated.get()); if (!isCurrentTableGeneration(nameMapping, tableGeneration)) { entry.invalidateKeyIfSame(key, schemaCacheValue); } @@ -280,7 +314,39 @@ private boolean isCurrentTableGeneration(NameMapping nameMapping, long tableGene } private PaimonTableCacheValue loadTableCacheValue(NameMapping nameMapping) { - return new PaimonTableCacheValue(tableLoader.load(nameMapping)); + try { + PaimonExternalCatalog catalog = tableLoader.catalog(nameMapping); + return new PaimonTableCacheValue( + tableLoader.load(nameMapping), catalog.getExecutionAuthenticator()); + } catch (java.io.IOException e) { + throw new CacheException("failed to load paimon table %s.%s.%s: %s", + e, nameMapping.getCtlId(), nameMapping.getLocalDbName(), + nameMapping.getLocalTblName(), e.getMessage()); + } + } + + /** + * Execute on the authenticated context captured with the table generation, falling back to + * the catalog's current authenticator for values that predate the capture. + */ + private T executeForGeneration(PaimonTableCacheValue tableValue, + NameMapping nameMapping, java.util.concurrent.Callable task) { + return executeForCapturedAuthenticator(tableValue.getAuthenticator(), nameMapping, task); + } + + private T executeForCapturedAuthenticator( + @Nullable org.apache.doris.common.security.authentication.ExecutionAuthenticator authenticator, + NameMapping nameMapping, java.util.concurrent.Callable task) { + if (authenticator == null) { + return executeAuthenticated(nameMapping, task); + } + try { + return authenticator.execute(task); + } catch (Exception e) { + throw new CacheException("failed to load authenticated paimon metadata %s.%s.%s: %s", + e, nameMapping.getCtlId(), nameMapping.getLocalDbName(), + nameMapping.getLocalTblName(), e.getMessage()); + } } private SchemaCacheValue loadSchemaCacheValue(PaimonSchemaCacheKey key) { @@ -307,9 +373,10 @@ private SchemaCacheValue loadSchemaCacheValue(PaimonSchemaCacheKey key, Table re return value; } - private PaimonSnapshotCacheValue loadLatestSnapshotFence(NameMapping nameMapping, Table retainedTable) { - return executeAuthenticated(nameMapping, - () -> latestSnapshotProjectionLoader.loadFence(nameMapping, retainedTable)); + private PaimonSnapshotCacheValue loadLatestSnapshotFence( + NameMapping nameMapping, PaimonTableCacheValue tableValue) { + return executeForGeneration(tableValue, nameMapping, + () -> latestSnapshotProjectionLoader.loadFence(nameMapping, tableValue.getPaimonTable())); } private T executeAuthenticated(NameMapping nameMapping, java.util.concurrent.Callable task) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java index 9e82df8f8f209c..a9bc7eace90af7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonTableCacheValue.java @@ -25,6 +25,7 @@ import java.util.Objects; import java.util.concurrent.atomic.AtomicLong; +import javax.annotation.Nullable; /** * Cache value for a Paimon table handle. Snapshot projections use a separate cache entry; the @@ -36,14 +37,31 @@ public class PaimonTableCacheValue { private final Table paimonTable; private final long generation; + // The execution authenticator active when this generation was loaded. Later fence/schema + // loads on the generation reuse it so a concurrent catalog reset (property ALTER) can + // neither fail an in-flight lookup nor pair replacement credentials with this handle. + @Nullable + private final org.apache.doris.common.security.authentication.ExecutionAuthenticator authenticator; private volatile long retainedTablePayloadBytes; private volatile MetaCacheSizeEstimate sizeEstimate; public PaimonTableCacheValue(Table paimonTable) { + this(paimonTable, + (org.apache.doris.common.security.authentication.ExecutionAuthenticator) null); + } + + public PaimonTableCacheValue(Table paimonTable, + @Nullable org.apache.doris.common.security.authentication.ExecutionAuthenticator authenticator) { this.paimonTable = paimonTable; + this.authenticator = authenticator; this.generation = NEXT_GENERATION.incrementAndGet(); } + @Nullable + org.apache.doris.common.security.authentication.ExecutionAuthenticator getAuthenticator() { + return authenticator; + } + public PaimonTableCacheValue(Table paimonTable, PaimonSnapshotCacheValue ignoredFence) { this(paimonTable); Objects.requireNonNull(ignoredFence, "latestSnapshotFence can not be null"); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 5c0e5bf9d8e570..3b50d173cc56b0 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -539,6 +539,44 @@ public void testTableEstimateReservesFileIoAllowancePerOwner() throws Exception snapshotEstimate >= 32L * 1024L); } + @Test + public void testGenerationLoadsUseTheAuthenticatorCapturedAtTableLoad() { + // A property ALTER resets the catalog (nulling its authenticator) before retiring the + // old group. A lookup that already captured the table generation must keep using the + // authenticator bound to that generation instead of resolving the resetting catalog. + MockedPaimonCatalog mocked = new MockedPaimonCatalog(); + AtomicInteger boundExecutions = new AtomicInteger(); + ExecutionAuthenticator boundAuthenticator = new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + boundExecutions.incrementAndGet(); + return task.call(); + } + }; + // The catalog's current authenticator is mid-reset and must never be consulted. + Mockito.when(mocked.catalog.getExecutionAuthenticator()) + .thenThrow(new IllegalStateException("catalog is resetting")); + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(mocked.env); + cache.initCatalog(1L, Collections.emptyMap()); + PaimonTableCacheValue tableValue = + new PaimonTableCacheValue(mocked.baseTable, boundAuthenticator); + cache.entry(1L, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class).put(mocked.mapping, tableValue); + ExternalTable dorisTable = mocked.dorisTable(); + + Assert.assertEquals(7L, cache.getSnapshotCache(dorisTable).getSnapshot().getSnapshotId()); + Assert.assertTrue("fence and projection loads must run on the captured authenticator", + boundExecutions.get() >= 2); + Assert.assertNotNull(cache.getPaimonSchemaCacheValue(mocked.mapping, 3L)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + @Test public void testContendedPolicyHandoffRetriesInsteadOfFailingTheLookup() { // A cache-policy ALTER may hold the lifecycle fence while a lookup re-prepares; the From d18acde946c1ac4edc3c1a54e2375e56d3d483b4 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 21 Aug 2026 17:35:28 +0800 Subject: [PATCH 31/45] [test](fe) Align cache property regression expectations and harden CatalogMgrTest timing The unified external meta cache property validation reports invalid ttl values as "Cache property '' must be >= -1" before the legacy per-engine checks run, so the regression suites that still expected the old "is wrong" wording are updated (hive create/alter and iceberg create cases). CatalogMgrTest now pays the one-time Env bootstrap cost before the latched validation window opens and uses CI-safe await timeouts, so a slow first Env.getCurrentEnv() on a loaded host can no longer consume the latch budget. --- .../org/apache/doris/datasource/CatalogMgrTest.java | 11 ++++++++--- .../hive/test_hive_meta_cache.groovy | 10 +++++----- .../iceberg/test_iceberg_table_meta_cache.groovy | 2 +- 3 files changed, 14 insertions(+), 9 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index 48c447342fc949..f4b358a585ac6c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource; import org.apache.doris.catalog.DatabaseIf; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.DdlException; import org.apache.doris.datasource.paimon.PaimonExternalCatalog; @@ -90,6 +91,10 @@ void testDetachedValidationNeverPublishesCandidateToConcurrentInitialization() t CatalogLog log = new CatalogLog(); log.setCatalogId(catalog.getId()); log.setNewProps(newProperties); + // Pay the one-time Env bootstrap cost here: the first Env.getCurrentEnv() call can take + // many seconds on a loaded CI host, and it must not be counted against the latched + // validation window below. + Assertions.assertNotNull(Env.getCurrentEnv().getExtMetaCacheMgr()); ExecutorService executor = Executors.newSingleThreadExecutor(); try { @@ -101,10 +106,10 @@ void testDetachedValidationNeverPublishesCandidateToConcurrentInitialization() t return e; } }); - Assertions.assertTrue(catalog.validationStarted.await(10, TimeUnit.SECONDS)); + Assertions.assertTrue(catalog.validationStarted.await(60, TimeUnit.SECONDS)); Assertions.assertThrows(RuntimeException.class, catalog::makeSureInitialized); - DdlException validationFailure = alterResult.get(10, TimeUnit.SECONDS); + DdlException validationFailure = alterResult.get(60, TimeUnit.SECONDS); Assertions.assertNotNull(validationFailure); Assertions.assertEquals(oldProperties, catalog.propertiesSeenByInitialization); @@ -152,7 +157,7 @@ public boolean validatePropertiesBeforeUpdate( Map currentProperties, Map updatedProperties) { validationStarted.countDown(); try { - Assertions.assertTrue(initializationReadProperties.await(10, TimeUnit.SECONDS)); + Assertions.assertTrue(initializationReadProperties.await(60, TimeUnit.SECONDS)); } catch (InterruptedException e) { Thread.currentThread().interrupt(); throw new IllegalStateException(e); diff --git a/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy b/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy index aa5ba31af17343..608bd0e7da81f5 100644 --- a/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy +++ b/regression-test/suites/external_table_p0/hive/test_hive_meta_cache.groovy @@ -76,7 +76,7 @@ suite("test_hive_meta_cache", "p0,external,hive,external_docker,external_docker_ 'file.meta.cache.ttl-second' = '-2' ); """ - exception "is wrong" + exception "must be >= -1" } // disable file list cache @@ -128,7 +128,7 @@ suite("test_hive_meta_cache", "p0,external,hive,external_docker,external_docker_ 'partition.cache.ttl-second' = '-2' ); """ - exception "is wrong" + exception "must be >= -1" } // disable partition cache @@ -200,7 +200,7 @@ suite("test_hive_meta_cache", "p0,external,hive,external_docker,external_docker_ // alter wrong catalog property test { sql """alter catalog ${catalog_name_no_cache} set properties ("file.meta.cache.ttl-second" = "-2")""" - exception "is wrong" + exception "must be >= -1" } // alter catalog property, disable file list cache sql """alter catalog ${catalog_name_no_cache} set properties ("file.meta.cache.ttl-second" = "0")""" @@ -218,7 +218,7 @@ suite("test_hive_meta_cache", "p0,external,hive,external_docker,external_docker_ // alter wrong catalog property test { sql """alter catalog ${catalog_name_no_cache} set properties ("partition.cache.ttl-second" = "-2")""" - exception "is wrong" + exception "must be >= -1" } // alter catalog property, disable partition cache sql """alter catalog ${catalog_name_no_cache} set properties ("partition.cache.ttl-second" = "0")""" @@ -286,7 +286,7 @@ suite("test_hive_meta_cache", "p0,external,hive,external_docker,external_docker_ // alter wrong catalog property test { sql """alter catalog ${catalog_name_no_cache} set properties ("schema.cache.ttl-second" = "-2")""" - exception "is wrong" + exception "must be >= -1" } sql """alter catalog ${catalog_name_no_cache} set properties ("schema.cache.ttl-second" = "0")""" // desc table, 5 columns diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy index f322a75f3e44f1..88016a49043e14 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy @@ -109,7 +109,7 @@ suite("test_iceberg_table_meta_cache", "p0,external,doris,external_docker,extern 'meta.cache.iceberg.table.ttl-second' = '-2' ); """ - exception "is wrong" + exception "must be >= -1" } // disable iceberg table meta cache From 9c193c7a14a3e89ddebfc8865d857d96fad88182 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 21 Aug 2026 18:19:44 +0800 Subject: [PATCH 32/45] [fix](iceberg) Drop manifest content caches for snapshot-retained FileIOs on catalog reset Catalog invalidation used to enumerate only the table entry when calling ManifestFiles.dropCache, and ran the cleanup before the entries were detached. A frozen table generation whose base table entry was already evicted (weight/TTL/soft collection) but which still lives in an independently admitted snapshot projection kept its FileIO out of that pass, so the SDK per-FileIO manifest content cache could outlive the reset until weak-key cleanup; a racing load could also repopulate a FileIO that the pre-detach pass had already dropped. Collect the FileIOs from both the table and snapshot entries (identity de-duplicated) while they are still enumerable, then drop the SDK caches only after super.invalidateCatalog*() has detached the entries. Add a regression covering a snapshot-only generation plus exactly-once cleanup of a generation shared by both entries. --- .../iceberg/IcebergExternalMetaCache.java | 54 ++++++++++++++++--- .../iceberg/IcebergExternalMetaCacheTest.java | 50 +++++++++++++++++ 2 files changed, 97 insertions(+), 7 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index 9ef7a74df83920..959e087cd7b4b2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -42,11 +42,14 @@ import org.apache.iceberg.ManifestReader; import org.apache.iceberg.Snapshot; import org.apache.iceberg.Table; +import org.apache.iceberg.io.FileIO; import org.apache.iceberg.view.View; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import java.io.IOException; +import java.util.ArrayList; +import java.util.IdentityHashMap; import java.util.List; import java.util.Map; import java.util.Optional; @@ -299,14 +302,19 @@ public ManifestCacheValue getManifestCacheValue(ExternalTable dorisTable, @Override public void invalidateCatalog(long catalogId) { - dropManifestFileIoCacheForCatalog(catalogId); + // Collect while the entries are still enumerable, drop only after the entries are + // detached: a load racing a pre-detach drop could repopulate the SDK content cache + // for a FileIO this reset already cleaned. + List retainedFileIos = collectManifestFileIos(catalogId); super.invalidateCatalog(catalogId); + dropManifestFileIoCaches(retainedFileIos); } @Override public void invalidateCatalogEntries(long catalogId) { - dropManifestFileIoCacheForCatalog(catalogId); + List retainedFileIos = collectManifestFileIos(catalogId); super.invalidateCatalogEntries(catalogId); + dropManifestFileIoCaches(retainedFileIos); } private IcebergTableCacheValue loadTableCacheValue(NameMapping nameMapping) { @@ -543,18 +551,50 @@ private ManifestCacheValue loadDeleteFiles( return builder.build(); } - private void dropManifestFileIoCacheForCatalog(long catalogId) { + /** + * Collect every FileIO a catalog's cached values still retain. A snapshot value can outlive + * its weighted/expired/collected table entry while retaining the same frozen table graph, so + * both entries are enumerated; identity de-duplication keeps the later drop pass bounded. + */ + private List collectManifestFileIos(long catalogId) { + IdentityHashMap seen = new IdentityHashMap<>(); + List fileIos = new ArrayList<>(); MetaCacheEntry tables = tableEntry.getIfInitialized(catalogId); if (tables != null) { - tables.forEach((key, value) -> dropManifestFileIoCache(value)); + tables.forEach((key, value) -> collectManifestFileIo(seen, fileIos, + value == null ? null : value.getIcebergTable())); + } + MetaCacheEntry snapshots = + snapshotEntry.getIfInitialized(catalogId); + if (snapshots != null) { + snapshots.forEach((key, value) -> collectManifestFileIo(seen, fileIos, + value == null ? null : value.getRetainedIcebergTable().orElse(null))); } + return fileIos; } - private void dropManifestFileIoCache(IcebergTableCacheValue tableCacheValue) { + private void collectManifestFileIo(IdentityHashMap seen, List fileIos, + @Nullable Table table) { + if (table == null) { + return; + } try { - ManifestFiles.dropCache(tableCacheValue.getIcebergTable().io()); + FileIO fileIo = table.io(); + if (fileIo != null && seen.put(fileIo, Boolean.TRUE) == null) { + fileIos.add(fileIo); + } } catch (Exception e) { - LOG.warn("Failed to drop iceberg manifest files cache", e); + LOG.warn("Failed to resolve iceberg table FileIO for manifest cache cleanup", e); + } + } + + private void dropManifestFileIoCaches(List fileIos) { + for (FileIO fileIo : fileIos) { + try { + ManifestFiles.dropCache(fileIo); + } catch (Exception e) { + LOG.warn("Failed to drop iceberg manifest files cache", e); + } } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index e7dafd059735b1..601ca148040db9 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -41,6 +41,7 @@ import org.apache.iceberg.HasTableOperations; import org.apache.iceberg.ManifestContent; import org.apache.iceberg.ManifestFile; +import org.apache.iceberg.ManifestFiles; import org.apache.iceberg.Metrics; import org.apache.iceberg.PartitionData; import org.apache.iceberg.PartitionSpec; @@ -67,6 +68,7 @@ import org.junit.Rule; import org.junit.Test; import org.junit.rules.TemporaryFolder; +import org.mockito.MockedStatic; import org.mockito.Mockito; import java.lang.reflect.Field; @@ -217,6 +219,54 @@ public void testReplacingTableGenerationRetiresSnapshotAndSchemaProjection() { } } + @Test + public void testCatalogInvalidationDropsManifestCacheForSnapshotOnlyFileIo() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + try { + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.emptyMap()); + NameMapping sharedMapping = NameMapping.createForTest(catalogId, "db", "shared"); + NameMapping evictedMapping = NameMapping.createForTest(catalogId, "db", "evicted"); + PropertiesFileIO sharedIo = new PropertiesFileIO("token", "shared"); + PropertiesFileIO snapshotOnlyIo = new PropertiesFileIO("token", "snapshot-only"); + Table sharedTable = tableWithMetadata( + metadataWithLocation("/metadata/shared-v1.json"), sharedIo); + Table evictedTable = tableWithMetadata( + metadataWithLocation("/metadata/evicted-v1.json"), snapshotOnlyIo); + + MetaCacheEntry tables = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class); + MetaCacheEntry snapshots = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_SNAPSHOT, + IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class); + + // The shared generation is retained by both the table entry and its projection: the + // reset must drop its per-FileIO manifest cache exactly once. + tables.put(sharedMapping, new IcebergTableCacheValue(sharedTable)); + snapshots.put(IcebergSnapshotEntryKey.tryCreate(sharedMapping, sharedTable).get(), + new IcebergSnapshotCacheValue(IcebergPartitionInfo.empty(), + new IcebergSnapshot(-1L, 0L), Optional.empty(), sharedTable)); + // The second generation survives only in its independently admitted snapshot + // projection, mimicking weight/TTL eviction of the base table entry before reset. + snapshots.put(IcebergSnapshotEntryKey.tryCreate(evictedMapping, evictedTable).get(), + new IcebergSnapshotCacheValue(IcebergPartitionInfo.empty(), + new IcebergSnapshot(-1L, 0L), Optional.empty(), evictedTable)); + + try (MockedStatic manifestFiles = Mockito.mockStatic(ManifestFiles.class)) { + cache.invalidateCatalog(catalogId); + manifestFiles.verify(() -> ManifestFiles.dropCache(sharedIo), Mockito.times(1)); + manifestFiles.verify(() -> ManifestFiles.dropCache(snapshotOnlyIo), Mockito.times(1)); + manifestFiles.verifyNoMoreInteractions(); + } + Assert.assertNull(tables.peekIfPresent(sharedMapping)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + @Test public void testSameGenerationRefreshWithRenewedFileIoRetiresSnapshotProjection() { ExecutorService executor = Executors.newSingleThreadExecutor(); From ad5a42de6c5a1f0ca78e2097826dbbdfe0a11896 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 21 Aug 2026 19:21:45 +0800 Subject: [PATCH 33/45] [fix](paimon) Keep statement-fence and relation hydration on the captured table generation A statement fence and a relation-options projection retain the physical table of the generation they were captured from, but the later hydration APIs (loadSnapshotAtFence, the effective-table sibling, the direct relation-options projection and generation-zero schema resolution) re-resolved the catalog's current execution context. A concurrent property/credential ALTER could therefore run the retained old-generation table under the new catalog resources, or fail while reset closes the old generation. PaimonSnapshotCacheValue now carries the ExecutionAuthenticator captured from its table generation; fence capture, cached and direct projections bind it, hydration executes under it (falling back to the catalog context only when nothing was captured) and propagates it to the hydrated value so later schema hydration stays on the same generation. The relation- options path captures the table generation once and derives the base handle and execution context from that single value. Regression: a capture -> ALTER (context swap + generation retirement) -> hydration sequence asserts every hydration path runs under the captured context and never consults the post-ALTER catalog context. --- .../paimon/PaimonExternalMetaCache.java | 51 +++++-- .../paimon/PaimonExternalTable.java | 15 +- .../paimon/PaimonSnapshotCacheValue.java | 22 +++ .../doris/datasource/paimon/PaimonUtils.java | 21 ++- .../paimon/PaimonExternalMetaCacheTest.java | 130 ++++++++++++++++++ .../paimon/PaimonExternalTableTest.java | 21 ++- 6 files changed, 230 insertions(+), 30 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index 8039b7daa93854..bce765fd87488c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -123,7 +123,8 @@ public PaimonSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { // be looked up again: serve it directly instead of churning the snapshot entry. return executeForGeneration(tableValue, nameMapping, () -> latestSnapshotProjectionLoader.loadAtFence( - nameMapping, fence, tableValue.getGeneration())); + nameMapping, fence, tableValue.getGeneration())) + .bindCapturedAuthenticator(tableValue.getAuthenticator()); } // Order fence observations, not snapshot ids: a rollback moves the latest snapshot // backwards, and a concurrent call may finish after a later observation (reversed @@ -138,7 +139,8 @@ public PaimonSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { ignored -> executeForGeneration(tableValue, nameMapping, () -> { loaded.set(true); return latestSnapshotProjectionLoader.loadAtFence( - nameMapping, fence, tableValue.getGeneration()); + nameMapping, fence, tableValue.getGeneration()) + .bindCapturedAuthenticator(tableValue.getAuthenticator()); })); LatestFenceOwner owner = new LatestFenceOwner(nameMapping, tableValue.getGeneration()); ObservedFence latest = latestObservedFences.compute(owner, (ignored, current) -> @@ -212,10 +214,25 @@ private ObservedFence(long observation, PaimonSnapshotEntryKey key) { } } - public PaimonSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTable, Table effectiveTable) { + /** + * Load a projection for a relation-scoped effective table. The caller derived + * {@code effectiveTable} from {@code tableGeneration}'s base handle, so the load and any later + * hydration of the returned value must run under that generation's captured execution context: + * re-resolving the catalog here could pair the retained handle with the resources of a + * concurrent property/credential ALTER, or fail while reset closes the old generation. + */ + public PaimonSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTable, Table effectiveTable, + PaimonTableCacheValue tableGeneration) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - return executeAuthenticated(nameMapping, - () -> latestSnapshotProjectionLoader.load(nameMapping, effectiveTable)); + return executeForGeneration(tableGeneration, nameMapping, + () -> latestSnapshotProjectionLoader.load(nameMapping, effectiveTable)) + .bindCapturedAuthenticator(tableGeneration.getAuthenticator()); + } + + /** Resolve the current table generation, exposing the handle and its captured context together. */ + public PaimonTableCacheValue getTableCacheValue(ExternalTable dorisTable) { + NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); + return tableEntry.get(nameMapping.getCtlId()).get(nameMapping); } public PaimonSnapshotCacheValue loadLatestSnapshotFence(ExternalTable dorisTable) { @@ -224,19 +241,28 @@ public PaimonSnapshotCacheValue loadLatestSnapshotFence(ExternalTable dorisTable return loadLatestSnapshotFence(nameMapping, tableValue); } + /** + * Hydrate a statement fence into a full projection. The fence retains the physical table of + * the generation it was captured from, so hydration runs under the fence's captured execution + * context instead of re-resolving the catalog's current one, and the returned value carries + * the same context forward for later schema hydration. + */ public PaimonSnapshotCacheValue loadSnapshotAtFence( - ExternalTable dorisTable, PaimonSnapshot fence) { + ExternalTable dorisTable, PaimonSnapshotCacheValue fenceValue) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - return executeAuthenticated(nameMapping, - () -> latestSnapshotProjectionLoader.loadAtFence(nameMapping, fence)); + return executeForCapturedAuthenticator(fenceValue.getCapturedAuthenticator(), nameMapping, + () -> latestSnapshotProjectionLoader.loadAtFence(nameMapping, fenceValue.getSnapshot())) + .bindCapturedAuthenticator(fenceValue.getCapturedAuthenticator()); } + /** Effective-table sibling of {@link #loadSnapshotAtFence(ExternalTable, PaimonSnapshotCacheValue)}. */ public PaimonSnapshotCacheValue loadSnapshotAtFence( - ExternalTable dorisTable, Table effectiveTable, PaimonSnapshot fence) { + ExternalTable dorisTable, Table effectiveTable, PaimonSnapshotCacheValue fenceValue) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); - return executeAuthenticated(nameMapping, + return executeForCapturedAuthenticator(fenceValue.getCapturedAuthenticator(), nameMapping, () -> latestSnapshotProjectionLoader.loadEffectiveAtFence( - nameMapping, effectiveTable, fence)); + nameMapping, effectiveTable, fenceValue.getSnapshot())) + .bindCapturedAuthenticator(fenceValue.getCapturedAuthenticator()); } public PaimonSchemaCacheValue getPaimonSchemaCacheValue(NameMapping nameMapping, long schemaId) { @@ -376,7 +402,8 @@ private SchemaCacheValue loadSchemaCacheValue(PaimonSchemaCacheKey key, Table re private PaimonSnapshotCacheValue loadLatestSnapshotFence( NameMapping nameMapping, PaimonTableCacheValue tableValue) { return executeForGeneration(tableValue, nameMapping, - () -> latestSnapshotProjectionLoader.loadFence(nameMapping, tableValue.getPaimonTable())); + () -> latestSnapshotProjectionLoader.loadFence(nameMapping, tableValue.getPaimonTable())) + .bindCapturedAuthenticator(tableValue.getAuthenticator()); } private T executeAuthenticated(NameMapping nameMapping, java.util.concurrent.Callable task) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index defa3dd4d00ec5..6656ea476251e2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -194,7 +194,11 @@ private PaimonSnapshotCacheValue getPaimonSnapshotCacheValue(Optional resolvedOptions = scanParams.get().getOrResolveMapParams( options -> PaimonScanParams.resolveOptions(baseTable, options)); Table effectiveTable = PaimonScanParams.applyOptions(baseTable, resolvedOptions); @@ -205,7 +209,7 @@ private PaimonSnapshotCacheValue getPaimonSnapshotCacheValue(Optional scanParams, Optional latestSnapshotFence) { if (latestSnapshotFence.isPresent() && !tableSnapshot.isPresent() && !scanParams.isPresent()) { - PaimonSnapshot fence = ((PaimonMvccSnapshot) latestSnapshotFence.get()) - .getSnapshotCacheValue().getSnapshot(); - return new PaimonMvccSnapshot(PaimonUtils.loadSnapshotAtFence(this, fence)); + return new PaimonMvccSnapshot(PaimonUtils.loadSnapshotAtFence(this, + ((PaimonMvccSnapshot) latestSnapshotFence.get()).getSnapshotCacheValue())); } if (!latestSnapshotFence.isPresent() || !requiresLatestSnapshotFence(tableSnapshot, scanParams)) { @@ -419,7 +422,7 @@ public MvccSnapshot loadSnapshot( FileStoreTable effectiveTable = PaimonScanParams.applyOptionsWithoutTimeTravel( (FileStoreTable) fenceSnapshot.getTable(), params.getResolvedMapParams().get()); return new PaimonMvccSnapshot( - PaimonUtils.loadSnapshotAtFence(this, effectiveTable, fenceSnapshot)); + PaimonUtils.loadSnapshotAtFence(this, effectiveTable, fenceValue)); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java index e6b37d4c020b72..c3890bbc6be1b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSnapshotCacheValue.java @@ -17,9 +17,12 @@ package org.apache.doris.datasource.paimon; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; +import javax.annotation.Nullable; + public class PaimonSnapshotCacheValue { private final PaimonPartitionInfo partitionInfo; @@ -28,6 +31,15 @@ public class PaimonSnapshotCacheValue { private final long tableGeneration; private long retainedTablePayloadBytes; private MetaCacheSizeEstimate sizeEstimate; + /** + * Execution context captured from the table generation this value retains. Later projection + * or schema hydration of the retained physical table must run under this context, never under + * the catalog's current one: a concurrent property/credential ALTER may already have replaced + * the catalog resources while this statement still operates the retained generation. + * Not part of the accounted payload; it references catalog-generation-lifetime resources. + */ + @Nullable + private transient volatile ExecutionAuthenticator capturedAuthenticator; public PaimonSnapshotCacheValue(PaimonPartitionInfo partitionInfo, PaimonSnapshot snapshot) { this(partitionInfo, snapshot, false, 0L); @@ -54,6 +66,16 @@ public PaimonSnapshot getSnapshot() { return snapshot; } + public PaimonSnapshotCacheValue bindCapturedAuthenticator(@Nullable ExecutionAuthenticator authenticator) { + this.capturedAuthenticator = authenticator; + return this; + } + + @Nullable + public ExecutionAuthenticator getCapturedAuthenticator() { + return capturedAuthenticator; + } + public boolean isSchemaFromSnapshotTable() { return schemaFromSnapshotTable; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtils.java index 93eff82dc1e969..383188efeab188 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtils.java @@ -35,8 +35,14 @@ public static PaimonSnapshotCacheValue getLatestSnapshotCacheValue(ExternalTable return paimonExternalMetaCache(dorisTable).getSnapshotCache(dorisTable); } - public static PaimonSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTable, Table effectiveTable) { - return paimonExternalMetaCache(dorisTable).loadSnapshotProjection(dorisTable, effectiveTable); + public static PaimonSnapshotCacheValue loadSnapshotProjection(ExternalTable dorisTable, Table effectiveTable, + PaimonTableCacheValue tableGeneration) { + return paimonExternalMetaCache(dorisTable).loadSnapshotProjection( + dorisTable, effectiveTable, tableGeneration); + } + + public static PaimonTableCacheValue getPaimonTableCacheValue(ExternalTable dorisTable) { + return paimonExternalMetaCache(dorisTable).getTableCacheValue(dorisTable); } public static PaimonSnapshotCacheValue loadLatestSnapshotFence(ExternalTable dorisTable) { @@ -44,14 +50,14 @@ public static PaimonSnapshotCacheValue loadLatestSnapshotFence(ExternalTable dor } public static PaimonSnapshotCacheValue loadSnapshotAtFence( - ExternalTable dorisTable, PaimonSnapshot fence) { - return paimonExternalMetaCache(dorisTable).loadSnapshotAtFence(dorisTable, fence); + ExternalTable dorisTable, PaimonSnapshotCacheValue fenceValue) { + return paimonExternalMetaCache(dorisTable).loadSnapshotAtFence(dorisTable, fenceValue); } public static PaimonSnapshotCacheValue loadSnapshotAtFence( - ExternalTable dorisTable, Table effectiveTable, PaimonSnapshot fence) { + ExternalTable dorisTable, Table effectiveTable, PaimonSnapshotCacheValue fenceValue) { return paimonExternalMetaCache(dorisTable).loadSnapshotAtFence( - dorisTable, effectiveTable, fence); + dorisTable, effectiveTable, fenceValue); } public static PaimonSnapshotCacheValue getSnapshotCacheValue(Optional snapshot, @@ -71,7 +77,8 @@ public static PaimonSchemaCacheValue getSchemaCacheValue(ExternalTable dorisTabl // The generation-zero path performs an authenticated uncached load. return paimonExternalMetaCache(dorisTable).getPaimonSchemaCacheValue( dorisTable.getOrBuildNameMapping(), snapshotValue.getSnapshot().getSchemaId(), - snapshotValue.getTableGeneration(), snapshotValue.getSnapshot().getTable()); + snapshotValue.getTableGeneration(), snapshotValue.getSnapshot().getTable(), + snapshotValue.getCapturedAuthenticator()); } return getSchemaCacheValue(dorisTable, snapshotValue.getSnapshot().getSchemaId()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 3b50d173cc56b0..bdfaf91a3f10ae 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -89,6 +89,7 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; public class PaimonExternalMetaCacheTest { @Rule @@ -945,6 +946,135 @@ public T execute(Callable task) throws Exception { } } + @Test + public void testFenceAndRelationHydrationStayOnCapturedGenerationAcrossAlter() { + AtomicInteger authenticationDepth = new AtomicInteger(); + ExecutionAuthenticator capturedAuthenticator = new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + authenticationDepth.incrementAndGet(); + try { + return task.call(); + } finally { + authenticationDepth.decrementAndGet(); + } + } + }; + ExecutionAuthenticator resettingAuthenticator = new ExecutionAuthenticator() { + @Override + public T execute(Callable task) { + throw new AssertionError( + "hydration of a captured generation must not consult the post-ALTER catalog context"); + } + }; + AtomicReference currentAuthenticator = + new AtomicReference<>(capturedAuthenticator); + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + PaimonExternalDatabase database = Mockito.mock(PaimonExternalDatabase.class); + PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.doReturn(catalog).when(catalogMgr) + .getCatalogOrException(Mockito.eq(1L), Mockito.any()); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog(1L); + Mockito.when(catalog.getExecutionAuthenticator()).thenAnswer( + invocation -> currentAuthenticator.get()); + Mockito.doReturn(database).when(catalog).getDbNullable("db"); + Mockito.when(database.getTableNullable("tbl")).thenReturn(externalTable); + Mockito.doReturn(Optional.of(database)).when(catalog).getDb("db"); + Mockito.doReturn(Optional.of(externalTable)).when(database).getTable("tbl"); + Mockito.doAnswer(invocation -> { + Assert.assertTrue("schema hydration must stay on the captured generation context", + authenticationDepth.get() > 0); + Column partitionColumn = new Column("part", Type.INT); + return new PaimonSchemaCacheValue( + Collections.singletonList(partitionColumn), + Collections.singletonList(partitionColumn), null); + }).when(externalTable).loadSchemaForCache(Mockito.any(), Mockito.anyLong()); + + FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); + FileStoreTable latestSchemaTable = Mockito.mock(FileStoreTable.class); + FileStoreTable fenceTable = Mockito.mock(FileStoreTable.class); + FileStoreTable snapshotTable = Mockito.mock(FileStoreTable.class); + Snapshot latestSnapshot = Mockito.mock(Snapshot.class); + SchemaManager schemaManager = Mockito.mock(SchemaManager.class); + TableSchema latestSchema = Mockito.mock(TableSchema.class); + ReadBuilder readBuilder = Mockito.mock(ReadBuilder.class); + TableScan tableScan = Mockito.mock(TableScan.class); + Mockito.when(baseTable.copyWithLatestSchema()).thenAnswer(invocation -> { + Assert.assertTrue("fence capture must run authenticated", authenticationDepth.get() > 0); + return latestSchemaTable; + }); + Mockito.when(latestSchemaTable.latestSnapshot()).thenReturn(Optional.of(latestSnapshot)); + Mockito.when(latestSnapshot.id()).thenReturn(7L); + Mockito.when(latestSchemaTable.schemaManager()).thenReturn(schemaManager); + Mockito.when(schemaManager.latest()).thenReturn(Optional.of(latestSchema)); + Mockito.when(latestSchema.id()).thenReturn(3L); + Mockito.when(latestSchemaTable.copyWithoutTimeTravel(Mockito.anyMap())).thenReturn(fenceTable); + Mockito.when(fenceTable.copyWithoutTimeTravel(Mockito.anyMap())).thenAnswer(invocation -> { + Assert.assertTrue("snapshot pinning must stay on the captured generation context", + authenticationDepth.get() > 0); + return snapshotTable; + }); + Mockito.when(fenceTable.options()).thenReturn(Collections.emptyMap()); + Mockito.when(fenceTable.newReadBuilder()).thenReturn(readBuilder); + Mockito.when(snapshotTable.options()).thenReturn(Collections.emptyMap()); + Mockito.when(snapshotTable.newReadBuilder()).thenReturn(readBuilder); + Mockito.when(readBuilder.newScan()).thenReturn(tableScan); + Mockito.when(tableScan.listPartitionEntries()).thenAnswer(invocation -> { + Assert.assertTrue("partition enumeration must stay on the captured generation context", + authenticationDepth.get() > 0); + return Collections.emptyList(); + }); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + cache.initCatalog(1L, Collections.emptyMap()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); + PaimonTableCacheValue tableValue = new PaimonTableCacheValue(baseTable, capturedAuthenticator); + org.apache.doris.datasource.metacache.MetaCacheEntry tables = + cache.entry(1L, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class); + tables.put(mapping, tableValue); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + + // 1. The statement captures its fence while generation one is current. + PaimonSnapshotCacheValue fence = cache.loadLatestSnapshotFence(dorisTable); + Assert.assertSame(capturedAuthenticator, fence.getCapturedAuthenticator()); + + // 2. A concurrent ALTER replaces the catalog execution context and retires the + // captured generation before the statement hydrates its relations. + currentAuthenticator.set(resettingAuthenticator); + tables.invalidateKey(mapping); + + // 3. Every later hydration path must stay on the captured context; consulting the + // catalog's current context throws above. + PaimonSnapshotCacheValue hydrated = cache.loadSnapshotAtFence(dorisTable, fence); + Assert.assertEquals(7L, hydrated.getSnapshot().getSnapshotId()); + Assert.assertSame(capturedAuthenticator, hydrated.getCapturedAuthenticator()); + + PaimonSnapshotCacheValue effectiveHydrated = + cache.loadSnapshotAtFence(dorisTable, fenceTable, fence); + Assert.assertSame(capturedAuthenticator, effectiveHydrated.getCapturedAuthenticator()); + + PaimonSnapshotCacheValue directProjection = + cache.loadSnapshotProjection(dorisTable, baseTable, tableValue); + Assert.assertSame(capturedAuthenticator, directProjection.getCapturedAuthenticator()); + + // 4. Generation-zero schema resolution follows the value's captured context as well. + cache.getPaimonSchemaCacheValue(mapping, 3L, 0L, fenceTable, + hydrated.getCapturedAuthenticator()); + Assert.assertEquals(0, authenticationDepth.get()); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + @Test public void testExpiredBaseTableRetiresSnapshotAndSchemaProjectionsBeforeReplacement() throws Exception { ExecutorService executor = Executors.newSingleThreadExecutor(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java index 7f5483928c8a97..dd57e57b55cf5d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java @@ -122,7 +122,8 @@ public void testStatementContextDefersPhysicalManifestValidationUntilRelationOpt try (MockedStatic paimonUtils = Mockito.mockStatic( PaimonUtils.class, Mockito.CALLS_REAL_METHODS)) { paimonUtils.when(() -> PaimonUtils.loadSnapshotAtFence( - Mockito.eq(externalTable), Mockito.eq(safeRelationTable), Mockito.any(PaimonSnapshot.class))) + Mockito.eq(externalTable), Mockito.eq(safeRelationTable), + Mockito.any(PaimonSnapshotCacheValue.class))) .thenReturn(projectedValue); StatementContext statementContext = new StatementContext(new ConnectContext(), null); @@ -240,8 +241,11 @@ public void testRelationOptionsLoadSnapshotProjectionFromEffectiveTable() { try (MockedStatic paimonUtils = Mockito.mockStatic(PaimonUtils.class)) { paimonUtils.when(() -> PaimonUtils.getPaimonTable(externalTable)).thenReturn(unsafeDataTable); + paimonUtils.when(() -> PaimonUtils.getPaimonTableCacheValue(externalTable)) + .thenReturn(new PaimonTableCacheValue(unsafeDataTable)); paimonUtils.when(() -> PaimonUtils.loadSnapshotProjection( - Mockito.eq(externalTable), Mockito.any(Table.class))).thenReturn(projection); + Mockito.eq(externalTable), Mockito.any(Table.class), + Mockito.any(PaimonTableCacheValue.class))).thenReturn(projection); PaimonMvccSnapshot snapshot = (PaimonMvccSnapshot) externalTable.loadSnapshot( Optional.empty(), Optional.of(scanParams)); @@ -249,7 +253,8 @@ public void testRelationOptionsLoadSnapshotProjectionFromEffectiveTable() { Assert.assertSame(projection, snapshot.getSnapshotCacheValue()); paimonUtils.verify(() -> PaimonUtils.loadSnapshotProjection( Mockito.eq(externalTable), Mockito.argThat(table -> - "1".equals(table.options().get("scan.manifest.parallelism"))))); + "1".equals(table.options().get("scan.manifest.parallelism"))), + Mockito.argThat(generation -> generation.getPaimonTable() == unsafeDataTable))); } } @@ -269,10 +274,13 @@ public void testReaderOnlyOptionsReusePartitionedMemoizedProjection() { try (MockedStatic paimonUtils = Mockito.mockStatic(PaimonUtils.class)) { paimonUtils.when(() -> PaimonUtils.getPaimonTable(externalTable)).thenReturn(partitionedTable); + paimonUtils.when(() -> PaimonUtils.getPaimonTableCacheValue(externalTable)) + .thenReturn(new PaimonTableCacheValue(partitionedTable)); paimonUtils.when(() -> PaimonUtils.getLatestSnapshotCacheValue(externalTable)) .thenReturn(memoizedProjection); paimonUtils.when(() -> PaimonUtils.loadSnapshotProjection( - Mockito.eq(externalTable), Mockito.any(Table.class))).thenAnswer(invocation -> { + Mockito.eq(externalTable), Mockito.any(Table.class), + Mockito.any(PaimonTableCacheValue.class))).thenAnswer(invocation -> { directProjectionLoads.incrementAndGet(); return directProjection; }); @@ -301,8 +309,11 @@ public void testFencedReaderOnlyOptionsReuseFenceProjection() { try (MockedStatic paimonUtils = Mockito.mockStatic(PaimonUtils.class)) { paimonUtils.when(() -> PaimonUtils.getPaimonTable(externalTable)).thenReturn(capturedTable); + paimonUtils.when(() -> PaimonUtils.getPaimonTableCacheValue(externalTable)) + .thenReturn(new PaimonTableCacheValue(capturedTable)); paimonUtils.when(() -> PaimonUtils.loadSnapshotProjection( - Mockito.eq(externalTable), Mockito.any(Table.class))) + Mockito.eq(externalTable), Mockito.any(Table.class), + Mockito.any(PaimonTableCacheValue.class))) .thenThrow(new AssertionError( "reader-only tuning must not enumerate a new partition projection")); paimonUtils.when(() -> PaimonUtils.getLatestSnapshotCacheValue(externalTable)) From 7fe5e769ccde86c9e86da13739ee24fc48007941 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Fri, 21 Aug 2026 20:33:54 +0800 Subject: [PATCH 34/45] [fix](fe) Acquire external table handles and execution context from one catalog generation The Iceberg table-entry miss loader captured its authenticator when entering the load but re-read the mutable catalog after ops.loadTable() returned to bind the published value, and getWritableIcebergTable resolved the metadata ops before entering the current authenticator. The Paimon miss loader similarly read the catalog's authenticator only after tableLoader.load() returned. A concurrent property/credential ALTER can reset and reinitialize the catalog in that window without retiring the engine's cache group, publishing a table of the old generation bound to the new generation's execution context - a splice every later fence, projection and schema hydration would then deliberately trust. Both loaders (and the writable-handle path) now capture the execution authenticator (and Iceberg metadata ops) once, run the external load under that exact context, and re-validate the acquisition against the catalog before the result is used or published. A mid-flight reset turns into a clean retryable failure; the retry runs entirely against the reinitialized catalog. Regressions: mid-flight-reset latch tests for the Paimon miss loader and for both Iceberg boundaries (miss loader and writable handle), each also asserting the retry publishes a coherent pair. --- .../iceberg/IcebergExternalMetaCache.java | 70 ++++++++++++-- .../paimon/PaimonExternalMetaCache.java | 32 ++++++- .../iceberg/IcebergExternalMetaCacheTest.java | 95 +++++++++++++++++++ .../paimon/PaimonExternalMetaCacheTest.java | 73 ++++++++++++++ 4 files changed, 258 insertions(+), 12 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index 959e087cd7b4b2..4a8becfafeca85 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -18,6 +18,7 @@ package org.apache.doris.datasource.iceberg; import org.apache.doris.common.AnalysisException; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.CacheException; import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalCatalog; @@ -142,11 +143,17 @@ public Table getWritableIcebergTable(ExternalTable dorisTable) { throw new RuntimeException("Cannot find catalog " + nameMapping.getCtlId() + " when loading a writable Iceberg table"); } - IcebergMetadataOps ops = resolveMetadataOps(catalog); // DDL/actions must start from the live catalog generation. DML that was planned against a - // retained read generation wraps this live table separately in IcebergTransaction. - return executeAuthenticated(catalog, () -> ops.loadTable( + // retained read generation wraps this live table separately in IcebergTransaction. The + // authenticator, ops and loaded handle must all come from that one generation, so the + // acquisition is re-validated afterwards: a concurrent property/credential ALTER can reset + // and reinitialize the catalog mid-flight without retiring this engine's cache group. + ExecutionAuthenticator authenticator = requireExecutionAuthenticator(catalog); + IcebergMetadataOps ops = resolveMetadataOps(catalog); + Table table = execute(authenticator, () -> ops.loadTable( nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName())); + ensureCatalogGenerationStable(catalog, ops, authenticator, nameMapping); + return table; } Table getQueryScopedIcebergTable(ExternalTable dorisTable) { @@ -324,20 +331,63 @@ private IcebergTableCacheValue loadTableCacheValue(NameMapping nameMapping) { nameMapping.getCtlId(), nameMapping.getLocalDbName(), nameMapping.getLocalTblName())); } + // One catalog generation must supply the ops, the loaded table and the bound + // authenticator together: re-reading the mutable catalog after the load could bind a + // handle of the old generation to the execution context a concurrent ALTER installed. + // The acquisition is re-validated before publication; a mid-flight reset fails the miss + // (the caller retries against the reinitialized catalog) instead of publishing a splice. + ExecutionAuthenticator authenticator = requireExecutionAuthenticator(catalog); IcebergMetadataOps ops = resolveMetadataOps(catalog); - return executeAuthenticated(catalog, () -> { + IcebergTableCacheValue value = execute(authenticator, () -> { Table table = ops.loadTable(nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName()); - IcebergTableCacheValue value = new IcebergTableCacheValue(table); - if (catalog instanceof ExternalCatalog) { - value.bindAuthenticator(((ExternalCatalog) catalog).getExecutionAuthenticator()); - } + IcebergTableCacheValue loaded = new IcebergTableCacheValue(table); + loaded.bindAuthenticator(authenticator); MetaCacheEntry currentEntry = tableEntry.getIfInitialized(nameMapping.getCtlId()); if (currentEntry != null && currentEntry.isWeightAccounting()) { - prepareTableForCachePublication(nameMapping, value); + prepareTableForCachePublication(nameMapping, loaded); } - return value; + return loaded; }); + ensureCatalogGenerationStable(catalog, ops, authenticator, nameMapping); + return value; + } + + private static ExecutionAuthenticator requireExecutionAuthenticator(CatalogIf catalog) { + if (!(catalog instanceof ExternalCatalog)) { + throw new RuntimeException("Iceberg metadata cache requires an external catalog"); + } + return ((ExternalCatalog) catalog).getExecutionAuthenticator(); + } + + private T execute(ExecutionAuthenticator authenticator, Callable task) { + try { + return authenticator.execute(task); + } catch (Exception e) { + throw new RuntimeException(ExceptionUtils.getRootCauseMessage(e), e); + } + } + + /** + * Validate that the catalog still serves the generation this acquisition started from. The + * check runs after the external load and before the result is used or published, so a + * concurrent reset turns into a clean retryable failure instead of a handle spliced with the + * next generation's execution context. + */ + private void ensureCatalogGenerationStable(CatalogIf catalog, IcebergMetadataOps ops, + ExecutionAuthenticator authenticator, NameMapping nameMapping) { + boolean stable; + try { + stable = resolveMetadataOps(catalog) == ops + && requireExecutionAuthenticator(catalog) == authenticator; + } catch (RuntimeException e) { + stable = false; + } + if (!stable) { + throw new RuntimeException(String.format( + "Catalog %d was reset while acquiring iceberg table %s.%s, please retry.", + nameMapping.getCtlId(), nameMapping.getLocalDbName(), nameMapping.getLocalTblName())); + } } MetaCacheSizeEstimate prepareTableForCachePublication( diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index bce765fd87488c..1ba3104dad13e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -341,9 +341,20 @@ private boolean isCurrentTableGeneration(NameMapping nameMapping, long tableGene private PaimonTableCacheValue loadTableCacheValue(NameMapping nameMapping) { try { + // One catalog generation must supply the loaded table and the captured execution + // context together: reading the authenticator only after the load could pair the old + // generation's table with the context a concurrent ALTER installed mid-flight (an + // ordinary credential ALTER does not retire this engine's cache group). Capture + // first, load, then re-validate before publication; a mid-flight reset fails the + // miss (the caller retries against the reinitialized catalog) instead of publishing + // a spliced synthetic generation that every later fence and schema hydration would + // deliberately trust. PaimonExternalCatalog catalog = tableLoader.catalog(nameMapping); - return new PaimonTableCacheValue( - tableLoader.load(nameMapping), catalog.getExecutionAuthenticator()); + org.apache.doris.common.security.authentication.ExecutionAuthenticator authenticator = + catalog.getExecutionAuthenticator(); + Table table = tableLoader.load(nameMapping); + ensureCatalogGenerationStable(catalog, authenticator, nameMapping); + return new PaimonTableCacheValue(table, authenticator); } catch (java.io.IOException e) { throw new CacheException("failed to load paimon table %s.%s.%s: %s", e, nameMapping.getCtlId(), nameMapping.getLocalDbName(), @@ -351,6 +362,23 @@ private PaimonTableCacheValue loadTableCacheValue(NameMapping nameMapping) { } } + private void ensureCatalogGenerationStable(PaimonExternalCatalog catalog, + org.apache.doris.common.security.authentication.ExecutionAuthenticator authenticator, + NameMapping nameMapping) { + boolean stable; + try { + stable = tableLoader.catalog(nameMapping) == catalog + && catalog.getExecutionAuthenticator() == authenticator; + } catch (Exception e) { + stable = false; + } + if (!stable) { + throw new CacheException("catalog %s was reset while loading paimon table %s.%s, please retry", + null, nameMapping.getCtlId(), nameMapping.getLocalDbName(), + nameMapping.getLocalTblName()); + } + } + /** * Execute on the authenticated context captured with the table generation, falling back to * the catalog's current authenticator for values that predate the capture. diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 601ca148040db9..ab900b15fe842c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -333,6 +333,101 @@ public void testSameGenerationRefreshWithRenewedFileIoRetiresSnapshotProjection( } } + @Test + public void testAcquisitionsFailWhenCatalogResetsMidFlight() { + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + IcebergMetadataOps opsOne = Mockito.mock(IcebergMetadataOps.class); + IcebergMetadataOps opsTwo = Mockito.mock(IcebergMetadataOps.class); + java.util.concurrent.atomic.AtomicReference currentOps = + new java.util.concurrent.atomic.AtomicReference<>(opsOne); + ExecutionAuthenticator generationOne = new ExecutionAuthenticator() { + @Override + public T execute(java.util.concurrent.Callable task) throws Exception { + return task.call(); + } + }; + ExecutionAuthenticator generationTwo = new ExecutionAuthenticator() { + @Override + public T execute(java.util.concurrent.Callable task) throws Exception { + return task.call(); + } + }; + java.util.concurrent.atomic.AtomicReference currentAuthenticator = + new java.util.concurrent.atomic.AtomicReference<>(generationOne); + java.util.concurrent.atomic.AtomicBoolean alterCompletesDuringLoad = + new java.util.concurrent.atomic.AtomicBoolean(true); + Mockito.when(catalog.getMetadataOps()).thenAnswer(invocation -> currentOps.get()); + Mockito.when(catalog.getExecutionAuthenticator()).thenAnswer( + invocation -> currentAuthenticator.get()); + Table table = tableWithMetadataLocation("/metadata/coherent-acquisition-v1.json"); + org.mockito.stubbing.Answer
    loadFlipsGeneration = invocation -> { + if (alterCompletesDuringLoad.get()) { + // The concurrent property/credential ALTER reinitializes the catalog while the + // external load is still in flight. + currentAuthenticator.set(generationTwo); + currentOps.set(opsTwo); + } + return table; + }; + Mockito.when(opsOne.loadTable("remote_db", "remote_tbl")).thenAnswer(loadFlipsGeneration); + Mockito.when(opsTwo.loadTable("remote_db", "remote_tbl")).thenAnswer(loadFlipsGeneration); + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor) { + @Override + protected CatalogIf getCatalog(long catalogId) { + return catalog; + } + }; + try { + cache.initCatalog(1L, Collections.emptyMap()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); + IcebergExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + MetaCacheEntry tables = cache.entry( + 1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class); + + // Miss-load boundary: the spliced acquisition must fail instead of publishing. + try { + tables.get(mapping); + Assert.fail("a table load spliced by a mid-flight catalog reset must not publish"); + } catch (RuntimeException e) { + Assert.assertTrue(String.valueOf(e.getMessage()), + exceptionChainContains(e, "was reset while acquiring iceberg table")); + } + Assert.assertNull(tables.peekIfPresent(mapping)); + + // Writable-handle boundary: same acquisition contract for DDL handles. + currentAuthenticator.set(generationOne); + currentOps.set(opsOne); + try { + cache.getWritableIcebergTable(dorisTable); + Assert.fail("a writable handle spliced by a mid-flight catalog reset must not be served"); + } catch (RuntimeException e) { + Assert.assertTrue(String.valueOf(e.getMessage()), + exceptionChainContains(e, "was reset while acquiring iceberg table")); + } + + // Retries run entirely against the settled second generation and stay coherent. + alterCompletesDuringLoad.set(false); + IcebergTableCacheValue published = tables.get(mapping); + Assert.assertSame(currentAuthenticator.get(), published.getAuthenticator()); + Assert.assertSame(table, cache.getWritableIcebergTable(dorisTable)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + private static boolean exceptionChainContains(Throwable throwable, String fragment) { + for (Throwable current = throwable; current != null; current = current.getCause()) { + if (current.getMessage() != null && current.getMessage().contains(fragment)) { + return true; + } + } + return false; + } + @Test public void testRejectedTableGenerationsDoNotAccumulateSnapshotOrSchemaProjections() { ExecutorService executor = Executors.newSingleThreadExecutor(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index bdfaf91a3f10ae..ead37b3b50f8c6 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -946,6 +946,79 @@ public T execute(Callable task) throws Exception { } } + @Test + public void testTableLoadSplicedByMidFlightCatalogResetFailsInsteadOfPublishing() { + ExecutionAuthenticator generationOne = new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + return task.call(); + } + }; + ExecutionAuthenticator generationTwo = new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + return task.call(); + } + }; + AtomicReference currentAuthenticator = + new AtomicReference<>(generationOne); + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.doReturn(catalog).when(catalogMgr) + .getCatalogOrException(Mockito.eq(1L), Mockito.any()); + Mockito.when(catalog.getExecutionAuthenticator()).thenAnswer( + invocation -> currentAuthenticator.get()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); + Table paimonTable = Mockito.mock(Table.class); + java.util.concurrent.atomic.AtomicBoolean alterCompletesDuringLoad = + new java.util.concurrent.atomic.AtomicBoolean(true); + Mockito.when(catalog.getPaimonTable(mapping)).thenAnswer(invocation -> { + if (alterCompletesDuringLoad.get()) { + // The concurrent credential ALTER reinitializes the catalog while the external + // load is still in flight. + currentAuthenticator.set(generationTwo); + } + return paimonTable; + }); + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + cache.initCatalog(1L, Collections.emptyMap()); + org.apache.doris.datasource.metacache.MetaCacheEntry tables = + cache.entry(1L, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class); + try { + tables.get(mapping); + Assert.fail("a table load spliced by a mid-flight catalog reset must not publish"); + } catch (RuntimeException e) { + Assert.assertTrue(String.valueOf(e.getMessage()), + exceptionChainContains(e, "was reset while loading paimon table")); + } + Assert.assertNull(tables.peekIfPresent(mapping)); + + // The retry runs entirely against the settled second generation and publishes a + // coherent (table, execution context) pair. + alterCompletesDuringLoad.set(false); + PaimonTableCacheValue published = tables.get(mapping); + Assert.assertSame(generationTwo, published.getAuthenticator()); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + + private static boolean exceptionChainContains(Throwable throwable, String fragment) { + for (Throwable current = throwable; current != null; current = current.getCause()) { + if (current.getMessage() != null && current.getMessage().contains(fragment)) { + return true; + } + } + return false; + } + @Test public void testFenceAndRelationHydrationStayOnCapturedGenerationAcrossAlter() { AtomicInteger authenticationDepth = new AtomicInteger(); From da14048c124a5b478af552cd6a807277a816987f Mon Sep 17 00:00:00 2001 From: guoqiang Date: Sat, 22 Aug 2026 17:12:35 +0800 Subject: [PATCH 35/45] [test](iceberg) Restore legacy wording expectation for create-time cache ttl validation CREATE-time validation of a new-style meta.cache.iceberg.*.ttl-second key reports the legacy "The parameter ... is wrong" wording (CacheSpec long checks), unlike ALTER-time strict validation and unlike legacy hive keys routed through the compatibility mapping, which report "must be >= -1". The previous sweep aligned this create case to the strict wording; CI shows the create path kept the legacy message, so restore the original expectation. The five hive cases and the iceberg ALTER case all passed with the strict wording and stay as they are. --- .../iceberg/test_iceberg_table_meta_cache.groovy | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy index 88016a49043e14..f322a75f3e44f1 100644 --- a/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy +++ b/regression-test/suites/external_table_p0/iceberg/test_iceberg_table_meta_cache.groovy @@ -109,7 +109,7 @@ suite("test_iceberg_table_meta_cache", "p0,external,doris,external_docker,extern 'meta.cache.iceberg.table.ttl-second' = '-2' ); """ - exception "must be >= -1" + exception "is wrong" } // disable iceberg table meta cache From 8cb50adebc0de44a6c07fe937e9ba832f7ea8fb0 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Sat, 22 Aug 2026 18:07:32 +0800 Subject: [PATCH 36/45] [fix](fe) Initialize before capturing catalog context and anchor writable DDL/DML to the dispatch generation The coherent-acquisition change read the execution authenticator before the call that used to trigger lazy catalog initialization, so a miss or writable acquisition reaching a reset-to-uninitialized catalog (retained external table across a credential/storage ALTER) threw before the catalog could reinitialize. Both the Iceberg and Paimon acquisitions now run makeSureInitialized() before capturing the generation's execution context; the post-load stability check reads without re-initializing so a mid-flight reset still counts as unstable. Writable acquisitions gained a dispatch-generation anchor: DDL methods of an IcebergMetadataOps instance and the DML transaction retain the ops and authenticator of the generation they were dispatched on, but previously fetched the live writable table from whatever generation the catalog served at that moment, splicing an old execution context with a newer handle. getWritableIcebergTable(dorisTable, expectedOps) now fails the acquisition (retryable) when the catalog no longer serves the caller's generation; every IcebergMetadataOps update site and the transaction's createTransactionTable pass their retained ops. Regressions: reset-before-miss and reset-before-writable initialization ordering for both engines, and the dispatch-generation anchor accepting the current ops while rejecting a retained stale ops. --- .../iceberg/IcebergExternalMetaCache.java | 33 +++++++-- .../iceberg/IcebergMetadataOps.java | 36 +++++----- .../iceberg/IcebergTransaction.java | 6 +- .../datasource/iceberg/IcebergUtils.java | 5 ++ .../paimon/PaimonExternalMetaCache.java | 5 ++ .../iceberg/IcebergExternalMetaCacheTest.java | 68 +++++++++++++++++++ .../iceberg/IcebergTransactionTest.java | 12 ++-- .../paimon/PaimonExternalMetaCacheTest.java | 50 ++++++++++++++ 8 files changed, 188 insertions(+), 27 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index 4a8becfafeca85..9e212ba54e6f0e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -137,6 +137,17 @@ public Table getIcebergTable(ExternalTable dorisTable) { } public Table getWritableIcebergTable(ExternalTable dorisTable) { + return getWritableIcebergTable(dorisTable, null); + } + + /** + * Acquire a writable table for a caller that retains the {@code IcebergMetadataOps} of the + * generation its operation was dispatched on (a DDL method of that ops instance, a DML + * transaction). The acquisition fails when the catalog has moved to a different generation, + * so the caller's later updates - executed through its retained ops and authenticator - + * can never operate a newer generation's handle. + */ + public Table getWritableIcebergTable(ExternalTable dorisTable, @Nullable IcebergMetadataOps expectedOps) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); CatalogIf catalog = getCatalog(nameMapping.getCtlId()); if (catalog == null) { @@ -150,6 +161,9 @@ public Table getWritableIcebergTable(ExternalTable dorisTable) { // and reinitialize the catalog mid-flight without retiring this engine's cache group. ExecutionAuthenticator authenticator = requireExecutionAuthenticator(catalog); IcebergMetadataOps ops = resolveMetadataOps(catalog); + if (expectedOps != null && ops != expectedOps) { + throw catalogGenerationMoved(nameMapping); + } Table table = execute(authenticator, () -> ops.loadTable( nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName())); ensureCatalogGenerationStable(catalog, ops, authenticator, nameMapping); @@ -357,6 +371,11 @@ private static ExecutionAuthenticator requireExecutionAuthenticator(CatalogIf if (!(catalog instanceof ExternalCatalog)) { throw new RuntimeException("Iceberg metadata cache requires an external catalog"); } + // A credential/storage ALTER can leave the catalog reset-to-uninitialized while queries, + // actions or transactions still retain this external table. The authenticator exists + // again only after lazy initialization, which the loads below used to trigger + // implicitly, so initialize before capturing the generation's execution context. + ((ExternalCatalog) catalog).makeSureInitialized(); return ((ExternalCatalog) catalog).getExecutionAuthenticator(); } @@ -378,18 +397,24 @@ private void ensureCatalogGenerationStable(CatalogIf catalog, IcebergMetadata ExecutionAuthenticator authenticator, NameMapping nameMapping) { boolean stable; try { + // Read without re-initializing: a catalog that was reset mid-flight must count as + // unstable here, not get quietly reinitialized by the validation itself. stable = resolveMetadataOps(catalog) == ops - && requireExecutionAuthenticator(catalog) == authenticator; + && ((ExternalCatalog) catalog).getExecutionAuthenticator() == authenticator; } catch (RuntimeException e) { stable = false; } if (!stable) { - throw new RuntimeException(String.format( - "Catalog %d was reset while acquiring iceberg table %s.%s, please retry.", - nameMapping.getCtlId(), nameMapping.getLocalDbName(), nameMapping.getLocalTblName())); + throw catalogGenerationMoved(nameMapping); } } + private static RuntimeException catalogGenerationMoved(NameMapping nameMapping) { + return new RuntimeException(String.format( + "Catalog %d was reset while acquiring iceberg table %s.%s, please retry.", + nameMapping.getCtlId(), nameMapping.getLocalDbName(), nameMapping.getLocalTblName())); + } + MetaCacheSizeEstimate prepareTableForCachePublication( NameMapping nameMapping, IcebergTableCacheValue value) { return value.prepareForCachePublication(nameMapping); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java index d996f80754a37f..0635e0a9a0c22a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java @@ -483,7 +483,7 @@ public void truncateTableImpl(ExternalTable dorisTable, List partitions) @Override public void createOrReplaceBranchImpl(ExternalTable dorisTable, CreateOrReplaceBranchInfo branchInfo) throws UserException { - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); BranchOptions branchOptions = branchInfo.getBranchOptions(); Long snapshotId = branchOptions.getSnapshotId() @@ -571,7 +571,7 @@ public void afterOperateOnBranchOrTag(String dbName, String tblName) { @Override public void createOrReplaceTagImpl(ExternalTable dorisTable, CreateOrReplaceTagInfo tagInfo) throws UserException { - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); TagOptions tagOptions = tagInfo.getTagOptions(); Long snapshotId = tagOptions.getSnapshotId() .orElse( @@ -623,7 +623,7 @@ public void createOrReplaceTagImpl(ExternalTable dorisTable, CreateOrReplaceTagI public void dropTagImpl(ExternalTable dorisTable, DropTagInfo tagInfo) throws UserException { String tagName = tagInfo.getTagName(); boolean ifExists = tagInfo.getIfExists(); - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); SnapshotRef snapshotRef = icebergTable.refs().get(tagName); if (snapshotRef != null || !ifExists) { @@ -644,7 +644,7 @@ public void dropTagImpl(ExternalTable dorisTable, DropTagInfo tagInfo) throws Us public void dropBranchImpl(ExternalTable dorisTable, DropBranchInfo branchInfo) throws UserException { String branchName = branchInfo.getBranchName(); boolean ifExists = branchInfo.getIfExists(); - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); SnapshotRef snapshotRef = icebergTable.refs().get(branchName); if (snapshotRef != null || !ifExists) { @@ -747,7 +747,7 @@ private void refreshTable(ExternalTable dorisTable, long updateTime) { public void addColumn(ExternalTable dorisTable, Column column, ColumnPosition position, long updateTime) throws UserException { validateAddColumnMetadata(column, true); - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); validateVariantSchema(icebergTable, column.getType(), column.getName(), false, true); validateRowLineageColumnMutation(icebergTable, column.getName(), "add"); Schema schema = icebergTable.schema(); @@ -778,7 +778,7 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co if (!column.isAllowNull()) { throw new UserException("New nested field '" + columnPath.getFullPath() + "' must be nullable"); } - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); validateVariantSchema(icebergTable, column.getType(), columnPath.getFullPath(), true, true); ResolvedColumnPath parentPath = resolveColumnPath(icebergTable.schema(), columnPath.getParentPath(), "add"); if (!parentPath.getType().isStructType()) { @@ -808,7 +808,7 @@ public void addColumn(ExternalTable dorisTable, ColumnPath columnPath, Column co @Override public void addColumns(ExternalTable dorisTable, List columns, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); for (Column column : columns) { validateAddColumnMetadata(column, true); validateVariantSchema(icebergTable, column.getType(), column.getName(), false, true); @@ -831,7 +831,7 @@ public void addColumns(ExternalTable dorisTable, List columns, long upda @Override public void dropColumn(ExternalTable dorisTable, String columnName, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); validateRowLineageColumnMutation(icebergTable, columnName, "drop"); ResolvedColumnPath columnPath = resolveColumnPath(icebergTable.schema(), ColumnPath.of(columnName), "drop"); UpdateSchema updateSchema = icebergTable.updateSchema(); @@ -851,7 +851,7 @@ public void dropColumn(ExternalTable dorisTable, ColumnPath columnPath, long upd dropColumn(dorisTable, columnPath.getTopLevelName(), updateTime); return; } - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); ResolvedColumnPath resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "drop"); UpdateSchema updateSchema = icebergTable.updateSchema(); @@ -868,7 +868,7 @@ public void dropColumn(ExternalTable dorisTable, ColumnPath columnPath, long upd @Override public void renameColumn(ExternalTable dorisTable, String oldName, String newName, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); validateRowLineageColumnMutation(icebergTable, oldName, "rename"); validateRowLineageColumnMutation(icebergTable, newName, "rename to"); Schema schema = icebergTable.schema(); @@ -893,7 +893,7 @@ public void renameColumn(ExternalTable dorisTable, ColumnPath columnPath, String renameColumn(dorisTable, columnPath.getTopLevelName(), newName, updateTime); return; } - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); ResolvedColumnPath resolvedPath = validateNestedStructFieldPath(icebergTable.schema(), columnPath, "rename"); ResolvedColumnPath parentPath = resolveColumnPath(icebergTable.schema(), columnPath.getParentPath(), "rename"); validateNoCaseInsensitiveSiblingCollision(parentPath.getType().asStructType(), @@ -955,7 +955,7 @@ public void modifyColumn(ExternalTable dorisTable, Column column, ColumnPosition private void modifyTopLevelColumn(ExternalTable dorisTable, ColumnPath columnPath, Column column, ColumnPosition position, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); validateRowLineageColumnMutation(icebergTable, columnPath.getTopLevelName(), "modify"); NestedField currentCol = icebergTable.schema().asStruct() .caseInsensitiveField(columnPath.getTopLevelName()); @@ -1024,7 +1024,7 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column return; } - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); ResolvedColumnPath resolvedPath = resolveColumnPath(icebergTable.schema(), columnPath, "modify"); NestedField currentCol = resolvedPath.getField(); validateCollectionPseudoFieldComment( @@ -1075,7 +1075,7 @@ public void modifyColumn(ExternalTable dorisTable, ColumnPath columnPath, Column @Override public void modifyColumnComment(ExternalTable dorisTable, ColumnPath columnPath, String comment, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); if (!columnPath.isNested()) { validateRowLineageColumnMutation(icebergTable, columnPath.getTopLevelName(), "modify comment for"); } @@ -1642,7 +1642,7 @@ public void reorderColumns(ExternalTable dorisTable, List newOrder, long if (newOrder == null || newOrder.isEmpty()) { throw new UserException("Reorder column failed, new order is empty."); } - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); List canonicalOrder = new ArrayList<>(newOrder.size()); Set canonicalNames = new TreeSet<>(String.CASE_INSENSITIVE_ORDER); for (String columnName : newOrder) { @@ -1709,7 +1709,7 @@ private Term getTransform(String transformName, String columnName, Integer trans */ public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldClause clause, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); String transformName = clause.getTransformName(); @@ -1738,7 +1738,7 @@ public void addPartitionField(ExternalTable dorisTable, AddPartitionFieldClause */ public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldClause clause, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); if (clause.getPartitionFieldName() != null) { @@ -1765,7 +1765,7 @@ public void dropPartitionField(ExternalTable dorisTable, DropPartitionFieldClaus */ public void replacePartitionField(ExternalTable dorisTable, ReplacePartitionFieldClause clause, long updateTime) throws UserException { - Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable); + Table icebergTable = IcebergUtils.getWritableIcebergTable(dorisTable, this); UpdatePartitionSpec updateSpec = icebergTable.updateSpec(); // remove old partition field diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java index 50d245ab0ec6d6..d2366b704d7a98 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java @@ -298,8 +298,12 @@ private Table createTransactionTable(ExternalTable dorisTable, Table retainedTab } // Reads stay on the retained generation; commit refreshes may follow data-only snapshots, // while writer-contract changes still invalidate files produced for the retained metadata. + // Anchor the live handle to this transaction's retained ops generation: if a + // credential/storage ALTER reinitialized the catalog since dispatch, acquiring the newer + // generation's table here would splice it with the retained ops/authenticator this + // transaction keeps using for begin/update/commit. return IcebergSnapshotCacheValue.createWritableTable( - retainedTable, IcebergUtils.getWritableIcebergTable(dorisTable)); + retainedTable, IcebergUtils.getWritableIcebergTable(dorisTable, ops)); } /** Begin UPDATE/MERGE against the metadata generation retained by the merge sink. */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index 078fd00c2b5726..e88b9d1b09c0ff 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -1066,6 +1066,11 @@ public static Table getWritableIcebergTable(ExternalTable dorisTable) { return icebergExternalMetaCache(dorisTable).getWritableIcebergTable(dorisTable); } + /** Writable acquisition anchored to the caller's retained catalog generation. */ + public static Table getWritableIcebergTable(ExternalTable dorisTable, IcebergMetadataOps expectedOps) { + return icebergExternalMetaCache(dorisTable).getWritableIcebergTable(dorisTable, expectedOps); + } + private static IcebergExternalMetaCache icebergExternalMetaCache(ExternalCatalog catalog) { Preconditions.checkNotNull(catalog, "catalog can not be null"); return Env.getCurrentEnv().getExtMetaCacheMgr().iceberg(catalog.getId()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index 1ba3104dad13e6..47054047dbce47 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -350,6 +350,11 @@ private PaimonTableCacheValue loadTableCacheValue(NameMapping nameMapping) { // a spliced synthetic generation that every later fence and schema hydration would // deliberately trust. PaimonExternalCatalog catalog = tableLoader.catalog(nameMapping); + // A credential/storage ALTER can leave the catalog reset-to-uninitialized while a + // query still retains this external table. The authenticator exists again only after + // lazy initialization, which the load below used to trigger implicitly, so + // initialize before capturing the generation's execution context. + catalog.makeSureInitialized(); org.apache.doris.common.security.authentication.ExecutionAuthenticator authenticator = catalog.getExecutionAuthenticator(); Table table = tableLoader.load(nameMapping); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index ab900b15fe842c..063e6f9bb0fd6c 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -419,6 +419,74 @@ protected CatalogIf getCatalog(long catalogId) { } } + @Test + public void testResetCatalogReinitializesBeforeCaptureAndWritableStaysOnDispatchGeneration() { + IcebergExternalCatalog catalog = Mockito.mock(IcebergExternalCatalog.class); + IcebergMetadataOps retainedOps = Mockito.mock(IcebergMetadataOps.class); + IcebergMetadataOps currentOps = Mockito.mock(IcebergMetadataOps.class); + ExecutionAuthenticator authenticator = new ExecutionAuthenticator() { + @Override + public T execute(java.util.concurrent.Callable task) throws Exception { + return task.call(); + } + }; + java.util.concurrent.atomic.AtomicBoolean initialized = + new java.util.concurrent.atomic.AtomicBoolean(false); + Mockito.doAnswer(invocation -> { + initialized.set(true); + return null; + }).when(catalog).makeSureInitialized(); + Mockito.when(catalog.getExecutionAuthenticator()).thenAnswer(invocation -> { + if (!initialized.get()) { + throw new RuntimeException( + "ExecutionAuthenticator is null, please confirm it is initialized."); + } + return authenticator; + }); + Mockito.when(catalog.getMetadataOps()).thenReturn(currentOps); + Table table = tableWithMetadataLocation("/metadata/reset-before-capture-v1.json"); + Mockito.when(currentOps.loadTable("remote_db", "remote_tbl")).thenReturn(table); + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor) { + @Override + protected CatalogIf getCatalog(long catalogId) { + return catalog; + } + }; + try { + cache.initCatalog(1L, Collections.emptyMap()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); + IcebergExternalTable dorisTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + MetaCacheEntry tables = cache.entry( + 1L, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class); + + // A reset-to-uninitialized catalog must be initialized before the miss captures its + // execution context, exactly as the load used to trigger implicitly. + IcebergTableCacheValue published = tables.get(mapping); + Assert.assertSame(authenticator, published.getAuthenticator()); + + initialized.set(false); + Assert.assertSame(table, cache.getWritableIcebergTable(dorisTable)); + Assert.assertTrue(initialized.get()); + + // A caller that retains the ops of an earlier dispatch generation must not be handed + // the reinitialized generation's writable handle. + try { + cache.getWritableIcebergTable(dorisTable, retainedOps); + Assert.fail("a writable acquisition must stay on the caller's dispatch generation"); + } catch (RuntimeException e) { + Assert.assertTrue(String.valueOf(e.getMessage()), + exceptionChainContains(e, "was reset while acquiring iceberg table")); + } + Assert.assertSame(table, cache.getWritableIcebergTable(dorisTable, currentOps)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + private static boolean exceptionChainContains(Throwable throwable, String fragment) { for (Throwable current = throwable; current != null; current = current.getCause()) { if (current.getMessage() != null && current.getMessage().contains(fragment)) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java index bd966a75332568..4f6ac1fb2bf34f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergTransactionTest.java @@ -732,7 +732,8 @@ public void testQueryScopedGenerationCommitsThroughWritableOperations() throws U try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(liveTable); + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable( + Mockito.eq(dorisTable), ArgumentMatchers.any())).thenReturn(liveTable); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, queryScopedTable, Optional.empty()); @@ -762,7 +763,8 @@ public void testRetainedGenerationCommitsThroughWritableOperations() throws User try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(liveTable); + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable( + Mockito.eq(dorisTable), ArgumentMatchers.any())).thenReturn(liveTable); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, retainedTable, Optional.empty()); @@ -812,7 +814,8 @@ public void testRetainedGenerationRejectsConcurrentMetadataAdvance() throws User try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(liveTable); + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable( + Mockito.eq(dorisTable), ArgumentMatchers.any())).thenReturn(liveTable); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, retainedTable, Optional.empty()); @@ -841,7 +844,8 @@ public void testRetainedGenerationRetriesAfterConcurrentDataCommit() throws User try (MockedStatic mockedUtils = Mockito.mockStatic( IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(liveTable); + mockedUtils.when(() -> IcebergUtils.getWritableIcebergTable( + Mockito.eq(dorisTable), ArgumentMatchers.any())).thenReturn(liveTable); IcebergTransaction txn = getTxn(); txn.updateIcebergCommitData(Collections.singletonList(commitData)); txn.beginInsert(dorisTable, retainedTable, Optional.empty()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index ead37b3b50f8c6..19963bb1307c07 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -1010,6 +1010,56 @@ public T execute(Callable task) throws Exception { } } + @Test + public void testResetCatalogReinitializesBeforeCapturingAuthenticator() { + ExecutionAuthenticator authenticator = new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + return task.call(); + } + }; + java.util.concurrent.atomic.AtomicBoolean initialized = + new java.util.concurrent.atomic.AtomicBoolean(false); + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.doReturn(catalog).when(catalogMgr) + .getCatalogOrException(Mockito.eq(1L), Mockito.any()); + Mockito.doAnswer(invocation -> { + initialized.set(true); + return null; + }).when(catalog).makeSureInitialized(); + Mockito.when(catalog.getExecutionAuthenticator()).thenAnswer(invocation -> { + if (!initialized.get()) { + throw new RuntimeException( + "ExecutionAuthenticator is null, please confirm it is initialized."); + } + return authenticator; + }); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); + Table paimonTable = Mockito.mock(Table.class); + Mockito.when(catalog.getPaimonTable(mapping)).thenReturn(paimonTable); + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + cache.initCatalog(1L, Collections.emptyMap()); + org.apache.doris.datasource.metacache.MetaCacheEntry tables = + cache.entry(1L, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class); + + // A reset-to-uninitialized catalog must be initialized before the miss captures its + // execution context, exactly as the load used to trigger implicitly. + PaimonTableCacheValue published = tables.get(mapping); + Assert.assertSame(authenticator, published.getAuthenticator()); + Assert.assertTrue(initialized.get()); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + private static boolean exceptionChainContains(Throwable throwable, String fragment) { for (Throwable current = throwable; current != null; current = current.getCause()) { if (current.getMessage() != null && current.getMessage().contains(fragment)) { From fc76ea6b9675551b30cbb9ee18abc25ecad13adb Mon Sep 17 00:00:00 2001 From: guoqiang Date: Sun, 23 Aug 2026 00:45:13 +0800 Subject: [PATCH 37/45] [fix](paimon) Serve the memoized latest projection instead of re-reading the fence per query The fence-keyed latest model read the snapshot fence from storage on every latest lookup, so a plain query observed external commits immediately whenever the SDK layer did not serve a stale snapshot. That intermittently broke the established external metadata cache contract (latest metadata stays as stale as the cached table handle until TTL/refresh, verified by the merged test_paimon_table_meta_cache regression) and paid one snapshot read per query. getSnapshotCache now serves the projection of the most recently observed fence while it is still published for the current table generation, and re-observes the fence only when no projection of the generation is reachable anymore (first read, expiry, weight eviction, explicit invalidation). Staleness and IO now match the memoized pre-governance behavior, while fence observation ordering keeps handling concurrent re-observation and rollback replacement exactly as before. --- .../paimon/PaimonExternalMetaCache.java | 17 +++++++++++++++++ .../paimon/PaimonExternalMetaCacheTest.java | 18 ++++++++++++------ 2 files changed, 29 insertions(+), 6 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index 47054047dbce47..04747a2a46ef8a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -116,6 +116,23 @@ public PaimonSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); MetaCacheEntry tables = tableEntry.get(nameMapping.getCtlId()); PaimonTableCacheValue tableValue = tables.get(nameMapping); + if (tables.isEffectivelyEnabled()) { + // Serve the memoized latest projection of this table generation while it is still + // published: the latest read is as stale-until-TTL/refresh as the cached table + // handle itself and costs no snapshot IO, preserving the pre-existing external + // metadata cache contract. The fence is re-observed only when no projection of this + // generation is reachable anymore (first read, expiry, weight eviction, explicit + // invalidation), which is also when rollback ordering below matters. + ObservedFence observed = latestObservedFences.get( + new LatestFenceOwner(nameMapping, tableValue.getGeneration())); + if (observed != null) { + PaimonSnapshotCacheValue memoized = + snapshotEntry.get(nameMapping.getCtlId()).peekIfPresent(observed.key); + if (memoized != null) { + return memoized; + } + } + } PaimonSnapshot fence = loadLatestSnapshotFence(nameMapping, tableValue).getSnapshot(); if (!tables.isEffectivelyEnabled()) { // Projections are keyed by the synthetic generation of a published table handle. An diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 19963bb1307c07..715272c021961d 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -779,7 +779,7 @@ public void testReplacingTableGenerationRetiresSnapshotAndSchemaProjection() { } @Test - public void testSnapshotHitRefreshesFenceWithoutReloadingProjection() { + public void testMemoizedLatestProjectionServesHitsWithoutFenceReads() { ExecutorService executor = Executors.newSingleThreadExecutor(); PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); @@ -830,7 +830,9 @@ public T execute(Callable task) throws Exception { Assert.assertEquals(7L, cache.loadLatestSnapshotFence(dorisTable).getSnapshot().getSnapshotId()); Assert.assertEquals(7L, cache.loadLatestSnapshotFence(dorisTable).getSnapshot().getSnapshotId()); - Mockito.verify(table, Mockito.times(4)).copyWithLatestSchema(); + // The first latest read observes the fence (one IO); the second is served from the + // memoized projection without touching storage. Explicit statement fences always read. + Mockito.verify(table, Mockito.times(3)).copyWithLatestSchema(); } finally { cache.close(); executor.shutdownNow(); @@ -1457,9 +1459,11 @@ public void testAdvancingLatestFenceKeepsOnlyNewestProjectionOfTableGeneration() Assert.assertSame(at7, snapshots.peekIfPresent(key7)); Assert.assertSame(at7, cache.getSnapshotCache(dorisTable)); - // Commits observed before the table handle refreshes advance the fence: only the - // newest projection of this generation stays reachable. + // Commits observed before the table handle refreshes advance the fence once the + // memoized projection is gone (expiry/eviction/invalidation): only the newest + // projection of this generation stays reachable. mocked.latestSnapshotId.set(8L); + snapshots.invalidateKey(key7); PaimonSnapshotCacheValue at8 = cache.getSnapshotCache(dorisTable); PaimonSnapshotEntryKey key8 = new PaimonSnapshotEntryKey(mapping, 8L, 3L, tableValue.getGeneration()); Assert.assertEquals(8L, at8.getSnapshot().getSnapshotId()); @@ -1499,9 +1503,11 @@ public void testAdvancingLatestFenceKeepsOnlyNewestProjectionOfTableGeneration() olderCall.shutdownNow(); } - // Rollback: the latest snapshot moves backwards; the newly observed fence replaces the - // projection of the higher snapshot id instead of being retired by it. + // Rollback: the latest snapshot moves backwards; once the memoized projection is + // dropped, the newly observed fence replaces the projection of the higher snapshot id + // instead of being retired by it. mocked.latestSnapshotId.set(8L); + snapshots.invalidateKey(key9); PaimonSnapshotCacheValue rolledBack = cache.getSnapshotCache(dorisTable); Assert.assertEquals(8L, rolledBack.getSnapshot().getSnapshotId()); Assert.assertSame(rolledBack, snapshots.peekIfPresent(key8)); From a4cbf89a6042388b8f8ca32010a7dcbecf890391 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Sun, 23 Aug 2026 00:45:13 +0800 Subject: [PATCH 38/45] [test](iceberg) Align remaining writable-acquisition stubs with the dispatch-generation overload IcebergMetadataOps DDL sites now acquire writable tables through the two-argument dispatch-generation overload; the validation, branch/tag and DDL-plan suites still stubbed only the single-argument form, so the real method ran without an Env and failed (and the leaked static mock state could poison unrelated tests sharing the fork). --- .../iceberg/IcebergDDLAndDMLPlanTest.java | 3 + .../IcebergExternalTableBranchAndTagTest.java | 2 + .../IcebergMetadataOpsValidationTest.java | 70 +++++++++---------- 3 files changed, 40 insertions(+), 35 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java index fd044a0d3ea4ea..3ac94656dde4eb 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergDDLAndDMLPlanTest.java @@ -233,6 +233,9 @@ protected void runBeforeAll() throws Exception { }); icebergUtilsMock.when(() -> IcebergUtils.getWritableIcebergTable( ArgumentMatchers.any(ExternalTable.class))).thenReturn(mockedIcebergTable); + icebergUtilsMock.when(() -> IcebergUtils.getWritableIcebergTable( + ArgumentMatchers.any(ExternalTable.class), ArgumentMatchers.any())) + .thenReturn(mockedIcebergTable); } @Override diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java index de3ab8c9746397..5a3590602a7a17 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalTableBranchAndTagTest.java @@ -96,6 +96,8 @@ public void setUp() throws IOException { mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class); mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.any())) .thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.any(), Mockito.any())) + .thenReturn(icebergTable); // mock Env.getCurrentEnv().getEditLog().logBranchOrTag(info) to do nothing Env mockEnv = Mockito.mock(Env.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java index cc019c73ed12a7..de3a20eca59dcc 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergMetadataOpsValidationTest.java @@ -138,7 +138,7 @@ public void testTopLevelVariantModifyOnlyUpdatesMetadataOnOrcTable() throws Thro try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)) + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())) .thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("payload"), column, ColumnPosition.FIRST, 1L); @@ -166,7 +166,7 @@ public void testTopLevelVariantModifyRejectsTypeConversions() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)) + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())) .thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("variant_col"), @@ -296,7 +296,7 @@ public void testRejectUnsupportedIcebergTargetTypesBeforeUpdateSchema() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); assertUserException(() -> ops.addColumn(dorisTable, ColumnPath.fromDotName("info.new_field"), new Column("new_field", Type.LARGEINT, true), null, 1L), @@ -330,7 +330,7 @@ public void testComplexModifyPreservesRequiredNestedFields() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.child"), new Column("child", new StructType(new StructField("value", Type.BIGINT)), true), null, 1L); @@ -365,7 +365,7 @@ public void testComplexModifyPersistsDecodedStructMemberComment() throws Throwab try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.payload"), column, null, 1L); } @@ -393,7 +393,7 @@ public void testPrimitiveModifyPreservesOmittedCommentAndClearsExplicitEmptyComm try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), new Column("metric", Type.BIGINT, true), null, 1L); @@ -435,7 +435,7 @@ public void testFullStructModifyPreservesOmittedChildComments() throws Throwable try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.payload"), new Column("payload", payloadType, true), null, 1L); @@ -465,7 +465,7 @@ public void testPrimitiveModifyPreservesRequiredNestedField() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), new Column("metric", Type.BIGINT, true), null, 1L); @@ -491,7 +491,7 @@ public void testTopLevelModifyPreservesRequiredMixedCaseFields() throws Throwabl try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("id"), new Column("id", Type.BIGINT, true), null, 1L); @@ -517,7 +517,7 @@ public void testTopLevelModifyDoesNotResolveQuotedComponentAsNestedPath() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.of("a.b"), new Column("a.b", Type.BIGINT, true), null, 1L), @@ -539,7 +539,7 @@ public void testTopLevelModifyPreservesDottedTopLevelName() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("a.b"), new Column("a.b", Type.BIGINT, true), null, 1L); @@ -570,7 +570,7 @@ public void testPrimitiveModifyPreservesActualTypeWhenMappingDisabled() throws T try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("top_uuid"), topUuid, ColumnPosition.FIRST, 1L); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.uuid_value"), nestedUuid, @@ -605,7 +605,7 @@ public void testPrimitiveModifyPreservesActualTypeWhenMappingEnabled() throws Th try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("top_uuid"), new Column("top_uuid", ScalarType.createVarbinaryType(16), true), null, 1L); @@ -635,7 +635,7 @@ public void testComplexModifyIgnoresUnchangedMappedChildren() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("outer.payload"), new Column("payload", mappedPayloadDorisType(Type.BIGINT, 8, @@ -663,7 +663,7 @@ public void testComplexModifyRejectsChangedUnsupportedMappedChildrenBeforeUpdate try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("outer.payload"), new Column("payload", mappedPayloadDorisType(Type.LARGEINT, 8, @@ -694,7 +694,7 @@ public void testLegacyModifyColumnTreatsNullabilityAsExplicit() throws Throwable try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); // Iceberg schema columns are represented as keys in Doris, so the legacy API must not // interpret isKey as an explicit KEY clause. @@ -725,7 +725,7 @@ public void testLegacyComplexModifyDoesNotInferRecursiveNullableChanges() throws try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); ops.modifyColumn(dorisTable, column, null, 1L); } @@ -759,7 +759,7 @@ public void testExplicitNullableModifyMakesRequiredFieldsOptional() throws Throw try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.of("info"), topLevelColumn, null, 1L); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("info.metric"), nestedColumn, null, 1L); @@ -812,7 +812,7 @@ public void execute(Runnable task) { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(staleTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(staleTable); try { conflictOps.modifyColumn(dorisTable, ColumnPath.of("info"), @@ -863,7 +863,7 @@ public void testRenamePreservesNestedIdentifierFieldPaths() throws Exception { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); ops.renameColumn(dorisTable, ColumnPath.fromDotName("root.child.id"), "renamed_id", 1L); icebergTable.refresh(); @@ -913,7 +913,7 @@ public void testRenameDoesNotRewriteDottedIdentifierSibling() throws Exception { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); ops.renameColumn(dorisTable, "a", "renamed", 1L); icebergTable.refresh(); @@ -942,7 +942,7 @@ public void testNestedColumnOperationsRejectDefaultMetadata() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); assertUserException(() -> ops.addColumn(dorisTable, ColumnPath.fromDotName("s.new_col"), nestedAddDefaultColumn, null, 1L), @@ -973,7 +973,7 @@ public void testTopLevelColumnOperationsRejectUnsupportedDefaultMetadata() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, defaultColumn, null, 1L), "Modifying default values is not supported for Iceberg columns: id"); @@ -1004,7 +1004,7 @@ public void testUnsupportedPrimitiveModifyFailsBeforeUpdateSchema() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn( dorisTable, ColumnPath.of("info"), new Column("info", Type.INT, true), null, 1L), @@ -1037,7 +1037,7 @@ public void testRejectKeyAndGeneratedMetadataBeforeUpdateSchema() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); assertUserException(() -> ops.addColumn(dorisTable, keyColumn, null, 1L), "KEY is not supported for Iceberg ADD/MODIFY COLUMN"); @@ -1084,7 +1084,7 @@ public void testModifyComplexColumnRejectsCaseInsensitiveStructFieldAdditions() try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn( dorisTable, new Column("info", infoType, true), null, 1L), @@ -1160,7 +1160,7 @@ public void testTopLevelCaseInsensitiveCollisionsAndCaseOnlyRename() throws Thro try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); assertUserException(() -> ops.addColumn( dorisTable, new Column("id", Type.STRING, true), null, 1L), @@ -1196,7 +1196,7 @@ public void testReorderColumnsUsesCanonicalIcebergNames() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); ops.reorderColumns(dorisTable, Arrays.asList("label", "id"), 1L); } @@ -1218,7 +1218,7 @@ public void testModifyColumnSupportsDirectArrayElementAndMapValue() throws Throw try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); ops.modifyColumn(dorisTable, ColumnPath.fromDotName("arr.element"), new Column("element", Type.BIGINT, true), null, 1L); @@ -1241,7 +1241,7 @@ public void testModifyColumnRejectsPositionForDirectArrayElementAndMapValue() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumn(dorisTable, ColumnPath.fromDotName("arr.element"), new Column("element", Type.BIGINT, true), ColumnPosition.FIRST, 1L), @@ -1272,7 +1272,7 @@ public void testModifyColumnCommentUsesCanonicalNestedPaths() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); ops.modifyColumnComment(dorisTable, ColumnPath.fromDotName("info.metric"), "struct comment", 1L); @@ -1298,7 +1298,7 @@ public void testRejectsCommentsOnDirectArrayElementAndMapValue() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); assertUserException(() -> ops.modifyColumnComment( dorisTable, ColumnPath.fromDotName("arr.element"), "array element comment", 1L), @@ -1338,7 +1338,7 @@ public void testRejectsTopLevelRowLineageMutationsForV3Tables() { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(dorisTable)).thenReturn(icebergTable); + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(dorisTable), Mockito.any())).thenReturn(icebergTable); assertUserException(() -> ops.addColumn(dorisTable, new Column("_row_id", Type.BIGINT, true), null, 1L), @@ -1394,9 +1394,9 @@ public void testAllowsV3NestedAndV2TopLevelRowLineageNames() throws Throwable { try (MockedStatic mockedIcebergUtils = Mockito.mockStatic(IcebergUtils.class, Mockito.CALLS_REAL_METHODS)) { - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(v3DorisTable)) + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(v3DorisTable), Mockito.any())) .thenReturn(v3IcebergTable); - mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(v2DorisTable)) + mockedIcebergUtils.when(() -> IcebergUtils.getWritableIcebergTable(Mockito.eq(v2DorisTable), Mockito.any())) .thenReturn(v2IcebergTable); ops.addColumn(v3DorisTable, ColumnPath.fromDotName("s._last_updated_sequence_number"), From 246b5445097c18e66c527718aa01099c80739eb6 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Sun, 23 Aug 2026 01:53:00 +0800 Subject: [PATCH 39/45] [fix](fe) Fence frozen-generation planning on its captured context and charge the retained context A statement pinned to a frozen Iceberg generation planned that generation's FrozenTableOperations/FileIO with whatever authenticator, storage state and pre-authenticated executor the catalog currently served: doInitialize captures the current context independently of the relation-pinned table, so a credential/storage ALTER between binding and planning spliced the generations. Iceberg snapshot projections now carry the ExecutionAuthenticator captured from the table generation that built them, and IcebergScanNode.useFrozenTableGeneration - the shared entry for regular relations and snapshot-selectable system tables - validates it against the catalog's current context before planning; a replaced context fails the statement retryably instead of planning a spliced scan. Estimator side, an admitted table or snapshot value is a strong owner of its generation's execution context (for Kerberized catalogs that graph includes the Hadoop authentication state and credential collections), and a retired generation can stay alive only through such values. Both the Iceberg and Paimon estimators now add a flat rounded-up retained-context allowance (16KB, mirroring the existing FileIO and encryption-manager allowances) to every value that carries a bound context, so those retentions are never entirely unaccounted. Regressions: allowance deltas for bound vs unbound table and snapshot values in both engines, projection context propagation from the published generation, and the planning fence accepting the captured context while rejecting a replaced one. --- .../iceberg/IcebergCacheSizeEstimator.java | 13 +++++ .../iceberg/IcebergExternalMetaCache.java | 6 ++- .../iceberg/IcebergSnapshotCacheValue.java | 35 +++++++++++++ .../iceberg/source/IcebergScanNode.java | 8 +++ .../paimon/PaimonCacheSizeEstimator.java | 8 +++ .../iceberg/IcebergExternalMetaCacheTest.java | 51 +++++++++++++++++++ .../paimon/PaimonExternalMetaCacheTest.java | 27 ++++++++++ 7 files changed, 146 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java index 0b51299ac4dbed..62922f2a6f9016 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergCacheSizeEstimator.java @@ -118,6 +118,13 @@ final class IcebergCacheSizeEstimator { // The frozen operations also retain the handle's EncryptionManager; scans effectively meet // the shared plaintext singleton, so a small fixed allowance covers the object graph. private static final long ENCRYPTION_MANAGER_WEIGHT = 1024L; + /** + * Flat allowance for the execution context a published value retains (authenticator, + * Hadoop authentication state, credential collections). The graph is shared by every value + * of the catalog generation; each retaining value carries the rounded-up allowance so a + * retired generation kept alive only by old cached values is never entirely unaccounted. + */ + private static final long AUTHENTICATION_CONTEXT_WEIGHT = 16L * 1024L; private static final long MANIFEST_ENTRY_BASE_WEIGHT = 512L; private static final long DATA_FILE_WEIGHT = 1024L; private static final long DELETE_FILE_WEIGHT = 1024L; @@ -146,6 +153,9 @@ static MetaCacheSizeEstimate estimateTableEntry(NameMapping key, IcebergTableCac bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedTablePayloadBytes()); bytes = MetaCacheWeightUtils.saturatedAdd( bytes, value.getRetainedCurrentSnapshotPayloadBytes()); + if (value.getAuthenticator() != null) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, AUTHENTICATION_CONTEXT_WEIGHT); + } return MetaCacheSizeEstimate.complete(bytes); } @@ -187,6 +197,9 @@ static MetaCacheSizeEstimate estimateSnapshotEntry( bytes = MetaCacheWeightUtils.saturatedAdd( bytes, value.getRetainedCurrentSnapshotPayloadBytes()); } + if (value.getCapturedAuthenticator() != null) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, AUTHENTICATION_CONTEXT_WEIGHT); + } return MetaCacheSizeEstimate.complete(bytes); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index 9e212ba54e6f0e..a1409715ba0f45 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -206,7 +206,8 @@ public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { isolateForQueries ? tableValue.newQueryScopedTable() : tableValue.getIcebergTable(), tableValue.getRetainedIcebergTable(), - tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries)); + tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries)) + .bindCapturedAuthenticator(tableValue.getAuthenticator()); } IcebergSnapshotEntryKey key = optionalKey.get(); MetaCacheEntry entry = @@ -220,7 +221,8 @@ public IcebergSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { IcebergSnapshotCacheValue value = loadSnapshotProjection( dorisTable, projectionTable, tableValue.getRetainedIcebergTable(), - tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries); + tableValue.getRetainedCurrentSnapshotJson(), isolateForQueries) + .bindCapturedAuthenticator(tableValue.getAuthenticator()); if (entry.isWeightAccounting()) { value.prepareForCachePublication(key); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java index f7dbed2effd183..d1a4e1be59dd4b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergSnapshotCacheValue.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.iceberg; +import org.apache.doris.common.security.authentication.ExecutionAuthenticator; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimate; import org.apache.doris.datasource.metacache.MetaCacheSizeEstimator; import org.apache.doris.datasource.metacache.MetaCacheWeightUtils; @@ -42,6 +43,7 @@ import java.util.Map; import java.util.Objects; import java.util.Optional; +import javax.annotation.Nullable; public class IcebergSnapshotCacheValue { @@ -54,6 +56,13 @@ public class IcebergSnapshotCacheValue { private boolean queryIsolationPrepared; private long retainedTablePayloadBytes; private MetaCacheSizeEstimate sizeEstimate; + /** + * Execution context captured from the table generation this projection retains. Planning and + * scanning the retained frozen table must run under this context; it is not part of the + * per-value counted payload beyond the flat retained-context allowance the estimator adds. + */ + @Nullable + private transient volatile ExecutionAuthenticator capturedAuthenticator; public IcebergSnapshotCacheValue(IcebergPartitionInfo partitionInfo, IcebergSnapshot snapshot) { this(partitionInfo, snapshot, Optional.empty(), Optional.empty(), null, false); @@ -122,6 +131,32 @@ public IcebergSnapshot getSnapshot() { return snapshot; } + public IcebergSnapshotCacheValue bindCapturedAuthenticator(@Nullable ExecutionAuthenticator authenticator) { + this.capturedAuthenticator = authenticator; + return this; + } + + @Nullable + public ExecutionAuthenticator getCapturedAuthenticator() { + return capturedAuthenticator; + } + + /** + * A relation pinned to this projection plans and scans the retained frozen table. That work + * must run on the execution context captured with the projection's table generation: after a + * credential/storage ALTER has installed a new catalog context, planning would otherwise run + * the old generation's frozen operations and FileIO under the new authenticator, storage + * state and pre-authenticated executor. Fail before planning instead; the retried statement + * binds a coherent current generation. + */ + public void ensurePlannableUnder(@Nullable ExecutionAuthenticator currentAuthenticator, String tableName) { + ExecutionAuthenticator captured = capturedAuthenticator; + if (captured != null && currentAuthenticator != null && captured != currentAuthenticator) { + throw new IllegalStateException("Catalog execution context changed since this statement pinned" + + " its snapshot of " + tableName + ", please retry the query."); + } + } + public Optional>> getNameMapping() { return nameMapping; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index cd5c569fc39e8e..d4ce6d2632b2b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -792,6 +792,14 @@ private Table useFrozenTableGeneration(Table currentTable) { if (snapshot.filter(IcebergMvccSnapshot.class::isInstance).isPresent()) { IcebergSnapshotCacheValue cacheValue = ((IcebergMvccSnapshot) snapshot.get()).getSnapshotCacheValue(); + // Planning the frozen generation (regular relations and snapshot-selectable system + // tables alike) uses the catalog's current authenticator, storage state and + // pre-authenticated executor. Those are only coherent with the retained frozen + // operations/FileIO while the catalog still serves the generation this statement + // pinned; after a credential/storage ALTER the statement must fail and be retried. + cacheValue.ensurePlannableUnder( + source.getCatalog().getExecutionAuthenticator(), + source.getTargetTable().getName()); Optional
    frozenTable = cacheValue.getIcebergTable(); if (frozenTable.isPresent()) { Table frozenBaseTable = frozenTable.get(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java index 364309552a4cb3..75d187fba07118 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonCacheSizeEstimator.java @@ -77,6 +77,8 @@ final class PaimonCacheSizeEstimator { // rotation replaces the retained token rather than growing it, and typical token/config // payloads stay far below this bound. private static final long FILE_IO_WEIGHT = 16L * 1024L; + /** See the Iceberg estimator: flat allowance for the retained execution context. */ + private static final long AUTHENTICATION_CONTEXT_WEIGHT = 16L * 1024L; private static final long PARTITION_WEIGHT = 320L; private static final long PARTITION_ITEM_WEIGHT = 768L; @@ -97,6 +99,9 @@ static MetaCacheSizeEstimate estimateTableEntry(NameMapping key, PaimonTableCach KEY_BASE_WEIGHT, MetaCacheWeightUtils.estimatedNameMappingBytes(key)); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, TABLE_VALUE_BASE_WEIGHT); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedTablePayloadBytes()); + if (value.getAuthenticator() != null) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, AUTHENTICATION_CONTEXT_WEIGHT); + } return MetaCacheSizeEstimate.complete( MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(value.getPaimonTable()))); } @@ -117,6 +122,9 @@ static MetaCacheSizeEstimate estimateSnapshotEntry( bytes = MetaCacheWeightUtils.saturatedAdd( bytes, value.getPartitionInfo().getRetainedPayloadBytes()); bytes = MetaCacheWeightUtils.saturatedAdd(bytes, value.getRetainedTablePayloadBytes()); + if (value.getCapturedAuthenticator() != null) { + bytes = MetaCacheWeightUtils.saturatedAdd(bytes, AUTHENTICATION_CONTEXT_WEIGHT); + } return MetaCacheSizeEstimate.complete( MetaCacheWeightUtils.saturatedAdd(bytes, estimateTable(table))); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 063e6f9bb0fd6c..5d239c80b98025 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -466,6 +466,10 @@ protected CatalogIf getCatalog(long catalogId) { // execution context, exactly as the load used to trigger implicitly. IcebergTableCacheValue published = tables.get(mapping); Assert.assertSame(authenticator, published.getAuthenticator()); + // Projections built from the published generation carry its execution context for + // later planning-time validation. + Assert.assertSame(authenticator, + cache.getSnapshotCache(dorisTable).getCapturedAuthenticator()); initialized.set(false); Assert.assertSame(table, cache.getWritableIcebergTable(dorisTable)); @@ -487,6 +491,53 @@ protected CatalogIf getCatalog(long catalogId) { } } + @Test + public void testRetainedExecutionContextAllowanceAndPlanningFence() { + ExecutionAuthenticator captured = new ExecutionAuthenticator() { + @Override + public T execute(java.util.concurrent.Callable task) throws Exception { + return task.call(); + } + }; + ExecutionAuthenticator replaced = new ExecutionAuthenticator() { + @Override + public T execute(java.util.concurrent.Callable task) throws Exception { + return task.call(); + } + }; + NameMapping mapping = NameMapping.createForTest(1L, "db", "tbl"); + Table tableA = tableWithMetadataLocation("/metadata/auth-allowance-v1.json"); + Table tableB = tableWithMetadataLocation("/metadata/auth-allowance-v1.json"); + IcebergTableCacheValue unbound = new IcebergTableCacheValue(tableA); + IcebergTableCacheValue bound = new IcebergTableCacheValue(tableB); + bound.bindAuthenticator(captured); + Assert.assertEquals("bound values carry the retained-context allowance", 16L * 1024L, + IcebergCacheSizeEstimator.estimateTableEntry(mapping, bound).getBytes() + - IcebergCacheSizeEstimator.estimateTableEntry(mapping, unbound).getBytes()); + + IcebergSnapshotEntryKey key = IcebergSnapshotEntryKey.tryCreate(mapping, tableA).get(); + IcebergSnapshotCacheValue plain = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), Optional.empty(), tableA); + IcebergSnapshotCacheValue boundSnapshot = new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), Optional.empty(), tableA) + .bindCapturedAuthenticator(captured); + Assert.assertEquals(16L * 1024L, + IcebergCacheSizeEstimator.estimateSnapshotEntry(key, boundSnapshot).getBytes() + - IcebergCacheSizeEstimator.estimateSnapshotEntry(key, plain).getBytes()); + + // The pinned generation is plannable only under its captured execution context. + boundSnapshot.ensurePlannableUnder(captured, "tbl"); + boundSnapshot.ensurePlannableUnder(null, "tbl"); + plain.ensurePlannableUnder(replaced, "tbl"); + try { + boundSnapshot.ensurePlannableUnder(replaced, "tbl"); + Assert.fail("planning a pinned generation under a replaced catalog context must fail"); + } catch (IllegalStateException e) { + Assert.assertTrue(String.valueOf(e.getMessage()), + e.getMessage().contains("please retry")); + } + } + private static boolean exceptionChainContains(Throwable throwable, String fragment) { for (Throwable current = throwable; current != null; current = current.getCause()) { if (current.getMessage() != null && current.getMessage().contains(fragment)) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 715272c021961d..a6e72b93ab53a1 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -1062,6 +1062,33 @@ public T execute(Callable task) throws Exception { } } + @Test + public void testRetainedExecutionContextAllowanceIsCharged() { + ExecutionAuthenticator authenticator = new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + return task.call(); + } + }; + FileStoreTable table = newTableWithPayloadType("auth_allowance", new IntType()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "db", "tbl"); + PaimonTableCacheValue unbound = new PaimonTableCacheValue(table); + PaimonTableCacheValue bound = new PaimonTableCacheValue(table, authenticator); + Assert.assertEquals("bound values carry the retained-context allowance", 16L * 1024L, + PaimonCacheSizeEstimator.estimateTableEntry(mapping, bound).getBytes() + - PaimonCacheSizeEstimator.estimateTableEntry(mapping, unbound).getBytes()); + + PaimonSnapshotEntryKey key = new PaimonSnapshotEntryKey(mapping, 1L, table.schema().id(), 1L); + PaimonSnapshotCacheValue plain = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, table.schema().id(), table)); + PaimonSnapshotCacheValue bound2 = new PaimonSnapshotCacheValue( + PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, table.schema().id(), table)) + .bindCapturedAuthenticator(authenticator); + Assert.assertEquals(16L * 1024L, + PaimonCacheSizeEstimator.estimateSnapshotEntry(key, bound2).getBytes() + - PaimonCacheSizeEstimator.estimateSnapshotEntry(key, plain).getBytes()); + } + private static boolean exceptionChainContains(Throwable throwable, String fragment) { for (Throwable current = throwable; current != null; current = current.getCause()) { if (current.getMessage() != null && current.getMessage().contains(fragment)) { From 1d223139d9a1530e0604d6ccb8520e4c9f1843ce Mon Sep 17 00:00:00 2001 From: guoqiang Date: Sun, 23 Aug 2026 02:28:18 +0800 Subject: [PATCH 40/45] [fix](iceberg) Carry the generation context through explicit VERSION/TIME and branch/tag snapshots The explicit-snapshot path resolved a query-scoped table of the current generation but built its IcebergSnapshotCacheValue without the captured execution context, so the planning fence saw no binding and permitted a relation bound before a credential/storage ALTER to plan the old generation's frozen operations and FileIO under the replaced context. The generation is now resolved once - handle and captured authenticator together - and newExplicitSnapshotValue binds that context onto the constructed value, putting VERSION/TIME and branch/tag relations on the same planning fence as latest projections. Regression covers binding and the fence rejecting a replaced context for an explicit ref. --- .../iceberg/IcebergExternalMetaCache.java | 11 ++++++ .../datasource/iceberg/IcebergUtils.java | 24 ++++++++++--- .../iceberg/IcebergExternalMetaCacheTest.java | 34 +++++++++++++++++++ 3 files changed, 64 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index a1409715ba0f45..0ccdac5d5f7a27 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -179,6 +179,17 @@ Table getQueryScopedIcebergTable(ExternalTable dorisTable) { return createQueryTable(nameMapping, tableValue); } + /** Resolve the current table generation, exposing the handle and its captured context together. */ + IcebergTableCacheValue getTableCacheValue(ExternalTable dorisTable) { + NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); + return tableEntry.get(nameMapping.getCtlId()).get(nameMapping); + } + + /** Query-scoped view of an already-resolved generation; see {@link #getTableCacheValue}. */ + Table createQueryScopedTable(ExternalTable dorisTable, IcebergTableCacheValue tableValue) { + return createQueryTable(dorisTable.getOrBuildNameMapping(), tableValue); + } + private Table createQueryTable( NameMapping nameMapping, IcebergTableCacheValue tableValue) { boolean isolateForQueries = tableValue.isQueryIsolationPrepared() diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java index e88b9d1b09c0ff..4e9369be5e9e50 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java @@ -2044,22 +2044,36 @@ public static IcebergSnapshotCacheValue getSnapshotCacheValue( Optional scanParams) { if (tableSnapshot.isPresent() || IcebergUtils.isIcebergBranchOrTag(scanParams)) { // If a snapshot is specified, use the specified snapshot and the corresponding schema (not latest). + // Resolve the generation once so the retained query-scoped table and the execution + // context it is planned under always come from the same catalog generation. IcebergExternalMetaCache metaCache = icebergExternalMetaCache(dorisTable); - Table icebergTable = metaCache.getQueryScopedIcebergTable(dorisTable); + IcebergTableCacheValue tableValue = metaCache.getTableCacheValue(dorisTable); + Table icebergTable = metaCache.createQueryScopedTable(dorisTable, tableValue); IcebergTableQueryInfo info; try { info = getQuerySpecSnapshot(icebergTable, tableSnapshot, scanParams); } catch (UserException e) { throw new RuntimeException(e); } - return new IcebergSnapshotCacheValue( - IcebergPartitionInfo.empty(), - new IcebergSnapshot(info.getSnapshotId(), info.getSchemaId()), - getNameMapping(icebergTable), icebergTable); + return newExplicitSnapshotValue(info, icebergTable, tableValue); } return getLatestSnapshotCacheValue(dorisTable); } + /** + * An explicit VERSION/TIME or branch/tag relation retains its query-scoped table exactly like + * a latest projection retains the frozen generation, so the value must carry the generation's + * captured execution context for the planning-time fence in IcebergScanNode. + */ + static IcebergSnapshotCacheValue newExplicitSnapshotValue( + IcebergTableQueryInfo info, Table queryScopedTable, IcebergTableCacheValue generation) { + return new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), + new IcebergSnapshot(info.getSnapshotId(), info.getSchemaId()), + getNameMapping(queryScopedTable), queryScopedTable) + .bindCapturedAuthenticator(generation.getAuthenticator()); + } + public static List getIcebergSchema(ExternalTable dorisTable) { return getIcebergSchema(dorisTable, MvccUtil.getSnapshotFromContext(dorisTable)); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 5d239c80b98025..2cdcc31a9749ac 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -24,6 +24,7 @@ import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.iceberg.cache.ManifestCacheValue; +import org.apache.doris.datasource.iceberg.source.IcebergTableQueryInfo; import org.apache.doris.datasource.metacache.EstimatorCalibrationAssertions; import org.apache.doris.datasource.metacache.MetaCacheEntry; import org.apache.doris.datasource.metacache.MetaCacheEntryStats; @@ -538,6 +539,39 @@ public T execute(java.util.concurrent.Callable task) throws Exception { } } + @Test + public void testExplicitSnapshotValueCarriesItsGenerationContext() { + ExecutionAuthenticator captured = new ExecutionAuthenticator() { + @Override + public T execute(java.util.concurrent.Callable task) throws Exception { + return task.call(); + } + }; + ExecutionAuthenticator replaced = new ExecutionAuthenticator() { + @Override + public T execute(java.util.concurrent.Callable task) throws Exception { + return task.call(); + } + }; + Table table = tableWithMetadataLocation("/metadata/explicit-snapshot-v1.json"); + IcebergTableCacheValue generation = new IcebergTableCacheValue(table); + generation.bindAuthenticator(captured); + + // VERSION/TIME and branch/tag relations retain the query-scoped table of the generation + // they were bound on; the constructed value must carry that generation's context so the + // planning fence can reject a catalog that was reset in between. + IcebergSnapshotCacheValue value = IcebergUtils.newExplicitSnapshotValue( + new IcebergTableQueryInfo(1L, "main", 0), table, generation); + Assert.assertSame(captured, value.getCapturedAuthenticator()); + value.ensurePlannableUnder(captured, "tbl"); + try { + value.ensurePlannableUnder(replaced, "tbl"); + Assert.fail("an explicit snapshot bound before a catalog reset must not plan under the new context"); + } catch (IllegalStateException e) { + Assert.assertTrue(String.valueOf(e.getMessage()), e.getMessage().contains("please retry")); + } + } + private static boolean exceptionChainContains(Throwable throwable, String fragment) { for (Throwable current = throwable; current != null; current = current.getCause()) { if (current.getMessage() != null && current.getMessage().contains(fragment)) { From 105739582c80bd3a0d7d76b6dea8bf6eed254a92 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Sun, 23 Aug 2026 03:22:08 +0800 Subject: [PATCH 41/45] [fix](fe) Converge catalog transitions: retire old-context projections and fail dropped-catalog lookups terminally An auth-only catalog ALTER hands out the same metadata and operationally equivalent FileIO under a new execution context. The captured authenticator was not part of the Iceberg operational-generation equality, so replacement kept the old projection, hit revalidation kept serving it, the planning fence kept rejecting it, and every retried statement failed until expiry. The captured context is now part of isSameOperationalGeneration and of hit-side revalidation, so the refresh retires old-context projections and the rebuilt one is plannable again; the planning fence itself now applies only when a frozen handle is actually planned (count-mode values plan the live table). After DROP CATALOG the group preparer could never restore the group, but the bounded contended-handoff retry could not distinguish that terminal state from lock contention and slept through the full two-second window before failing. Permanent removal now records a tombstone (catalog ids are never reused; defensively cleared on re-init) that fails lookups immediately with a dropped-catalog message, while rename and contended handoffs keep the bounded retry. Regressions: same-metadata refresh under a new context retires the old projection (with operational-equality assertions), and dropped-catalog lookups fail in far under the retry window while a transiently absent group is still re-prepared. --- .../hive/HiveExternalMetaCache.java | 1 + .../iceberg/IcebergExternalMetaCache.java | 12 +++- .../iceberg/IcebergTableCacheValue.java | 7 ++- .../iceberg/source/IcebergScanNode.java | 18 +++--- .../metacache/AbstractExternalMetaCache.java | 22 ++++++- .../paimon/PaimonExternalMetaCache.java | 1 + .../iceberg/IcebergExternalMetaCacheTest.java | 62 +++++++++++++++++++ .../paimon/PaimonExternalMetaCacheTest.java | 49 +++++++++++++++ 8 files changed, 159 insertions(+), 13 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java index 981b1f640138e3..78ad798912bf5a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java @@ -208,6 +208,7 @@ public void invalidateCatalog(long catalogId) { @Override public void onCatalogPermanentlyRemoved(long catalogId) { + super.onCatalogPermanentlyRemoved(catalogId); // The id is never reused; without this, create/use/drop churn would accumulate counter // map nodes for the FE lifetime. In-flight scans cannot recreate the records because // the counter helpers only allocate while the catalog's entry group exists. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java index 0ccdac5d5f7a27..b58e659abfce76 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCache.java @@ -521,8 +521,16 @@ private static boolean sharesOperationalResources( return true; } Optional
    retainedTable = projection.getRetainedIcebergTable(); - // Count-mode projections do not retain a table handle; nothing to rebind. - return !retainedTable.isPresent() || currentValue.sharesOperationalResources(retainedTable.get()); + // Count-mode projections do not retain a table handle; nothing to rebind (and nothing is + // planned through a frozen generation, so a stale captured context is inert). + if (!retainedTable.isPresent()) { + return true; + } + // The captured execution context must match as well: after an auth-only ALTER the frozen + // handle is operationally equivalent but unplannable under the new context, and serving + // it would make every retried statement hit the same rejected projection until expiry. + return currentValue.sharesOperationalResources(retainedTable.get()) + && currentValue.getAuthenticator() == projection.getCapturedAuthenticator(); } private IcebergSnapshotCacheValue loadSnapshotProjection( diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java index dbb5e4c71148d8..8d0dd33641260e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTableCacheValue.java @@ -155,7 +155,12 @@ boolean isSamePhysicalGeneration(IcebergTableCacheValue other) { * credentials; projections frozen on the previous handle must not outlive that rotation. */ boolean isSameOperationalGeneration(IcebergTableCacheValue other) { - return isSamePhysicalGeneration(other) && sharesOperationalResources(other.icebergTable); + // The captured execution context is part of the operational generation: an auth-only + // ALTER hands out the same metadata and equivalent FileIO under a new authenticator, + // and projections frozen on the old context would fail the planning fence forever + // instead of being rebuilt. + return isSamePhysicalGeneration(other) && sharesOperationalResources(other.icebergTable) + && authenticator == other.authenticator; } boolean sharesOperationalResources(Table table) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index d4ce6d2632b2b1..d851c695e4f8bd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -792,16 +792,18 @@ private Table useFrozenTableGeneration(Table currentTable) { if (snapshot.filter(IcebergMvccSnapshot.class::isInstance).isPresent()) { IcebergSnapshotCacheValue cacheValue = ((IcebergMvccSnapshot) snapshot.get()).getSnapshotCacheValue(); - // Planning the frozen generation (regular relations and snapshot-selectable system - // tables alike) uses the catalog's current authenticator, storage state and - // pre-authenticated executor. Those are only coherent with the retained frozen - // operations/FileIO while the catalog still serves the generation this statement - // pinned; after a credential/storage ALTER the statement must fail and be retried. - cacheValue.ensurePlannableUnder( - source.getCatalog().getExecutionAuthenticator(), - source.getTargetTable().getName()); Optional
    frozenTable = cacheValue.getIcebergTable(); if (frozenTable.isPresent()) { + // Planning the frozen generation (regular relations and snapshot-selectable + // system tables alike) uses the catalog's current authenticator, storage state + // and pre-authenticated executor. Those are only coherent with the retained + // frozen operations/FileIO while the catalog still serves the generation this + // statement pinned; after a credential/storage ALTER the statement must fail and + // be retried. Count-mode values retain no frozen handle and plan the live table, + // so they are not fenced. + cacheValue.ensurePlannableUnder( + source.getCatalog().getExecutionAuthenticator(), + source.getTargetTable().getName()); Table frozenBaseTable = frozenTable.get(); if (isSystemTable && source.getTargetTable() instanceof IcebergSysExternalTable) { IcebergSysExternalTable systemTable = (IcebergSysExternalTable) source.getTargetTable(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java index 2f012d6e1b2c87..886df002141156 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java @@ -36,6 +36,8 @@ import java.util.Map; import java.util.Objects; import java.util.OptionalLong; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.function.Consumer; import java.util.function.Function; @@ -78,6 +80,10 @@ protected AbstractExternalMetaCache(String engine, ExecutorService refreshExecut } private volatile LongConsumer catalogPreparer; + // Catalog ids are never reused, so a permanently dropped id is a terminal state for this + // engine: lookups must fail immediately instead of consuming the contended-handoff retry + // window. A rename produces only a transient map absence and never lands here. + private final Set permanentlyRemovedCatalogs = ConcurrentHashMap.newKeySet(); protected AbstractExternalMetaCache(String engine, ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { @@ -165,6 +171,7 @@ public void initCatalog(long catalogId, Map catalogProperties) { if (catalogEntries.containsKey(catalogId)) { return; } + permanentlyRemovedCatalogs.remove(catalogId); Map safeCatalogProperties = sanitizeCatalogPropertiesForRuntime( catalogProperties, warning -> LOG.warn("{} (engine={}, catalog={})", warning, engine, catalogId)); @@ -345,10 +352,11 @@ private CatalogEntryGroup requireCatalogEntryGroup(long catalogId) { // sleep-and-retry: the ALTER finishes within the window, or the lookup fails as // before without any deadlock. long deadlineNanos = System.nanoTime() + PREPARE_RETRY_WINDOW_NANOS; - while (true) { + while (!permanentlyRemovedCatalogs.contains(catalogId)) { catalogPreparer.accept(catalogId); group = catalogEntries.get(catalogId); - if (group != null || System.nanoTime() >= deadlineNanos) { + if (group != null || System.nanoTime() >= deadlineNanos + || permanentlyRemovedCatalogs.contains(catalogId)) { break; } try { @@ -360,6 +368,11 @@ private CatalogEntryGroup requireCatalogEntryGroup(long catalogId) { } } if (group == null) { + if (permanentlyRemovedCatalogs.contains(catalogId)) { + throw new IllegalStateException(String.format( + "Catalog %d was dropped; engine '%s' serves no metadata for it.", + catalogId, engine)); + } throw new IllegalStateException(String.format( "Catalog %d is not initialized for engine '%s'.", catalogId, engine)); @@ -372,6 +385,11 @@ public void bindCatalogPreparer(LongConsumer catalogPreparer) { this.catalogPreparer = catalogPreparer; } + @Override + public void onCatalogPermanentlyRemoved(long catalogId) { + permanentlyRemovedCatalogs.add(catalogId); + } + protected CatalogIf getCatalog(long catalogId) { if (Env.getCurrentEnv() == null || Env.getCurrentEnv().getCatalogMgr() == null) { return null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index 04747a2a46ef8a..be11a7723210c5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -520,6 +520,7 @@ public void invalidateCatalog(long catalogId) { @Override public void onCatalogPermanentlyRemoved(long catalogId) { + super.onCatalogPermanentlyRemoved(catalogId); // A lookup racing the drop may re-insert a fence owner after invalidateCatalog cleaned // the map; this hook runs even when the entry group is already retired and the id is // never reused, so the owners cannot leak for the FE lifetime. diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java index 2cdcc31a9749ac..7d559a23b075ae 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/IcebergExternalMetaCacheTest.java @@ -572,6 +572,68 @@ public T execute(java.util.concurrent.Callable task) throws Exception { } } + @Test + public void testAuthOnlyAlterRetiresProjectionsOfTheOldContext() { + ExecutionAuthenticator oldContext = new ExecutionAuthenticator() { + @Override + public T execute(java.util.concurrent.Callable task) throws Exception { + return task.call(); + } + }; + ExecutionAuthenticator newContext = new ExecutionAuthenticator() { + @Override + public T execute(java.util.concurrent.Callable task) throws Exception { + return task.call(); + } + }; + ExecutorService executor = Executors.newSingleThreadExecutor(); + IcebergExternalMetaCache cache = new IcebergExternalMetaCache(executor); + try { + long catalogId = 1L; + cache.initCatalog(catalogId, Collections.emptyMap()); + NameMapping mapping = NameMapping.createForTest(catalogId, "db", "tbl"); + TableMetadata metadata = metadataWithLocation("/metadata/auth-only-alter-v1.json"); + // Same metadata file and operationally equivalent FileIO resources; only the + // captured execution context differs, as after an auth-only catalog ALTER. + IcebergTableCacheValue first = new IcebergTableCacheValue( + tableWithMetadata(metadata, new PropertiesFileIO("token", "same"))); + first.bindAuthenticator(oldContext); + IcebergTableCacheValue refreshed = new IcebergTableCacheValue( + tableWithMetadata(metadata, new PropertiesFileIO("token", "same"))); + refreshed.bindAuthenticator(newContext); + Assert.assertTrue(first.isSamePhysicalGeneration(refreshed)); + Assert.assertFalse("a replaced execution context is a new operational generation", + first.isSameOperationalGeneration(refreshed)); + IcebergTableCacheValue sameContext = new IcebergTableCacheValue( + tableWithMetadata(metadata, new PropertiesFileIO("token", "same"))); + sameContext.bindAuthenticator(oldContext); + Assert.assertTrue(first.isSameOperationalGeneration(sameContext)); + + MetaCacheEntry tables = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_TABLE, + NameMapping.class, IcebergTableCacheValue.class); + MetaCacheEntry snapshots = cache.entry( + catalogId, IcebergExternalMetaCache.ENTRY_SNAPSHOT, + IcebergSnapshotEntryKey.class, IcebergSnapshotCacheValue.class); + tables.put(mapping, first); + IcebergSnapshotEntryKey key = IcebergSnapshotEntryKey.tryCreate( + mapping, first.getRetainedIcebergTable()).get(); + snapshots.put(key, new IcebergSnapshotCacheValue( + IcebergPartitionInfo.empty(), new IcebergSnapshot(-1L, 0L), Optional.empty(), + first.getRetainedIcebergTable()) + .bindCapturedAuthenticator(oldContext)); + + // The same-metadata refresh under the new context must retire the old projection so + // the next lookup rebuilds one that is plannable again. + tables.put(mapping, refreshed); + Assert.assertNull("projections of the replaced execution context must be retired", + snapshots.peekIfPresent(key)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + private static boolean exceptionChainContains(Throwable throwable, String fragment) { for (Throwable current = throwable; current != null; current = current.getCause()) { if (current.getMessage() != null && current.getMessage().contains(fragment)) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index a6e72b93ab53a1..dbb6a97522b431 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -1089,6 +1089,55 @@ PaimonPartitionInfo.EMPTY, new PaimonSnapshot(1L, table.schema().id(), table)) - PaimonCacheSizeEstimator.estimateSnapshotEntry(key, plain).getBytes()); } + @Test + public void testPermanentlyDroppedCatalogFailsLookupsImmediately() { + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try { + cache.initCatalog(1L, Collections.emptyMap()); + AtomicInteger dropPreparerCalls = new AtomicInteger(); + cache.bindCatalogPreparer(id -> dropPreparerCalls.incrementAndGet()); + // DROP CATALOG: the group is retired and the removal is permanent - no preparer can + // ever restore it, so the lookup must fail terminally instead of consuming the + // contended-handoff retry window. + cache.invalidateCatalog(1L); + cache.onCatalogPermanentlyRemoved(1L); + long startNanos = System.nanoTime(); + try { + cache.entry(1L, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class); + Assert.fail("a permanently dropped catalog must fail lookups terminally"); + } catch (IllegalStateException e) { + Assert.assertTrue(String.valueOf(e.getMessage()), + String.valueOf(e.getMessage()).contains("was dropped")); + } + Assert.assertTrue("a dropped catalog must not consume the retry window", + System.nanoTime() - startNanos < 1_000_000_000L); + + // A rename/contended handoff produces only a transient absence: the bounded retry + // must still absorb it and observe the re-prepared group. + cache.initCatalog(2L, Collections.emptyMap()); + cache.invalidateCatalog(2L); + AtomicInteger renamePreparerCalls = new AtomicInteger(); + cache.bindCatalogPreparer(id -> { + if (renamePreparerCalls.incrementAndGet() >= 2) { + cache.initCatalog(2L, Collections.emptyMap()); + } + }); + Assert.assertNotNull(cache.entry(2L, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class)); + Assert.assertTrue(renamePreparerCalls.get() >= 2); + + // Re-creating a catalog id clears the tombstone (defensive: ids are never reused). + cache.initCatalog(1L, Collections.emptyMap()); + Assert.assertNotNull(cache.entry(1L, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + private static boolean exceptionChainContains(Throwable throwable, String fragment) { for (Throwable current = throwable; current != null; current = current.getCause()) { if (current.getMessage() != null && current.getMessage().contains(fragment)) { From bfca15ad5d43930b06afe5f7163ccb3880657ae5 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Sun, 23 Aug 2026 04:18:25 +0800 Subject: [PATCH 42/45] [fix](fe) Converge catalog transitions across ALTER, DROP, cleanup and fence capture Committed catalog property ALTERs reset the execution context and close SDK resources without retiring engine cache groups, leaving cached base generations that every planning fence rejects until managed refresh. CatalogMgr now notifies ExternalMetaCacheMgr.onCatalogOperationalContext- Changed after the commit (fresh and replayed), which retires the routed engines' cached entries - groups and policies stay - so the next statement loads a generation bound to the new context. Failed validations retire nothing. The dropped-catalog terminal state no longer uses per-engine dropped-id membership (which grew per engine for the FE lifetime and was published only after groups were detached and closed): the lookup now probes the catalog manager, which removes the catalog before any engine detachment, so a lookup during a blocked close fails immediately while rename and contended handoffs - where the catalog stays registered - keep the bounded retry. No per-engine state is retained. The shared removal-cleanup worker now releases dead reservations before running dependency-retirement listeners, and the removal callback queues the reservation before the notification, so one expensive listener can no longer keep already-freed quota charged while admissions are rejected or peers reclaimed. Paimon fence capture and observation-number assignment are serialized per owner: a capture pausing between reading the fence and taking its number could otherwise outnumber a later capture that already published a newer fence and replace it with the older one; lock entries retire with their owners. Regressions: post-ALTER retirement wiring (and none on failed validation), immediate dropped-catalog failure with preserved transient-absence retry, quota release while a listener is blocked, and serialized capture with the newest-read fence owning the memoized latest. --- .../apache/doris/datasource/CatalogMgr.java | 9 ++ .../datasource/ExternalMetaCacheMgr.java | 13 ++ .../hive/HiveExternalMetaCache.java | 1 - .../metacache/AbstractExternalMetaCache.java | 43 ++++-- .../datasource/metacache/MetaCacheEntry.java | 25 +++- .../paimon/PaimonExternalMetaCache.java | 61 +++++--- .../doris/datasource/CatalogMgrTest.java | 50 +++++++ .../metacache/MetaCacheEntryTest.java | 37 +++++ .../paimon/PaimonExternalMetaCacheTest.java | 135 +++++++++++++++++- 9 files changed, 327 insertions(+), 47 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java index c0a92fa3c96331..776ca6392dea4b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogMgr.java @@ -750,6 +750,15 @@ private void alterExternalCatalogPropsFenced(ExternalCatalog externalCatalog, Ca Env.getCurrentEnv().getRefreshManager().addToRefreshMap(catalogId, sec); } externalCatalog.modifyCatalogProps(newProps); + // The commit reset the catalog's execution context and closed its SDK resources. Cached + // base generations and projections are bound to the replaced context; retire them now so + // the next statement loads a generation the planning fences accept, instead of retrying + // against an unplannable cached generation until managed refresh. + Env currentEnv = Env.getCurrentEnv(); + ExternalMetaCacheMgr cacheMgr = currentEnv == null ? null : currentEnv.getExtMetaCacheMgr(); + if (cacheMgr != null) { + cacheMgr.onCatalogOperationalContextChanged(externalCatalog.getId()); + } } public void unregisterExternalTable(String dbName, String tableName, String catalogName, boolean ignoreIfExists) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java index 7b4f208fe9cdb7..bb62d178f968a8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalMetaCacheMgr.java @@ -423,6 +423,19 @@ public void removeCatalog(long catalogId) { * same-id policy rebuilds; the hook reaches every engine even when its entry group is * already retired. */ + /** + * A committed catalog property ALTER (fresh or replayed) resets the catalog's execution + * context and closes its SDK resources without retiring the engine cache groups. Cached base + * generations and their projections are bound to the replaced context: served as-is they + * would fail the planning fences until expiry, so retire the entries (groups and policies + * stay) and let the next statement load a generation bound to the new context. + */ + public void onCatalogOperationalContextChanged(long catalogId) { + routeCatalogEngines(catalogId, cache -> safeInvalidate( + cache, catalogId, "onCatalogOperationalContextChanged", + () -> cache.invalidateCatalogEntries(catalogId))); + } + public void removeCatalogPermanently(long catalogId) { Lock lifecycleLock = catalogLifecycleLocks.get(catalogId); lifecycleLock.lock(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java index 78ad798912bf5a..981b1f640138e3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveExternalMetaCache.java @@ -208,7 +208,6 @@ public void invalidateCatalog(long catalogId) { @Override public void onCatalogPermanentlyRemoved(long catalogId) { - super.onCatalogPermanentlyRemoved(catalogId); // The id is never reused; without this, create/use/drop churn would accumulate counter // map nodes for the FE lifetime. In-flight scans cannot recreate the records because // the counter helpers only allocate while the catalog's entry group exists. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java index 886df002141156..79ee1ab78bf5e5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCache.java @@ -36,8 +36,6 @@ import java.util.Map; import java.util.Objects; import java.util.OptionalLong; -import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import java.util.concurrent.ExecutorService; import java.util.function.Consumer; import java.util.function.Function; @@ -80,10 +78,8 @@ protected AbstractExternalMetaCache(String engine, ExecutorService refreshExecut } private volatile LongConsumer catalogPreparer; - // Catalog ids are never reused, so a permanently dropped id is a terminal state for this - // engine: lookups must fail immediately instead of consuming the contended-handoff retry - // window. A rename produces only a transient map absence and never lands here. - private final Set permanentlyRemovedCatalogs = ConcurrentHashMap.newKeySet(); + // Test hook; production probes the catalog manager. See catalogPermanentlyDropped. + private volatile java.util.function.LongPredicate droppedCatalogProbeForTest; protected AbstractExternalMetaCache(String engine, ExecutorService refreshExecutor, ExternalMetaCacheBudgetManager budgetManager) { @@ -171,7 +167,6 @@ public void initCatalog(long catalogId, Map catalogProperties) { if (catalogEntries.containsKey(catalogId)) { return; } - permanentlyRemovedCatalogs.remove(catalogId); Map safeCatalogProperties = sanitizeCatalogPropertiesForRuntime( catalogProperties, warning -> LOG.warn("{} (engine={}, catalog={})", warning, engine, catalogId)); @@ -352,11 +347,11 @@ private CatalogEntryGroup requireCatalogEntryGroup(long catalogId) { // sleep-and-retry: the ALTER finishes within the window, or the lookup fails as // before without any deadlock. long deadlineNanos = System.nanoTime() + PREPARE_RETRY_WINDOW_NANOS; - while (!permanentlyRemovedCatalogs.contains(catalogId)) { + while (!catalogPermanentlyDropped(catalogId)) { catalogPreparer.accept(catalogId); group = catalogEntries.get(catalogId); if (group != null || System.nanoTime() >= deadlineNanos - || permanentlyRemovedCatalogs.contains(catalogId)) { + || catalogPermanentlyDropped(catalogId)) { break; } try { @@ -368,7 +363,7 @@ private CatalogEntryGroup requireCatalogEntryGroup(long catalogId) { } } if (group == null) { - if (permanentlyRemovedCatalogs.contains(catalogId)) { + if (catalogPermanentlyDropped(catalogId)) { throw new IllegalStateException(String.format( "Catalog %d was dropped; engine '%s' serves no metadata for it.", catalogId, engine)); @@ -380,14 +375,32 @@ private CatalogEntryGroup requireCatalogEntryGroup(long catalogId) { return group; } - @Override - public void bindCatalogPreparer(LongConsumer catalogPreparer) { - this.catalogPreparer = catalogPreparer; + /** + * DROP CATALOG is terminal: ids are never reused, and the catalog manager removes the + * catalog before any engine group is detached or closed, so a lookup that finds neither a + * group nor a live catalog can fail immediately instead of consuming the contended-handoff + * retry window. A rename keeps the catalog registered under the same id, so its transient + * group absence still gets the bounded retry, and no per-engine dropped-id state is retained. + */ + private boolean catalogPermanentlyDropped(long catalogId) { + java.util.function.LongPredicate probe = droppedCatalogProbeForTest; + if (probe != null) { + return probe.test(catalogId); + } + org.apache.doris.catalog.Env env = org.apache.doris.catalog.Env.getCurrentEnv(); + org.apache.doris.datasource.CatalogMgr catalogMgr = env == null ? null : env.getCatalogMgr(); + // An absent manager (isolated construction/boot) proves nothing; keep the bounded retry. + return catalogMgr != null && catalogMgr.getCatalog(catalogId) == null; + } + + /** Test hook: overrides the live-catalog probe used to detect a permanent DROP. */ + public void bindDroppedCatalogProbeForTest(java.util.function.LongPredicate probe) { + this.droppedCatalogProbeForTest = probe; } @Override - public void onCatalogPermanentlyRemoved(long catalogId) { - permanentlyRemovedCatalogs.add(catalogId); + public void bindCatalogPreparer(LongConsumer catalogPreparer) { + this.catalogPreparer = catalogPreparer; } protected CatalogIf getCatalog(long catalogId) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java index daa49efcd892ae..64a52f1a307dc2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/MetaCacheEntry.java @@ -843,10 +843,9 @@ private void onRemoval(@Nullable K key, @Nullable V value, RemovalCause cause) { if (cause == RemovalCause.REPLACED) { return; } - if (removalListener != null) { - pendingRemovalNotifications.add(new RemovedToken<>(key, removalToken(value))); - scheduleRemovalCleanup(); - } + // The dead reservation is queued before the dependency notification so the shared + // cleanup worker can always release quota before entering potentially expensive listener + // work; see drainRemovalCleanups. if (Thread.holdsLock(admissionLock)) { // Other removals have already removed the Caffeine mapping and can release their owner // inline. A stale callback cannot release a replacement while its mapping is visible. @@ -867,6 +866,7 @@ private void onRemoval(@Nullable K key, @Nullable V value, RemovalCause cause) { } } } + queueRemovalNotification(key, value); return; } beforeRemovalOwnerSnapshotForTest(key); @@ -874,6 +874,7 @@ private void onRemoval(@Nullable K key, @Nullable V value, RemovalCause cause) { if (ownerGeneration >= 0L) { beforeRemovalReleaseForTest(key); if (closed.get()) { + queueRemovalNotification(key, value); return; } if (cause.wasEvicted()) { @@ -883,10 +884,20 @@ private void onRemoval(@Nullable K key, @Nullable V value, RemovalCause cause) { if (closed.get()) { pendingRemovalGenerations.remove(key, ownerGeneration); pendingEvictionGenerations.remove(key, ownerGeneration); + queueRemovalNotification(key, value); return; } scheduleRemovalCleanup(); } + queueRemovalNotification(key, value); + } + + private void queueRemovalNotification(K key, @Nullable V value) { + if (removalListener == null) { + return; + } + pendingRemovalNotifications.add(new RemovedToken<>(key, removalToken(value))); + scheduleRemovalCleanup(); } private void scheduleRemovalCleanup() { @@ -905,7 +916,6 @@ private void scheduleRemovalCleanup() { private void drainRemovalCleanups() { try { - drainRemovalNotifications(); int processed = 0; for (Map.Entry cleanup : pendingRemovalGenerations.entrySet()) { if (processed++ >= REMOVAL_CLEANUP_BATCH_SIZE) { @@ -938,6 +948,11 @@ private void drainRemovalCleanups() { name, e); } } + // Dependency retirement can be expensive (a table listener may scan every child + // projection) and the cleanup executor is shared by every entry, so already-dead + // reservations release their global/catalog quota first; generations reported by + // these listeners are picked up by the requeued drain below. + drainRemovalNotifications(); } finally { removalCleanupScheduled.set(false); if (!closed.get() diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index be11a7723210c5..275d6c5e045dc2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -71,6 +71,12 @@ public class PaimonExternalMetaCache extends AbstractExternalMetaCache { private final EntryHandle schemaEntry; private final PaimonTableLoader tableLoader; private final PaimonLatestSnapshotProjectionLoader latestSnapshotProjectionLoader; + // Serializes fence capture plus observation-number assignment per owner: a capture that + // pauses between reading the fence and taking its number could otherwise outnumber a later + // capture that already published a newer fence, replacing it with the older one. Entries are + // retired together with their owners. + private final java.util.concurrent.ConcurrentHashMap fenceCaptureLocks = + new java.util.concurrent.ConcurrentHashMap<>(); // Most recently observed latest fence per (table, generation); see getSnapshotCache. private final AtomicLong fenceObservations = new AtomicLong(); private final ConcurrentHashMap latestObservedFences = @@ -116,37 +122,44 @@ public PaimonSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { NameMapping nameMapping = dorisTable.getOrBuildNameMapping(); MetaCacheEntry tables = tableEntry.get(nameMapping.getCtlId()); PaimonTableCacheValue tableValue = tables.get(nameMapping); - if (tables.isEffectivelyEnabled()) { - // Serve the memoized latest projection of this table generation while it is still - // published: the latest read is as stale-until-TTL/refresh as the cached table - // handle itself and costs no snapshot IO, preserving the pre-existing external - // metadata cache contract. The fence is re-observed only when no projection of this - // generation is reachable anymore (first read, expiry, weight eviction, explicit - // invalidation), which is also when rollback ordering below matters. - ObservedFence observed = latestObservedFences.get( - new LatestFenceOwner(nameMapping, tableValue.getGeneration())); - if (observed != null) { - PaimonSnapshotCacheValue memoized = - snapshotEntry.get(nameMapping.getCtlId()).peekIfPresent(observed.key); - if (memoized != null) { - return memoized; - } - } - } - PaimonSnapshot fence = loadLatestSnapshotFence(nameMapping, tableValue).getSnapshot(); if (!tables.isEffectivelyEnabled()) { // Projections are keyed by the synthetic generation of a published table handle. An // ineffective table entry publishes nothing, so nothing keyed by this load could ever // be looked up again: serve it directly instead of churning the snapshot entry. + PaimonSnapshot fence = loadLatestSnapshotFence(nameMapping, tableValue).getSnapshot(); return executeForGeneration(tableValue, nameMapping, () -> latestSnapshotProjectionLoader.loadAtFence( nameMapping, fence, tableValue.getGeneration())) .bindCapturedAuthenticator(tableValue.getAuthenticator()); } + LatestFenceOwner owner = new LatestFenceOwner(nameMapping, tableValue.getGeneration()); + // Serve the memoized latest projection of this table generation while it is still + // published: the latest read is as stale-until-TTL/refresh as the cached table + // handle itself and costs no snapshot IO, preserving the pre-existing external + // metadata cache contract. The fence is re-observed only when no projection of this + // generation is reachable anymore (first read, expiry, weight eviction, explicit + // invalidation), which is also when rollback ordering below matters. + ObservedFence observed = latestObservedFences.get(owner); + if (observed != null) { + PaimonSnapshotCacheValue memoized = + snapshotEntry.get(nameMapping.getCtlId()).peekIfPresent(observed.key); + if (memoized != null) { + return memoized; + } + } // Order fence observations, not snapshot ids: a rollback moves the latest snapshot // backwards, and a concurrent call may finish after a later observation (reversed - // completion). Either way the most recently observed fence is the one future lookups read. - long observation = fenceObservations.incrementAndGet(); + // completion). Either way the most recently observed fence is the one future lookups + // read. Capture and number assignment are serialized per owner so the observation order + // always matches the fence-read order; without this, a capture pausing between the read + // and the increment could replace a newer already-published fence with an older one. + PaimonSnapshot fence; + long observation; + Object captureLock = fenceCaptureLocks.computeIfAbsent(owner, ignored -> new Object()); + synchronized (captureLock) { + fence = loadLatestSnapshotFence(nameMapping, tableValue).getSnapshot(); + observation = fenceObservations.incrementAndGet(); + } PaimonSnapshotEntryKey key = PaimonSnapshotEntryKey.of( nameMapping, fence, tableValue.getGeneration()); MetaCacheEntry entry = @@ -159,7 +172,6 @@ public PaimonSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { nameMapping, fence, tableValue.getGeneration()) .bindCapturedAuthenticator(tableValue.getAuthenticator()); })); - LatestFenceOwner owner = new LatestFenceOwner(nameMapping, tableValue.getGeneration()); ObservedFence latest = latestObservedFences.compute(owner, (ignored, current) -> current == null || current.observation < observation ? new ObservedFence(observation, key) : current); if (loaded.get()) { @@ -172,6 +184,7 @@ public PaimonSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { // persistently rejected tables cannot grow the map, and so a delayed old-generation // load cannot resurrect an owner that catalog cleanup already removed. latestObservedFences.remove(owner); + fenceCaptureLocks.remove(owner); } return snapshotValue; } @@ -191,6 +204,8 @@ private static void retireSupersededLatestProjections( private void forgetObservedFences(NameMapping nameMapping, java.util.function.LongPredicate retiredGeneration) { latestObservedFences.keySet().removeIf(owner -> owner.nameMapping.equals(nameMapping) && retiredGeneration.test(owner.generation)); + fenceCaptureLocks.keySet().removeIf(owner -> owner.nameMapping.equals(nameMapping) + && retiredGeneration.test(owner.generation)); } private static final class LatestFenceOwner { @@ -515,21 +530,23 @@ private void retireRemovedTableGeneration(NameMapping nameMapping, @Nullable Lon @Override public void invalidateCatalog(long catalogId) { latestObservedFences.keySet().removeIf(owner -> owner.nameMapping.getCtlId() == catalogId); + fenceCaptureLocks.keySet().removeIf(owner -> owner.nameMapping.getCtlId() == catalogId); super.invalidateCatalog(catalogId); } @Override public void onCatalogPermanentlyRemoved(long catalogId) { - super.onCatalogPermanentlyRemoved(catalogId); // A lookup racing the drop may re-insert a fence owner after invalidateCatalog cleaned // the map; this hook runs even when the entry group is already retired and the id is // never reused, so the owners cannot leak for the FE lifetime. latestObservedFences.keySet().removeIf(owner -> owner.nameMapping.getCtlId() == catalogId); + fenceCaptureLocks.keySet().removeIf(owner -> owner.nameMapping.getCtlId() == catalogId); } @Override public void invalidateCatalogEntries(long catalogId) { latestObservedFences.keySet().removeIf(owner -> owner.nameMapping.getCtlId() == catalogId); + fenceCaptureLocks.keySet().removeIf(owner -> owner.nameMapping.getCtlId() == catalogId); super.invalidateCatalogEntries(catalogId); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java index f4b358a585ac6c..e17153e10f6593 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/CatalogMgrTest.java @@ -27,6 +27,7 @@ import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.MockedStatic; import org.mockito.Mockito; import java.lang.reflect.Field; @@ -119,6 +120,55 @@ void testDetachedValidationNeverPublishesCandidateToConcurrentInitialization() t } } + @Test + void testCommittedAlterRetiresTheOperationalContextButFailedAlterDoesNot() throws Exception { + CatalogMgr catalogMgr = new CatalogMgr(); + ExternalCatalog catalog = Mockito.mock(ExternalCatalog.class); + long catalogId = 45L; + Mockito.when(catalog.getId()).thenReturn(catalogId); + Mockito.when(catalog.validatePropertiesBeforeUpdate(Mockito.any(), Mockito.any())) + .thenReturn(true); + addCatalog(catalogMgr, catalog); + Map oldProperties = ImmutableMap.of("s3.access_key", "old"); + Map newProperties = ImmutableMap.of("s3.access_key", "new"); + CatalogLog log = new CatalogLog(); + log.setCatalogId(catalogId); + log.setNewProps(newProperties); + + Env env = Mockito.mock(Env.class); + ExternalMetaCacheMgr cacheMgr = Mockito.mock(ExternalMetaCacheMgr.class); + Mockito.when(env.getExtMetaCacheMgr()).thenReturn(cacheMgr); + Mockito.when(cacheMgr.withCatalogLifecycleLock(Mockito.eq(catalogId), Mockito.any())) + .thenAnswer(invocation -> { + java.util.function.Supplier action = invocation.getArgument(1); + return action.get(); + }); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + catalogMgr.replayAlterCatalogProps(log, oldProperties, false); + } + // The commit reset the catalog execution context: cached generations bound to the old + // context must be retired so the next statement loads a plannable one. + Mockito.verify(catalog).modifyCatalogProps(newProperties); + Mockito.verify(cacheMgr).onCatalogOperationalContextChanged(catalogId); + + // A failed validation never commits, so nothing may be retired. + Mockito.reset(cacheMgr); + Mockito.when(cacheMgr.withCatalogLifecycleLock(Mockito.eq(catalogId), Mockito.any())) + .thenAnswer(invocation -> { + java.util.function.Supplier action = invocation.getArgument(1); + return action.get(); + }); + Mockito.when(catalog.validatePropertiesBeforeUpdate(Mockito.any(), Mockito.any())) + .thenThrow(new IllegalArgumentException("invalid")); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + Assertions.assertThrows(DdlException.class, + () -> catalogMgr.replayAlterCatalogProps(log, oldProperties, false)); + } + Mockito.verify(cacheMgr, Mockito.never()).onCatalogOperationalContextChanged(catalogId); + } + @Test void testReplayKeepsPersistedLegacyPaimonOptionLoadableButInactive() throws Exception { CatalogMgr catalogMgr = new CatalogMgr(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java index 563788e8944471..6c84c576655b28 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/MetaCacheEntryTest.java @@ -1844,6 +1844,43 @@ private void awaitValue(MetaCacheEntry entry, String key, String Assert.assertEquals(expected, entry.peekIfPresent(key)); } + @Test + public void testDeadReservationsReleaseBeforeDependencyRetirement() throws Exception { + ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); + ExternalMetaCacheBudgetManager manager = new ExternalMetaCacheBudgetManager(OptionalLong.of(1L << 20)); + ExternalMetaCacheBudgetManager.EntryBudget budget = manager.createEntryBudget( + 1L, "test", "release-order", OptionalLong.empty(), OptionalLong.empty()); + CountDownLatch listenerEntered = new CountDownLatch(1); + CountDownLatch releaseListener = new CountDownLatch(1); + MetaCacheEntry entry = new MetaCacheEntry<>( + "release-order", key -> new byte[1], + CacheSpec.ofWeight(true, CacheSpec.CACHE_NO_TTL, 10L, 1L << 20), + refreshExecutor, false, false, + (key, value) -> MetaCacheSizeEstimate.complete(value.length), budget, null, + value -> "token", + (key, token) -> { + listenerEntered.countDown(); + awaitLatch(releaseListener); + }); + try { + entry.put("k", new byte[1500]); + Assert.assertTrue(manager.getGlobalUsedWeight() >= 1500L); + + // Automatic eviction queues both the dead reservation and the dependency + // notification for the shared cleanup worker. + extractLoadingCache(entry).policy().eviction().get().setMaximum(1L); + Assert.assertTrue(listenerEntered.await(3L, TimeUnit.SECONDS)); + + // Dependency retirement can be arbitrarily slow (it is still blocked here), but the + // dead reservation must already have released its global quota. + Assert.assertEquals(0L, manager.getGlobalUsedWeight()); + } finally { + releaseListener.countDown(); + entry.close(); + refreshExecutor.shutdownNow(); + } + } + @Test public void testRemovalListenerReceivesRemovedValuesButNotReplacements() throws Exception { ExecutorService refreshExecutor = Executors.newSingleThreadExecutor(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index dbb6a97522b431..28a95c632df881 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -587,6 +587,9 @@ public void testContendedPolicyHandoffRetriesInsteadOfFailingTheLookup() { PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); try { AtomicInteger prepareAttempts = new AtomicInteger(); + // The catalog stays registered during a contended handoff; without an Env in this + // test the live-catalog probe must be pinned to "not dropped". + cache.bindDroppedCatalogProbeForTest(catalogId -> false); cache.bindCatalogPreparer(catalogId -> { // Simulate a fence contended for the first attempts, then a successful handoff. if (prepareAttempts.incrementAndGet() >= 3) { @@ -1097,11 +1100,13 @@ public void testPermanentlyDroppedCatalogFailsLookupsImmediately() { cache.initCatalog(1L, Collections.emptyMap()); AtomicInteger dropPreparerCalls = new AtomicInteger(); cache.bindCatalogPreparer(id -> dropPreparerCalls.incrementAndGet()); - // DROP CATALOG: the group is retired and the removal is permanent - no preparer can - // ever restore it, so the lookup must fail terminally instead of consuming the - // contended-handoff retry window. + // DROP CATALOG: the catalog manager no longer serves the id (ids are never reused), + // so the lookup must fail terminally instead of consuming the contended-handoff + // retry window. The probe stands in for the live catalog-manager lookup, which the + // manager updates before any engine group is detached or closed - no per-engine + // dropped-id state is retained. cache.invalidateCatalog(1L); - cache.onCatalogPermanentlyRemoved(1L); + cache.bindDroppedCatalogProbeForTest(id -> id == 1L); long startNanos = System.nanoTime(); try { cache.entry(1L, PaimonExternalMetaCache.ENTRY_TABLE, @@ -1138,6 +1143,128 @@ public void testPermanentlyDroppedCatalogFailsLookupsImmediately() { } } + @Test + public void testFenceCaptureAndObservationAssignmentAreSerializedPerOwner() throws Exception { + ExecutionAuthenticator authenticator = new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + return task.call(); + } + }; + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + PaimonExternalDatabase database = Mockito.mock(PaimonExternalDatabase.class); + PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.doReturn(catalog).when(catalogMgr) + .getCatalogOrException(Mockito.eq(1L), Mockito.any()); + Mockito.doReturn(catalog).when(catalogMgr).getCatalog(1L); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(authenticator); + Mockito.doReturn(database).when(catalog).getDbNullable("db"); + Mockito.when(database.getTableNullable("tbl")).thenReturn(externalTable); + Mockito.doReturn(Optional.of(database)).when(catalog).getDb("db"); + Mockito.doReturn(Optional.of(externalTable)).when(database).getTable("tbl"); + Mockito.doAnswer(invocation -> { + Column partitionColumn = new Column("part", Type.INT); + return new PaimonSchemaCacheValue( + Collections.singletonList(partitionColumn), + Collections.singletonList(partitionColumn), null); + }).when(externalTable).loadSchemaForCache(Mockito.any(), Mockito.anyLong()); + + FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); + FileStoreTable latestSchemaTable = Mockito.mock(FileStoreTable.class); + FileStoreTable fenceTable = Mockito.mock(FileStoreTable.class); + FileStoreTable snapshotTable = Mockito.mock(FileStoreTable.class); + Snapshot latestSnapshot = Mockito.mock(Snapshot.class); + SchemaManager schemaManager = Mockito.mock(SchemaManager.class); + TableSchema latestSchema = Mockito.mock(TableSchema.class); + ReadBuilder readBuilder = Mockito.mock(ReadBuilder.class); + TableScan tableScan = Mockito.mock(TableScan.class); + java.util.concurrent.atomic.AtomicLong latestSnapshotId = + new java.util.concurrent.atomic.AtomicLong(8L); + AtomicReference blockNextFenceIdRead = + new AtomicReference<>(); + java.util.concurrent.CountDownLatch fenceIdReadEntered = + new java.util.concurrent.CountDownLatch(1); + Mockito.when(baseTable.copyWithLatestSchema()).thenReturn(latestSchemaTable); + Mockito.when(latestSchemaTable.latestSnapshot()).thenReturn(Optional.of(latestSnapshot)); + Mockito.when(latestSnapshot.id()).thenAnswer(invocation -> { + // Capture the id first, then optionally pause: the capture pauses between reading + // the fence and taking its observation number, exactly the racy window. + long id = latestSnapshotId.get(); + java.util.concurrent.CountDownLatch block = blockNextFenceIdRead.getAndSet(null); + if (block != null) { + fenceIdReadEntered.countDown(); + Assert.assertTrue(block.await(5L, java.util.concurrent.TimeUnit.SECONDS)); + } + return id; + }); + Mockito.when(latestSchemaTable.schemaManager()).thenReturn(schemaManager); + Mockito.when(schemaManager.latest()).thenReturn(Optional.of(latestSchema)); + Mockito.when(latestSchema.id()).thenReturn(3L); + Mockito.when(latestSchemaTable.copyWithoutTimeTravel(Mockito.anyMap())).thenReturn(fenceTable); + Mockito.when(fenceTable.copyWithoutTimeTravel(Mockito.anyMap())).thenReturn(snapshotTable); + Mockito.when(fenceTable.options()).thenReturn(Collections.emptyMap()); + Mockito.when(snapshotTable.options()).thenReturn(Collections.emptyMap()); + Mockito.when(snapshotTable.newReadBuilder()).thenReturn(readBuilder); + Mockito.when(readBuilder.newScan()).thenReturn(tableScan); + Mockito.when(tableScan.listPartitionEntries()).thenReturn(Collections.emptyList()); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + ExecutorService workers = Executors.newFixedThreadPool(2); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + cache.initCatalog(1L, Collections.emptyMap()); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); + PaimonTableCacheValue tableValue = new PaimonTableCacheValue(baseTable, authenticator); + cache.entry(1L, PaimonExternalMetaCache.ENTRY_TABLE, + NameMapping.class, PaimonTableCacheValue.class).put(mapping, tableValue); + org.apache.doris.datasource.metacache.MetaCacheEntry< + PaimonSnapshotEntryKey, PaimonSnapshotCacheValue> snapshots = cache.entry( + 1L, PaimonExternalMetaCache.ENTRY_SNAPSHOT, + PaimonSnapshotEntryKey.class, PaimonSnapshotCacheValue.class); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + + // A reads fence 8 and pauses before its observation number is assigned. + java.util.concurrent.CountDownLatch releaseOlderCapture = + new java.util.concurrent.CountDownLatch(1); + blockNextFenceIdRead.set(releaseOlderCapture); + java.util.concurrent.Future older = workers.submit(() -> { + try (MockedStatic workerEnv = Mockito.mockStatic(Env.class)) { + workerEnv.when(Env::getCurrentEnv).thenReturn(env); + return cache.getSnapshotCache(dorisTable); + } + }); + Assert.assertTrue(fenceIdReadEntered.await(5L, java.util.concurrent.TimeUnit.SECONDS)); + latestSnapshotId.set(9L); + // B must not capture fence 9 while A is still inside its capture window. + java.util.concurrent.Future newer = workers.submit(() -> { + try (MockedStatic workerEnv = Mockito.mockStatic(Env.class)) { + workerEnv.when(Env::getCurrentEnv).thenReturn(env); + return cache.getSnapshotCache(dorisTable); + } + }); + Thread.sleep(200L); + Assert.assertFalse("fence capture must be serialized per owner", newer.isDone()); + + releaseOlderCapture.countDown(); + Assert.assertEquals(8L, older.get(5L, java.util.concurrent.TimeUnit.SECONDS) + .getSnapshot().getSnapshotId()); + Assert.assertEquals(9L, newer.get(5L, java.util.concurrent.TimeUnit.SECONDS) + .getSnapshot().getSnapshotId()); + // The later capture read the later fence and must own the memoized latest. + Assert.assertEquals(9L, cache.getSnapshotCache(dorisTable).getSnapshot().getSnapshotId()); + Assert.assertEquals(1L, snapshots.stats().getEstimatedSize()); + } finally { + cache.close(); + executor.shutdownNow(); + workers.shutdownNow(); + } + } + private static boolean exceptionChainContains(Throwable throwable, String fragment) { for (Throwable current = throwable; current != null; current = current.getCause()) { if (current.getMessage() != null && current.getMessage().contains(fragment)) { From b090024bacd197aa013e24bf69921b19f1896ffc Mon Sep 17 00:00:00 2001 From: guoqiang Date: Sun, 23 Aug 2026 04:54:12 +0800 Subject: [PATCH 43/45] [fix](paimon) Release capture-lock owners on fence and projection failure paths The fence read or the projection load can throw before the unpublished-generation cleanup ran, and a weight-rejected table produces a fresh generation - and therefore a fresh capture-lock owner - on every lookup, so repeated failures for an oversized or unsupported table stranded fenceCaptureLocks entries outside every configured budget. Capture through projection publication now runs under a finally that performs the unpublished-generation cleanup on all paths, conditionally removing the exact registered lock. Regression drives repeated rejected loads through both failure points and asserts neither the lock map nor the observed-fence map retains an owner. --- .../paimon/PaimonExternalMetaCache.java | 72 +++++++------- .../paimon/PaimonExternalMetaCacheTest.java | 93 +++++++++++++++++++ 2 files changed, 134 insertions(+), 31 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java index 275d6c5e045dc2..86020bb001d280 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCache.java @@ -153,40 +153,50 @@ public PaimonSnapshotCacheValue getSnapshotCache(ExternalTable dorisTable) { // read. Capture and number assignment are serialized per owner so the observation order // always matches the fence-read order; without this, a capture pausing between the read // and the increment could replace a newer already-published fence with an older one. - PaimonSnapshot fence; - long observation; - Object captureLock = fenceCaptureLocks.computeIfAbsent(owner, ignored -> new Object()); - synchronized (captureLock) { - fence = loadLatestSnapshotFence(nameMapping, tableValue).getSnapshot(); - observation = fenceObservations.incrementAndGet(); - } - PaimonSnapshotEntryKey key = PaimonSnapshotEntryKey.of( - nameMapping, fence, tableValue.getGeneration()); MetaCacheEntry entry = snapshotEntry.get(nameMapping.getCtlId()); - AtomicBoolean loaded = new AtomicBoolean(); - PaimonSnapshotCacheValue snapshotValue = entry.get(key, - ignored -> executeForGeneration(tableValue, nameMapping, () -> { - loaded.set(true); - return latestSnapshotProjectionLoader.loadAtFence( - nameMapping, fence, tableValue.getGeneration()) - .bindCapturedAuthenticator(tableValue.getAuthenticator()); - })); - ObservedFence latest = latestObservedFences.compute(owner, (ignored, current) -> - current == null || current.observation < observation ? new ObservedFence(observation, key) : current); - if (loaded.get()) { - retireSupersededLatestProjections(entry, owner, latest.key); - } - if (!isCurrentTableGeneration(nameMapping, tableValue.getGeneration())) { - entry.invalidateKeyIfSame(key, snapshotValue); - // A generation that is not published (rejected admission, replaced or invalidated - // mid-load) can never be observed again; drop the owner this call registered so - // persistently rejected tables cannot grow the map, and so a delayed old-generation - // load cannot resurrect an owner that catalog cleanup already removed. - latestObservedFences.remove(owner); - fenceCaptureLocks.remove(owner); + Object captureLock = fenceCaptureLocks.computeIfAbsent(owner, ignored -> new Object()); + PaimonSnapshotEntryKey key = null; + PaimonSnapshotCacheValue snapshotValue = null; + try { + PaimonSnapshot fence; + long observation; + synchronized (captureLock) { + fence = loadLatestSnapshotFence(nameMapping, tableValue).getSnapshot(); + observation = fenceObservations.incrementAndGet(); + } + key = PaimonSnapshotEntryKey.of(nameMapping, fence, tableValue.getGeneration()); + PaimonSnapshotEntryKey loadKey = key; + AtomicBoolean loaded = new AtomicBoolean(); + snapshotValue = entry.get(key, + ignored -> executeForGeneration(tableValue, nameMapping, () -> { + loaded.set(true); + return latestSnapshotProjectionLoader.loadAtFence( + nameMapping, fence, tableValue.getGeneration()) + .bindCapturedAuthenticator(tableValue.getAuthenticator()); + })); + ObservedFence latest = latestObservedFences.compute(owner, (ignored, current) -> + current == null || current.observation < observation + ? new ObservedFence(observation, loadKey) : current); + if (loaded.get()) { + retireSupersededLatestProjections(entry, owner, latest.key); + } + return snapshotValue; + } finally { + if (!isCurrentTableGeneration(nameMapping, tableValue.getGeneration())) { + // A generation that is not published (rejected admission, replaced or invalidated + // mid-load) can never be observed again; drop everything this call registered - + // including on the failure paths of the fence read and the projection load, where + // every retried lookup would otherwise strand a fresh capture-lock owner - so + // persistently rejected tables cannot grow either map, and a delayed + // old-generation load cannot resurrect an owner catalog cleanup already removed. + if (key != null && snapshotValue != null) { + entry.invalidateKeyIfSame(key, snapshotValue); + } + latestObservedFences.remove(owner); + fenceCaptureLocks.remove(owner, captureLock); + } } - return snapshotValue; } /** diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 28a95c632df881..d682bcd3eabcc7 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -1265,6 +1265,90 @@ public T execute(Callable task) throws Exception { } } + @Test + public void testFailedFenceOrProjectionLoadsDoNotStrandCaptureLockOwners() { + ExecutionAuthenticator authenticator = new ExecutionAuthenticator() { + @Override + public T execute(Callable task) throws Exception { + return task.call(); + } + }; + PaimonExternalCatalog catalog = Mockito.mock(PaimonExternalCatalog.class); + CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class); + Env env = Mockito.mock(Env.class); + Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr); + Mockito.doReturn(catalog).when(catalogMgr) + .getCatalogOrException(Mockito.eq(1L), Mockito.any()); + Mockito.when(catalog.getExecutionAuthenticator()).thenReturn(authenticator); + NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); + FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); + FileStoreTable latestSchemaTable = Mockito.mock(FileStoreTable.class); + Snapshot latestSnapshot = Mockito.mock(Snapshot.class); + SchemaManager schemaManager = Mockito.mock(SchemaManager.class); + TableSchema latestSchema = Mockito.mock(TableSchema.class); + java.util.concurrent.atomic.AtomicBoolean fenceReadFails = + new java.util.concurrent.atomic.AtomicBoolean(true); + Mockito.when(baseTable.copyWithLatestSchema()).thenAnswer(invocation -> { + if (fenceReadFails.get()) { + throw new RuntimeException("fence read failed"); + } + return latestSchemaTable; + }); + Mockito.when(latestSchemaTable.latestSnapshot()).thenReturn(Optional.of(latestSnapshot)); + Mockito.when(latestSnapshot.id()).thenReturn(7L); + Mockito.when(latestSchemaTable.schemaManager()).thenReturn(schemaManager); + Mockito.when(schemaManager.latest()).thenReturn(Optional.of(latestSchema)); + Mockito.when(latestSchema.id()).thenReturn(3L); + // The projection load fails after a successful fence read. + Mockito.when(latestSchemaTable.copyWithoutTimeTravel(Mockito.anyMap())) + .thenThrow(new RuntimeException("projection load failed")); + Mockito.when(catalog.getPaimonTable(mapping)).thenReturn(baseTable); + + ExecutorService executor = Executors.newSingleThreadExecutor(); + PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); + try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { + mockedEnv.when(Env::getCurrentEnv).thenReturn(env); + // Weight-bounded table entry with an unsupported (mocked) table: every load is + // rejected, so each lookup runs on a fresh unpublished generation. + cache.initCatalog(1L, Collections.singletonMap( + "meta.cache.paimon.table.max-weight", "1MB")); + ExternalTable dorisTable = Mockito.mock(ExternalTable.class); + Mockito.when(dorisTable.getOrBuildNameMapping()).thenReturn(mapping); + + // Exception point 1: the fence read itself fails. + for (int i = 0; i < 3; i++) { + try { + cache.getSnapshotCache(dorisTable); + Assert.fail("the failing fence read must surface"); + } catch (RuntimeException e) { + Assert.assertTrue(String.valueOf(e.getMessage()), + exceptionChainContains(e, "fence read failed")); + } + } + Assert.assertEquals("failed fence reads must not strand capture-lock owners", + 0, fenceCaptureLockCount(cache)); + Assert.assertEquals(0, observedFenceOwnerCount(cache)); + + // Exception point 2: the fence read succeeds, the projection load fails. + fenceReadFails.set(false); + for (int i = 0; i < 3; i++) { + try { + cache.getSnapshotCache(dorisTable); + Assert.fail("the failing projection load must surface"); + } catch (RuntimeException e) { + Assert.assertTrue(String.valueOf(e.getMessage()), + exceptionChainContains(e, "projection load failed")); + } + } + Assert.assertEquals("failed projection loads must not strand capture-lock owners", + 0, fenceCaptureLockCount(cache)); + Assert.assertEquals(0, observedFenceOwnerCount(cache)); + } finally { + cache.close(); + executor.shutdownNow(); + } + } + private static boolean exceptionChainContains(Throwable throwable, String fragment) { for (Throwable current = throwable; current != null; current = current.getCause()) { if (current.getMessage() != null && current.getMessage().contains(fragment)) { @@ -2380,6 +2464,15 @@ private Map sizeOnlyMap(int size) { return map; } + private int fenceCaptureLockCount(PaimonExternalMetaCache cache) { + try { + return ((java.util.Map) readField( + cache, PaimonExternalMetaCache.class, "fenceCaptureLocks")).size(); + } catch (Exception e) { + throw new IllegalStateException(e); + } + } + private int observedFenceOwnerCount(PaimonExternalMetaCache cache) { try { return ((java.util.Map) readField( From e2e998c24783ec12bf66fe291f4e7499af5c0f89 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Mon, 24 Aug 2026 10:03:45 +0800 Subject: [PATCH 44/45] [test](fe) Resolve the planning-fence context lazily and pin both terminal lookup outcomes The planning fence resolved the catalog's current authenticator eagerly even for projections that carry no captured context, so scan-node rigs without a stubbed catalog hit an NPE while the fence itself would have passed trivially; the current context is now resolved only when a captured context exists to compare against. testEntryFailsFastAfterCatalogRemoved depended on whichever Env the surefire fork happened to carry to pick between the uninitialized and dropped terminal messages; it now pins both outcomes through the dropped-catalog probe: a still-registered catalog fails as uninitialized, a dropped id fails as dropped. --- .../datasource/iceberg/source/IcebergScanNode.java | 10 ++++++---- .../metacache/AbstractExternalMetaCacheTest.java | 12 +++++++++++- 2 files changed, 17 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index d851c695e4f8bd..c390a0594265e9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -800,10 +800,12 @@ private Table useFrozenTableGeneration(Table currentTable) { // frozen operations/FileIO while the catalog still serves the generation this // statement pinned; after a credential/storage ALTER the statement must fail and // be retried. Count-mode values retain no frozen handle and plan the live table, - // so they are not fenced. - cacheValue.ensurePlannableUnder( - source.getCatalog().getExecutionAuthenticator(), - source.getTargetTable().getName()); + // so they are not fenced; values without a captured context resolve nothing here. + if (cacheValue.getCapturedAuthenticator() != null) { + cacheValue.ensurePlannableUnder( + source.getCatalog().getExecutionAuthenticator(), + source.getTargetTable().getName()); + } Table frozenBaseTable = frozenTable.get(); if (isSystemTable && source.getTargetTable() instanceof IcebergSysExternalTable) { IcebergSysExternalTable systemTable = (IcebergSysExternalTable) source.getTargetTable(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java index 3decfbef1b0483..92eef16e764806 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/metacache/AbstractExternalMetaCacheTest.java @@ -93,9 +93,19 @@ public void testEntryFailsFastAfterCatalogRemoved() { cache.initCatalog(1L, Maps.newHashMap()); cache.invalidateCatalog(1L); + // While the catalog is still registered (rename/transient absence), the lookup + // fails as uninitialized; once the catalog manager no longer serves the id (DROP, + // ids are never reused), it fails as dropped. Pin both outcomes via the probe so + // the test does not depend on whichever Env this fork happens to carry. + cache.bindDroppedCatalogProbeForTest(catalogId -> false); IllegalStateException exception = Assert.assertThrows(IllegalStateException.class, () -> cache.entry(1L, "schema", SchemaCacheKey.class, SchemaCacheValue.class)); - Assert.assertTrue(exception.getMessage().contains("not initialized")); + Assert.assertTrue(exception.getMessage(), exception.getMessage().contains("not initialized")); + + cache.bindDroppedCatalogProbeForTest(catalogId -> catalogId == 1L); + IllegalStateException dropped = Assert.assertThrows(IllegalStateException.class, + () -> cache.entry(1L, "schema", SchemaCacheKey.class, SchemaCacheValue.class)); + Assert.assertTrue(dropped.getMessage(), dropped.getMessage().contains("was dropped")); Assert.assertFalse(cache.isCatalogInitialized(1L)); } finally { refreshExecutor.shutdownNow(); From 2df6c43f4731cd4297055606804fc31d20d6b8c6 Mon Sep 17 00:00:00 2001 From: guoqiang Date: Mon, 24 Aug 2026 11:46:54 +0800 Subject: [PATCH 45/45] [test](paimon) Align loader-path stubs with reloadPaimonTable after merging branch-4.1 Base commit a61493093f0 routes the Doris table-cache miss loader through PaimonExternalCatalog.reloadPaimonTable (invalidating Paimon's CachingCatalog entry before loading). Rigs that exercise the miss loader stubbed the old getPaimonTable entry point and received null tables in the merged tree; they now stub and verify reloadPaimonTable, and the retained-handle schema test asserts neither entry point is consulted. --- .../paimon/PaimonExternalMetaCacheTest.java | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index 66ecd26ffef58a..a03ffa8435ce7f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -515,6 +515,7 @@ public void testGenerationZeroFenceResolvesSchemaFromRetainedTable() { // Schema history came from the retained handle, not from a reloaded base table. Mockito.verify(dorisTable).loadSchemaForCache(Mockito.same(retainedTable), Mockito.eq(3L)); Mockito.verify(mocked.catalog, Mockito.never()).getPaimonTable(mocked.mapping); + Mockito.verify(mocked.catalog, Mockito.never()).reloadPaimonTable(mocked.mapping); } finally { cache.close(); executor.shutdownNow(); @@ -980,7 +981,7 @@ public T execute(Callable task) throws Exception { Table paimonTable = Mockito.mock(Table.class); java.util.concurrent.atomic.AtomicBoolean alterCompletesDuringLoad = new java.util.concurrent.atomic.AtomicBoolean(true); - Mockito.when(catalog.getPaimonTable(mapping)).thenAnswer(invocation -> { + Mockito.when(catalog.reloadPaimonTable(mapping)).thenAnswer(invocation -> { if (alterCompletesDuringLoad.get()) { // The concurrent credential ALTER reinitializes the catalog while the external // load is still in flight. @@ -1045,7 +1046,7 @@ public T execute(Callable task) throws Exception { }); NameMapping mapping = new NameMapping(1L, "db", "tbl", "remote_db", "remote_tbl"); Table paimonTable = Mockito.mock(Table.class); - Mockito.when(catalog.getPaimonTable(mapping)).thenReturn(paimonTable); + Mockito.when(catalog.reloadPaimonTable(mapping)).thenReturn(paimonTable); ExecutorService executor = Executors.newSingleThreadExecutor(); PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); try (MockedStatic mockedEnv = Mockito.mockStatic(Env.class)) { @@ -1303,7 +1304,7 @@ public T execute(Callable task) throws Exception { // The projection load fails after a successful fence read. Mockito.when(latestSchemaTable.copyWithoutTimeTravel(Mockito.anyMap())) .thenThrow(new RuntimeException("projection load failed")); - Mockito.when(catalog.getPaimonTable(mapping)).thenReturn(baseTable); + Mockito.when(catalog.reloadPaimonTable(mapping)).thenReturn(baseTable); ExecutorService executor = Executors.newSingleThreadExecutor(); PaimonExternalMetaCache cache = new PaimonExternalMetaCache(executor); @@ -1619,7 +1620,7 @@ public T execute(Callable task) throws Exception { } return Collections.emptyList(); }); - Mockito.when(catalog.getPaimonTable(mapping)).thenReturn(baseTable); + Mockito.when(catalog.reloadPaimonTable(mapping)).thenReturn(baseTable); } private ExternalTable dorisTable() { @@ -1664,7 +1665,7 @@ public void testRejectedBaseTableDoesNotAccumulateSnapshotOrSchemaProjections() 0, observedFenceOwnerCount(cache)); } Assert.assertEquals(10L, tables.stats().getWeightAdmissionRejectedCount()); - Mockito.verify(mocked.catalog, Mockito.times(10)).getPaimonTable(mapping); + Mockito.verify(mocked.catalog, Mockito.times(10)).reloadPaimonTable(mapping); } finally { cache.close(); executor.shutdownNow();