From 346f333a18da92326f59a2d0ad7f5588e555c522 Mon Sep 17 00:00:00 2001 From: zhangstar333 Date: Mon, 27 Jul 2026 15:40:37 +0800 Subject: [PATCH 1/5] [feature](paimon) Support Paimon table option passthrough (#65955) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit doc： https://github.com/apache/doris-website/pull/4012 Paimon provides many configurable table options, now introduces the `paimon.table-option.*` namespace. Doris removes the prefix, validates the option through Paimon's `SupportedTableOptions`, and applies it to the Paimon `Table` before serialization. Explicit options, such as `paimon.table-option.read.batch-size`, are preserved by the JNI scanner. Invalid or unsupported options fail fast during Catalog initialization. eg: ``` CREATE CATALOG paimon_catalog PROPERTIES ( "type" = "paimon", "paimon.catalog.type" = "filesystem", "warehouse" = "hdfs://127.0.0.1:8020/user/paimon/warehouse", "paimon.table-option.read.batch-size" = "4096", "paimon.jni.enable_jni_io_manager" = "true", "paimon.jni.io_manager.tmp_dir" = "/data/doris/paimon_jni_tmp" ); ``` 2. some jni params all named with paimon.jni.xxxx --- be/src/format/table/paimon_jni_reader.cpp | 4 +- be/src/format_v2/jni/paimon_jni_reader.cpp | 4 +- .../format_v2/table/paimon_reader_test.cpp | 18 +-- .../apache/doris/paimon/PaimonJniScanner.java | 11 +- .../paimon/PaimonExternalCatalog.java | 16 ++- .../paimon/source/PaimonScanNode.java | 6 +- .../metastore/AbstractPaimonProperties.java | 109 +++++++++++++++++- .../paimon/source/PaimonScanNodeTest.java | 12 +- .../AbstractPaimonPropertiesTest.java | 97 ++++++++++++++++ 9 files changed, 240 insertions(+), 37 deletions(-) diff --git a/be/src/format/table/paimon_jni_reader.cpp b/be/src/format/table/paimon_jni_reader.cpp index 772c97e2aa6067..b285f800ce5442 100644 --- a/be/src/format/table/paimon_jni_reader.cpp +++ b/be/src/format/table/paimon_jni_reader.cpp @@ -41,8 +41,8 @@ constexpr std::string_view PAIMON_JNI_SCANNER_IO_TMP_DIR = "paimon_jni_scanner_i const std::string PaimonJniReader::PAIMON_OPTION_PREFIX = "paimon."; const std::string PaimonJniReader::HADOOP_OPTION_PREFIX = "hadoop."; -const std::string PaimonJniReader::DORIS_ENABLE_JNI_IO_MANAGER = "doris.enable_jni_io_manager"; -const std::string PaimonJniReader::DORIS_JNI_IO_MANAGER_TMP_DIR = "doris.jni_io_manager.tmp_dir"; +const std::string PaimonJniReader::DORIS_ENABLE_JNI_IO_MANAGER = "jni.enable_jni_io_manager"; +const std::string PaimonJniReader::DORIS_JNI_IO_MANAGER_TMP_DIR = "jni.io_manager.tmp_dir"; PaimonJniReader::PaimonJniReader(const std::vector& file_slot_descs, RuntimeState* state, RuntimeProfile* profile, diff --git a/be/src/format_v2/jni/paimon_jni_reader.cpp b/be/src/format_v2/jni/paimon_jni_reader.cpp index 86d16ce6f7d7fd..730d431d4cb087 100644 --- a/be/src/format_v2/jni/paimon_jni_reader.cpp +++ b/be/src/format_v2/jni/paimon_jni_reader.cpp @@ -28,8 +28,8 @@ namespace { constexpr std::string_view PAIMON_OPTION_PREFIX = "paimon."; constexpr std::string_view HADOOP_OPTION_PREFIX = "hadoop."; -constexpr std::string_view DORIS_ENABLE_JNI_IO_MANAGER = "doris.enable_jni_io_manager"; -constexpr std::string_view DORIS_JNI_IO_MANAGER_TMP_DIR = "doris.jni_io_manager.tmp_dir"; +constexpr std::string_view DORIS_ENABLE_JNI_IO_MANAGER = "jni.enable_jni_io_manager"; +constexpr std::string_view DORIS_JNI_IO_MANAGER_TMP_DIR = "jni.io_manager.tmp_dir"; constexpr std::string_view PAIMON_JNI_SCANNER_IO_TMP_DIR = "paimon_jni_scanner_io_tmp"; const std::string* get_paimon_predicate(const TFileScanRangeParams* scan_params, diff --git a/be/test/format_v2/table/paimon_reader_test.cpp b/be/test/format_v2/table/paimon_reader_test.cpp index 06301815b49228..6ebec512f42fdf 100644 --- a/be/test/format_v2/table/paimon_reader_test.cpp +++ b/be/test/format_v2/table/paimon_reader_test.cpp @@ -888,17 +888,17 @@ TEST(PaimonHybridReaderTest, FirstNativeAndJniChildInitAreCountedOnce) { TEST(PaimonJniReaderTest, BuildScannerParamsKeepsExplicitIOManagerTempDir) { auto scan_params = make_paimon_jni_scan_params(); scan_params.__set_paimon_options({ - {"doris.enable_jni_io_manager", "true"}, - {"doris.jni_io_manager.tmp_dir", "/tmp/explicit-paimon-spill"}, - {"doris.jni_io_manager.impl_class", "org.example.CustomIOManager"}, + {"jni.enable_jni_io_manager", "true"}, + {"jni.io_manager.tmp_dir", "/tmp/explicit-paimon-spill"}, + {"jni.io_manager.impl_class", "org.example.CustomIOManager"}, }); RuntimeState state {TQueryOptions(), TQueryGlobals()}; state.set_exec_env(ExecEnv::GetInstance()); auto params = build_paimon_jni_scanner_params(&scan_params, &state); - EXPECT_EQ(params["paimon.doris.enable_jni_io_manager"], "true"); - EXPECT_EQ(params["paimon.doris.jni_io_manager.tmp_dir"], "/tmp/explicit-paimon-spill"); - EXPECT_EQ(params["paimon.doris.jni_io_manager.impl_class"], "org.example.CustomIOManager"); + EXPECT_EQ(params["paimon.jni.enable_jni_io_manager"], "true"); + EXPECT_EQ(params["paimon.jni.io_manager.tmp_dir"], "/tmp/explicit-paimon-spill"); + EXPECT_EQ(params["paimon.jni.io_manager.impl_class"], "org.example.CustomIOManager"); } TEST(PaimonJniReaderTest, BuildScannerParamsInjectsStorageRootTmpDirForEnabledIOManager) { @@ -908,14 +908,14 @@ TEST(PaimonJniReaderTest, BuildScannerParamsInjectsStorageRootTmpDirForEnabledIO }); auto scan_params = make_paimon_jni_scan_params(); scan_params.__set_paimon_options({ - {"doris.enable_jni_io_manager", "true"}, + {"jni.enable_jni_io_manager", "true"}, }); RuntimeState state {TQueryOptions(), TQueryGlobals()}; state.set_exec_env(ExecEnv::GetInstance()); auto params = build_paimon_jni_scanner_params(&scan_params, &state); - EXPECT_EQ(params["paimon.doris.enable_jni_io_manager"], "true"); - EXPECT_EQ(params["paimon.doris.jni_io_manager.tmp_dir"], + EXPECT_EQ(params["paimon.jni.enable_jni_io_manager"], "true"); + EXPECT_EQ(params["paimon.jni.io_manager.tmp_dir"], "/data1/doris/paimon_jni_scanner_io_tmp:/data2/doris/" "paimon_jni_scanner_io_tmp"); } diff --git a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java index 0749720840ad5b..3279aa3df746af 100644 --- a/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java +++ b/fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java @@ -24,7 +24,6 @@ import org.apache.doris.common.security.authentication.PreExecutionAuthenticatorCache; import com.google.common.base.Preconditions; -import org.apache.paimon.CoreOptions; import org.apache.paimon.data.InternalRow; import org.apache.paimon.disk.IOManager; import org.apache.paimon.disk.IOManagerImpl; @@ -50,7 +49,6 @@ import java.nio.file.Files; import java.nio.file.Paths; import java.util.Arrays; -import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Locale; @@ -66,9 +64,9 @@ public class PaimonJniScanner extends JniScanner { private static final String PAIMON_OPTION_PREFIX = "paimon."; private static final String ASYNC_READER_THREAD_NAME_PREFIX = "paimon-reader-async-thread"; private static final String FILE_READER_ASYNC_THRESHOLD = "file-reader-async-threshold"; - static final String ENABLE_JNI_IO_MANAGER = "paimon.doris.enable_jni_io_manager"; - static final String JNI_IO_MANAGER_TMP_DIR = "paimon.doris.jni_io_manager.tmp_dir"; - static final String JNI_IO_MANAGER_IMPL_CLASS = "paimon.doris.jni_io_manager.impl_class"; + static final String ENABLE_JNI_IO_MANAGER = "paimon.jni.enable_jni_io_manager"; + static final String JNI_IO_MANAGER_TMP_DIR = "paimon.jni.io_manager.tmp_dir"; + static final String JNI_IO_MANAGER_IMPL_CLASS = "paimon.jni.io_manager.impl_class"; private static final AtomicInteger ACTIVE_SCANNERS = new AtomicInteger(); private final Map params; @@ -570,8 +568,6 @@ static Optional parseDataSizeBytes(String value) { private void initTable() { Preconditions.checkState(params.containsKey("serialized_table")); table = PaimonUtils.deserialize(params.get("serialized_table")); - table = table.copy(Collections.singletonMap( - CoreOptions.READ_BATCH_SIZE.key(), String.valueOf(batchSize))); paimonAllFieldNames = PaimonUtils.getFieldNames(this.table.rowType()); if (LOG.isDebugEnabled()) { LOG.debug("paimonAllFieldNames:{}", paimonAllFieldNames); @@ -584,5 +580,4 @@ private static String[] splitParam(String value, String delimiter) { } return value.split(delimiter); } - } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java index 5d0c74488f2872..a0f489ad065d9a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalCatalog.java @@ -34,6 +34,7 @@ import org.apache.paimon.catalog.Catalog; import org.apache.paimon.catalog.Identifier; import org.apache.paimon.partition.Partition; +import org.apache.paimon.table.Table; import java.util.ArrayList; import java.util.List; @@ -114,12 +115,11 @@ public List getPaimonPartitions(NameMapping nameMapping) { } } - public org.apache.paimon.table.Table getPaimonTable(NameMapping nameMapping) { + public Table getPaimonTable(NameMapping nameMapping) { return getPaimonTable(nameMapping, null, null); } - public org.apache.paimon.table.Table getPaimonTable(NameMapping nameMapping, String branch, - String queryType) { + public Table getPaimonTable(NameMapping nameMapping, String branch, String queryType) { makeSureInitialized(); try { Identifier identifier; @@ -135,7 +135,12 @@ public org.apache.paimon.table.Table getPaimonTable(NameMapping nameMapping, Str } else { identifier = new Identifier(nameMapping.getRemoteDbName(), nameMapping.getRemoteTblName()); } - return executionAuthenticator.execute(() -> catalog.getTable(identifier)); + return executionAuthenticator.execute(() -> { + Table table = catalog.getTable(identifier); + Map tableOptions = + paimonProperties.getTableOptionsForCopy(table.options()); + return tableOptions.isEmpty() ? table : table.copy(tableOptions); + }); } catch (Exception e) { throw new RuntimeException("Failed to get Paimon table:" + getName() + "." + nameMapping.getRemoteDbName() + "." + nameMapping.getRemoteTblName() + "$" + queryType @@ -174,7 +179,8 @@ public void checkProperties() throws DdlException { public void notifyPropertiesUpdated(Map updatedProps) { super.notifyPropertiesUpdated(updatedProps); if (updatedProps.keySet().stream() - .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, PaimonExternalMetaCache.ENGINE))) { + .anyMatch(key -> CacheSpec.isMetaCacheKeyForEngine(key, PaimonExternalMetaCache.ENGINE) + || AbstractPaimonProperties.isTableOptionProperty(key))) { Env.getCurrentEnv().getExtMetaCacheMgr().removeCatalogByEngine(getId(), PaimonExternalMetaCache.ENGINE); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index aab64aeb034079..c269d4ded4298b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -95,9 +95,9 @@ public class PaimonScanNode extends FileQueryScanNode { private static final String DORIS_END_TIMESTAMP = "endTimestamp"; private static final String DORIS_INCREMENTAL_BETWEEN_SCAN_MODE = "incrementalBetweenScanMode"; private static final String PAIMON_PROPERTY_PREFIX = "paimon."; - private static final String DORIS_ENABLE_JNI_IO_MANAGER = "doris.enable_jni_io_manager"; - private static final String DORIS_JNI_IO_MANAGER_TMP_DIR = "doris.jni_io_manager.tmp_dir"; - private static final String DORIS_JNI_IO_MANAGER_IMPL_CLASS = "doris.jni_io_manager.impl_class"; + private static final String DORIS_ENABLE_JNI_IO_MANAGER = "jni.enable_jni_io_manager"; + private static final String DORIS_JNI_IO_MANAGER_TMP_DIR = "jni.io_manager.tmp_dir"; + private static final String DORIS_JNI_IO_MANAGER_IMPL_CLASS = "jni.io_manager.impl_class"; private static final List BACKEND_PAIMON_OPTIONS = Arrays.asList( DORIS_ENABLE_JNI_IO_MANAGER, DORIS_JNI_IO_MANAGER_TMP_DIR, diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java index 9c9631a516fdac..4ca7836b86cd78 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/property/metastore/AbstractPaimonProperties.java @@ -25,12 +25,18 @@ import lombok.Getter; import org.apache.commons.lang3.StringUtils; import org.apache.hadoop.conf.Configuration; +import org.apache.paimon.CoreOptions; import org.apache.paimon.catalog.Catalog; import org.apache.paimon.options.CatalogOptions; +import org.apache.paimon.options.ConfigOption; +import org.apache.paimon.options.FallbackKey; import org.apache.paimon.options.Options; +import java.util.Collections; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Locale; import java.util.Map; import java.util.concurrent.atomic.AtomicReference; @@ -53,6 +59,12 @@ public abstract class AbstractPaimonProperties extends MetastoreProperties { public abstract String getPaimonCatalogType(); private static final String USER_PROPERTY_PREFIX = "paimon."; + private static final String DORIS_JNI_PROPERTY_PREFIX = "paimon.jni."; + /** The suffix after this prefix is passed to Paimon as a dynamic table option. */ + public static final String TABLE_OPTION_PREFIX = "paimon.table-option."; + private static final SupportedTableOptions SUPPORTED_TABLE_OPTIONS = SupportedTableOptions.build(); + + private Map tableOptionsMap = Collections.emptyMap(); protected AbstractPaimonProperties(Map props) { super(Type.PAIMON, props); @@ -60,6 +72,12 @@ protected AbstractPaimonProperties(Map props) { public abstract Catalog initializeCatalog(String catalogName, List storagePropertiesList); + @Override + public void initNormalizeAndCheckProps() { + super.initNormalizeAndCheckProps(); + tableOptionsMap = extractTableOptions(); + } + protected void appendCatalogOptions() { if (StringUtils.isNotBlank(warehouse)) { catalogOptions.set(CatalogOptions.WAREHOUSE.key(), warehouse); @@ -68,10 +86,12 @@ protected void appendCatalogOptions() { // FIXME(cmy): Rethink these custom properties origProps.forEach((k, v) -> { - if (k.toLowerCase().startsWith(USER_PROPERTY_PREFIX)) { + if (k.toLowerCase(Locale.ROOT).startsWith(USER_PROPERTY_PREFIX)) { String newKey = k.substring(USER_PROPERTY_PREFIX.length()); if (StringUtils.isNotBlank(newKey)) { - boolean excluded = userStoragePrefixes.stream().anyMatch(k::startsWith); + boolean excluded = isTableOptionProperty(k) + || k.toLowerCase(Locale.ROOT).startsWith(DORIS_JNI_PROPERTY_PREFIX) + || userStoragePrefixes.stream().anyMatch(k::startsWith); if (!excluded) { catalogOptions.set(newKey, v); } @@ -121,6 +141,91 @@ public Map getCatalogOptionsMap() { } } + public Map getTableOptionsMap() { + return tableOptionsMap; + } + + /** + * Returns Catalog table options which are not explicitly configured by the Paimon table. + * + *

The comparison is based on Paimon {@link ConfigOption}s so canonical and fallback keys + * follow the same precedence rule. + */ + public Map getTableOptionsForCopy(Map currentTableOptions) { + if (tableOptionsMap.isEmpty() || currentTableOptions.isEmpty()) { + return tableOptionsMap; + } + + Options existingOptions = new Options(currentTableOptions); + Map optionsForCopy = new LinkedHashMap<>(); + tableOptionsMap.forEach((key, value) -> { + ConfigOption option = SUPPORTED_TABLE_OPTIONS.find(key); + if (!existingOptions.contains(option)) { + optionsForCopy.put(key, value); + } + }); + return Collections.unmodifiableMap(optionsForCopy); + } + + public static boolean isTableOptionProperty(String key) { + return key.toLowerCase(Locale.ROOT).startsWith(TABLE_OPTION_PREFIX); + } + + private Map extractTableOptions() { + Map tableOptions = new LinkedHashMap<>(); + origProps.forEach((key, value) -> { + if (isTableOptionProperty(key)) { + String tableOptionKey = key.substring(TABLE_OPTION_PREFIX.length()); + if (StringUtils.isBlank(tableOptionKey)) { + throw new IllegalArgumentException( + "Paimon table option name must not be empty after prefix " + TABLE_OPTION_PREFIX); + } + validateTableOption(tableOptionKey, value); + tableOptions.put(tableOptionKey, value); + } + }); + return Collections.unmodifiableMap(tableOptions); + } + + private void validateTableOption(String key, String value) { + ConfigOption option = SUPPORTED_TABLE_OPTIONS.find(key); + if (option == null) { + throw new IllegalArgumentException("Unsupported Paimon table option '" + key + + "' for the bundled Paimon version"); + } + + try { + new Options(Collections.singletonMap(key, value)).get(option); + } catch (IllegalArgumentException e) { + throw new IllegalArgumentException("Invalid value for Paimon table option '" + key + "': " + + e.getMessage(), e); + } + } + + private static final class SupportedTableOptions { + /** Canonical and fallback option names which support direct lookup. */ + private final Map> exactOptions; + + private SupportedTableOptions(Map> exactOptions) { + this.exactOptions = exactOptions; + } + + private static SupportedTableOptions build() { + Map> exactOptions = new HashMap<>(); + for (ConfigOption option : CoreOptions.getOptions()) { + exactOptions.put(option.key(), option); + for (FallbackKey fallbackKey : option.fallbackKeys()) { + exactOptions.put(fallbackKey.getKey(), option); + } + } + return new SupportedTableOptions(Collections.unmodifiableMap(exactOptions)); + } + + private ConfigOption find(String key) { + return exactOptions.get(key); + } + } + /** * @See org.apache.paimon.s3.S3FileIO * Possible S3 config key prefixes: diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java index 595f64758dfe97..303b4d7d36c270 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java @@ -565,9 +565,9 @@ public void testGetBackendPaimonOptionsForJdbcCatalog() throws Exception { @Test public void testGetBackendPaimonOptionsForJniIOManager() { Map props = new HashMap<>(); - props.put("paimon.doris.enable_jni_io_manager", "true"); - props.put("paimon.doris.jni_io_manager.tmp_dir", "/tmp/doris-paimon"); - props.put("paimon.doris.jni_io_manager.impl_class", "org.example.CustomIOManager"); + props.put("paimon.jni.enable_jni_io_manager", "true"); + props.put("paimon.jni.io_manager.tmp_dir", "/tmp/doris-paimon"); + props.put("paimon.jni.io_manager.impl_class", "org.example.CustomIOManager"); CatalogProperty catalogProperty = Mockito.mock(CatalogProperty.class); Mockito.when(catalogProperty.getProperties()).thenReturn(props); @@ -583,10 +583,10 @@ public void testGetBackendPaimonOptionsForJniIOManager() { node.setSource(source); Map backendOptions = node.getBackendPaimonOptions(); - Assert.assertEquals("true", backendOptions.get("doris.enable_jni_io_manager")); - Assert.assertEquals("/tmp/doris-paimon", backendOptions.get("doris.jni_io_manager.tmp_dir")); + Assert.assertEquals("true", backendOptions.get("jni.enable_jni_io_manager")); + Assert.assertEquals("/tmp/doris-paimon", backendOptions.get("jni.io_manager.tmp_dir")); Assert.assertEquals("org.example.CustomIOManager", - backendOptions.get("doris.jni_io_manager.impl_class")); + backendOptions.get("jni.io_manager.impl_class")); Assert.assertEquals(3, backendOptions.size()); } diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractPaimonPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractPaimonPropertiesTest.java index e5a775ba6e3ef2..9cac43830ff75b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractPaimonPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractPaimonPropertiesTest.java @@ -24,6 +24,7 @@ import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -86,4 +87,100 @@ void testNormalizeS3Config() { Assertions.assertTrue("3".equals(result.get("fs.s3a.replication.factor"))); } + @Test + void testExtractAndValidateTableOptions() { + Map input = new HashMap<>(); + input.put("warehouse", "s3://tmp/warehouse"); + input.put("paimon.jni.enable_jni_io_manager", "true"); + input.put("paimon.table-option.read.batch-size", "4096"); + input.put("paimon.table-option.file.compression.per.level", "0:lz4,1:zstd"); + TestPaimonProperties testProps = new TestPaimonProperties(input); + + testProps.initNormalizeAndCheckProps(); + testProps.buildCatalogOptions(); + + Assertions.assertEquals("4096", testProps.getTableOptionsMap().get("read.batch-size")); + Assertions.assertEquals( + "0:lz4,1:zstd", testProps.getTableOptionsMap().get("file.compression.per.level")); + Assertions.assertFalse(testProps.getCatalogOptionsMap().containsKey("table-option.read.batch-size")); + Assertions.assertFalse(testProps.getCatalogOptionsMap().containsKey("jni.enable_jni_io_manager")); + } + + @Test + void testPaimonTableOptionsTakePrecedenceOverCatalogOptions() { + Map input = new HashMap<>(); + input.put("warehouse", "s3://tmp/warehouse"); + input.put("paimon.table-option.read.batch-size", "4096"); + input.put("paimon.table-option.write.batch-size", "2048"); + input.put("paimon.table-option.file.compression.per.level", "0:lz4,1:zstd"); + TestPaimonProperties testProps = new TestPaimonProperties(input); + testProps.initNormalizeAndCheckProps(); + + Map currentTableOptions = new HashMap<>(); + currentTableOptions.put("read.batch-size", "1024"); + currentTableOptions.put("orc.write.batch-size", "512"); + currentTableOptions.put("file.compression.per.level", "0:snappy"); + + Map optionsForCopy = + testProps.getTableOptionsForCopy(currentTableOptions); + + Assertions.assertFalse(optionsForCopy.containsKey("read.batch-size")); + Assertions.assertFalse(optionsForCopy.containsKey("write.batch-size")); + Assertions.assertFalse(optionsForCopy.containsKey("file.compression.per.level")); + } + + @Test + void testCatalogTableOptionsFillMissingPaimonTableOptions() { + Map input = new HashMap<>(); + input.put("warehouse", "s3://tmp/warehouse"); + input.put("paimon.table-option.read.batch-size", "4096"); + TestPaimonProperties testProps = new TestPaimonProperties(input); + testProps.initNormalizeAndCheckProps(); + + Map optionsForCopy = + testProps.getTableOptionsForCopy(Collections.singletonMap( + "path", "s3://tmp/warehouse/test.db/test")); + + Assertions.assertEquals("4096", optionsForCopy.get("read.batch-size")); + } + + @Test + void testRejectUnknownTableOption() { + Map input = new HashMap<>(); + input.put("warehouse", "s3://tmp/warehouse"); + input.put("paimon.table-option.option-does-not-exist", "value"); + TestPaimonProperties testProps = new TestPaimonProperties(input); + + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, testProps::initNormalizeAndCheckProps); + + Assertions.assertTrue(exception.getMessage().contains("option-does-not-exist")); + } + + @Test + void testRejectPrefixMapTableOption() { + Map input = new HashMap<>(); + input.put("warehouse", "s3://tmp/warehouse"); + input.put("paimon.table-option.file.compression.per.level.0", "lz4"); + TestPaimonProperties testProps = new TestPaimonProperties(input); + + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, testProps::initNormalizeAndCheckProps); + + Assertions.assertTrue(exception.getMessage().contains("file.compression.per.level.0")); + } + + @Test + void testRejectInvalidTableOptionValue() { + Map input = new HashMap<>(); + input.put("warehouse", "s3://tmp/warehouse"); + input.put("paimon.table-option.read.batch-size", "not-an-integer"); + TestPaimonProperties testProps = new TestPaimonProperties(input); + + IllegalArgumentException exception = Assertions.assertThrows( + IllegalArgumentException.class, testProps::initNormalizeAndCheckProps); + + Assertions.assertTrue(exception.getMessage().contains("read.batch-size")); + } + } From 3b7d0274cb6fc3be5bb74080498436492f4a3e3e Mon Sep 17 00:00:00 2001 From: Gabriel Date: Mon, 27 Jul 2026 15:41:50 +0800 Subject: [PATCH 2/5] [test](regression) Complete Paimon read/write P0 coverage (#66065) ### What problem does this PR solve? Problem Summary: The Paimon P0 regression suite did not distinguish the four primary-key merge engines and did not verify that every unsupported Doris data-write shape leaves both data and snapshots unchanged. This PR: - adds Parquet and ORC coverage for deduplicate, partial-update, aggregation, and first-row tables under automatic and forced-JNI reader routing; - verifies dynamic-bucket cross-partition deduplication; - covers rejected INSERT, INSERT SELECT, INSERT OVERWRITE, UPDATE, DELETE, and MERGE statements with pre/post row and snapshot invariants; - records the Paimon CTAS metadata-atomicity defect as an isolated opt-in negative regression; and - documents the Paimon/Iceberg read/write P0 coverage matrix and why no additional Iceberg P0 case is needed for the currently supported surface. --- .../test_paimon_merge_engine_matrix.out | 105 +++++++++ .../paimon/test_paimon_write_boundary.out | 15 ++ .../PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md | 102 ++++++++ ...test_paimon_ctas_atomicity_negative.groovy | 74 ++++++ .../test_paimon_merge_engine_matrix.groovy | 223 ++++++++++++++++++ .../paimon/test_paimon_write_boundary.groovy | 107 +++++++++ 6 files changed, 626 insertions(+) create mode 100644 regression-test/data/external_table_p0/paimon/test_paimon_merge_engine_matrix.out create mode 100644 regression-test/data/external_table_p0/paimon/test_paimon_write_boundary.out create mode 100644 regression-test/suites/external_table_p0/PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md create mode 100644 regression-test/suites/external_table_p0/paimon/test_paimon_ctas_atomicity_negative.groovy create mode 100644 regression-test/suites/external_table_p0/paimon/test_paimon_merge_engine_matrix.groovy create mode 100644 regression-test/suites/external_table_p0/paimon/test_paimon_write_boundary.groovy diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_merge_engine_matrix.out b/regression-test/data/external_table_p0/paimon/test_paimon_merge_engine_matrix.out new file mode 100644 index 00000000000000..5b7470831c19eb --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/test_paimon_merge_engine_matrix.out @@ -0,0 +1,105 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !auto_parquet_deduplicate -- +1 11 new-1 +2 22 new-2 +3 30 only-3 + +-- !auto_parquet_partial_update -- +1 15 2 filled-1 +2 30 4 base-2 + +-- !auto_parquet_aggregation -- +1 7 35 new-1 +2 9 300 new-2 + +-- !auto_parquet_first_row -- +1 10 first-1 +2 20 first-2 +3 30 first-3 + +-- !auto_parquet_engine_aggregates -- +63 51 16 60 + +-- !auto_orc_deduplicate -- +1 11 new-1 +2 22 new-2 +3 30 only-3 + +-- !auto_orc_partial_update -- +1 15 2 filled-1 +2 30 4 base-2 + +-- !auto_orc_aggregation -- +1 7 35 new-1 +2 9 300 new-2 + +-- !auto_orc_first_row -- +1 10 first-1 +2 20 first-2 +3 30 first-3 + +-- !auto_orc_engine_aggregates -- +63 51 16 60 + +-- !auto_dynamic_cross_partition -- +1 new-part 11 +2 stable-part 20 + +-- !auto_dynamic_old_partition -- +0 + +-- !auto_dynamic_new_partition -- +1 + +-- !forced_jni_parquet_deduplicate -- +1 11 new-1 +2 22 new-2 +3 30 only-3 + +-- !forced_jni_parquet_partial_update -- +1 15 2 filled-1 +2 30 4 base-2 + +-- !forced_jni_parquet_aggregation -- +1 7 35 new-1 +2 9 300 new-2 + +-- !forced_jni_parquet_first_row -- +1 10 first-1 +2 20 first-2 +3 30 first-3 + +-- !forced_jni_parquet_engine_aggregates -- +63 51 16 60 + +-- !forced_jni_orc_deduplicate -- +1 11 new-1 +2 22 new-2 +3 30 only-3 + +-- !forced_jni_orc_partial_update -- +1 15 2 filled-1 +2 30 4 base-2 + +-- !forced_jni_orc_aggregation -- +1 7 35 new-1 +2 9 300 new-2 + +-- !forced_jni_orc_first_row -- +1 10 first-1 +2 20 first-2 +3 30 first-3 + +-- !forced_jni_orc_engine_aggregates -- +63 51 16 60 + +-- !forced_jni_dynamic_cross_partition -- +1 new-part 11 +2 stable-part 20 + +-- !forced_jni_dynamic_old_partition -- +0 + +-- !forced_jni_dynamic_new_partition -- +1 + diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_write_boundary.out b/regression-test/data/external_table_p0/paimon/test_paimon_write_boundary.out new file mode 100644 index 00000000000000..f1118d0bd7069e --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/test_paimon_write_boundary.out @@ -0,0 +1,15 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !before_rows -- +1 10 base-1 +2 20 base-2 + +-- !before_snapshots -- +1 + +-- !after_rows -- +1 10 base-1 +2 20 base-2 + +-- !after_snapshots -- +1 + diff --git a/regression-test/suites/external_table_p0/PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md b/regression-test/suites/external_table_p0/PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md new file mode 100644 index 00000000000000..f3760f4db68ced --- /dev/null +++ b/regression-test/suites/external_table_p0/PAIMON_ICEBERG_READ_WRITE_P0_COVERAGE.md @@ -0,0 +1,102 @@ + + +# Paimon / Iceberg Read and Write P0 Coverage + +## Scope + +This matrix compares the supported Doris surface with the format capabilities documented by +[Apache Paimon](https://paimon.apache.org/docs/1.0/), its +[ecosystem matrix](https://paimon.apache.org/docs/master/ecosystem/), and Iceberg's +[write](https://iceberg.apache.org/docs/latest/spark-writes/) and +[evolution](https://iceberg.apache.org/docs/latest/evolution/) documentation. A format feature is a Doris P0 contract +only when Doris supports it. Unsupported write or format boundaries require deterministic negative +coverage and no metadata or data mutation. + +Doris currently documents Paimon data access as read-only. Master also supports Paimon catalog, +database and table metadata creation/deletion, but it has no Paimon data sink. Iceberg supports read, +DDL, INSERT, INSERT OVERWRITE, CTAS and, for compatible V2/V3 tables, DELETE, UPDATE and MERGE INTO. + +## Audit result + +The master inventory before this change contained 41 Paimon suites and 145 Iceberg suites, including +35 Iceberg write suites and four dedicated Iceberg DML suites. Iceberg P0 is complete for the Doris- +supported read/write surface: its existing matrices cover file-format versions, delete encodings, +evolution, time travel, DDL, distributed writes, row-level DML, failure atomicity and unsupported +boundaries. + +Paimon P0 was incomplete. No suite selected a `merge-engine`, and write rejection was not checked +across all DML shapes with pre/post data and snapshot invariants. PM01-PM03 below close those gaps. +PM04 exposes one remaining product defect as an opt-in negative regression: failed Paimon CTAS leaves +target metadata. Streaming writes, overwrite, delete/update and merge listed as unsupported by the +Paimon ecosystem matrix are boundary tests rather than positive Doris P0 contracts. + +## Risks + +| ID | Risk | Source | Impact | Priority | +| --- | --- | --- | --- | --- | +| R01 | Append and primary-key tables are planned with the wrong merge semantics | Black box: Paimon table models | Silent wrong results | P0 | +| R02 | Deduplicate, partial-update, aggregation and first-row produce the same result for duplicate keys | White box: LSM sorted-run merge | Silent wrong results | P0 | +| R03 | Fixed and dynamic bucket tables expose duplicate or stale cross-partition keys | Paimon data distribution | Silent wrong results | P0 | +| R04 | Automatic and forced-JNI routing disagree on Paimon merged rows, or native raw-file reads disagree where conversion is supported | Doris split routing | Query correctness | P0 | +| R05 | A rejected Paimon write creates a snapshot, changes data or leaves a CTAS table | Doris read-only boundary | Data or metadata mutation | P0 | +| R06 | Iceberg schema or partition evolution binds old field/spec IDs during current or historical reads | Iceberg evolution | Silent wrong results | P0 | +| R07 | Position/equality deletes or V3 deletion vectors are applied to the wrong file or snapshot | Iceberg row-level deletes | Deleted data visible or live data lost | P0 | +| R08 | Iceberg writes lose rows, route them to the wrong transform, or publish partial failed commits | Doris distributed Iceberg sink | Data loss or corruption | P0 | +| R09 | Snapshot, tag or branch reads/writes leak schema or data across references | Both formats | Historical data corruption | P0 | +| R10 | Unsupported format, DML mode or CTAS mutates state before rejection | Capability boundary | Partial commits or orphan metadata | P0 | + +## Feature matrix + +| Format | Capability | P0 status | Main suites | +| --- | --- | --- | --- | +| Paimon | Append table, partitioned table, primitive and nested types | Covered | `test_paimon_catalog`, `test_paimon_partition_table`, `test_paimon_full_schema_change` | +| Paimon | Primary-key deduplicate, partial-update, aggregation and first-row | Covered | `test_paimon_merge_engine_matrix` | +| Paimon | Fixed bucket, dynamic bucket and cross-partition update | Covered | `test_paimon_merge_engine_matrix`, `test_paimon_partition_pk_delete_refs` | +| Paimon | Parquet/ORC and mixed-format reads; JNI/native parity | Covered | `test_paimon_merge_engine_matrix`, `paimon_tb_mix_format`, `test_paimon_cpp_reader` | +| Paimon | Snapshot/timestamp/tag/branch and incremental modes | Covered | `paimon_time_travel`, `paimon_incr_read`, `test_paimon_schema_time_travel_matrix` | +| Paimon | Schema evolution, partition-key restrictions and historical schema binding | Covered | `test_paimon_schema_time_travel_matrix`, `test_paimon_partition_mutation_atomicity` | +| Paimon | Deletion vectors, upsert/delete visibility and data/system tables | Covered | `test_paimon_deletion_vector`, `paimon_data_system_table`, `paimon_system_table` | +| Paimon | Catalog/database/table create and drop | Covered | `test_create_paimon_table` | +| Paimon | Doris data write-back | Negative boundary covered | `test_paimon_write_boundary` | +| Paimon | Failed CTAS metadata atomicity | Isolated known-bug regression | `test_paimon_ctas_atomicity_negative` | +| Iceberg | V1/V2/V3, Parquet/ORC, position/equality deletes and deletion vectors | Covered | `test_iceberg_position_delete`, `test_iceberg_equality_delete`, `test_iceberg_deletion_vector` | +| Iceberg | Schema, partition and sort-order evolution | Covered | `test_iceberg_schema_time_travel_matrix`, `test_iceberg_partition_evolution_format_scanner`, `iceberg_schema_change_ddl` | +| Iceberg | Snapshot/timestamp/tag/branch reads and reference actions | Covered | `test_iceberg_time_travel`, `iceberg_query_tag_branch`, `test_iceberg_schema_ref_actions_matrix` | +| Iceberg | INSERT, INSERT OVERWRITE, static/hybrid partition and CTAS | Covered | `write/test_iceberg_write_insert`, `write/test_iceberg_write_overwrite_evolution`, `write/test_iceberg_write_ctas_format_boundary` | +| Iceberg | DELETE, UPDATE and MERGE in supported MOR modes | Covered | `dml/test_iceberg_update_delete_advanced`, `dml/test_iceberg_merge_into_advanced`, `write/test_iceberg_write_dml_modes_evolution` | +| Iceberg | Distributed/concurrent commits, failure atomicity and Spark interoperability | Covered | `write/test_iceberg_write_concurrent_merge_invariants`, `write/test_iceberg_write_overwrite_atomicity` | +| Iceberg | System tables, views, caches and catalog variants | Covered | `test_iceberg_sys_table`, `test_iceberg_view_query_p0`, `test_iceberg_table_cache`, catalog-specific suites | +| Iceberg | COW row DML, tag writes and unsupported file writes | Negative boundary covered | `write/test_iceberg_write_dml_modes_evolution`, `write/test_iceberg_write_branch_dml_boundary`, `write/test_iceberg_write_ctas_format_boundary` | + +Detailed schema/time-travel and Iceberg write combinations are maintained in +`iceberg_paimon_schema_time_travel_coverage.md` and `iceberg/write/ICEBERG_WRITE_P0_COVERAGE.md`. + +## Added test design + +| Case | Goal | Risks | Dimension | Preconditions | Load | Expected | +| --- | --- | --- | --- | --- | --- | --- | +| PM01 | Distinguish all four primary-key merge engines | R01, R02, R04 | Functional, correctness, compatibility | Paimon Parquet/ORC tables | Duplicate keys across several commits | Each engine returns its documented merged row under automatic and forced-JNI routing | +| PM02 | Validate dynamic-bucket cross-partition deduplication | R03, R04 | Correctness | Primary key excludes partition key, bucket=-1 | Move one key between partitions | Exactly one current row remains in the new partition | +| PM03 | Preserve the Paimon read-only boundary | R05, R10 | Negative, atomicity | Existing Paimon PK table | VALUES, SELECT, OVERWRITE, UPDATE, DELETE, MERGE | Every statement fails before a snapshot or data change | +| PM04 | Reject or roll back Paimon CTAS atomically | R05, R10 | Isolated negative, atomicity | Paimon catalog with no target table | CREATE TABLE AS SELECT | Desired contract: the command fails and no target table remains; current bug leaves the table | + +Every P0 risk maps to at least one deterministic positive, boundary, or isolated known-bug +regression. Catalog authentication and cloud storage permutations stay in their existing connector +suites because they do not add format semantics to this matrix. diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_ctas_atomicity_negative.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_ctas_atomicity_negative.groovy new file mode 100644 index 00000000000000..6d449f4af1e1b4 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_ctas_atomicity_negative.groovy @@ -0,0 +1,74 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_ctas_atomicity_negative", + "p0,external,paimon,external_docker,external_docker_paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test") + return + } + + // CTAS currently creates Paimon metadata before discovering that no Paimon data sink exists. + // Keep this opt-in until the product rejects or rolls back the statement atomically. + String knownBugEnabled = context.config.otherConfigs.get("enablePaimonKnownBugTest") + if (knownBugEnabled == null || !knownBugEnabled.equalsIgnoreCase("true")) { + logger.info("skip isolated Paimon known-bug regression") + return + } + + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_paimon_ctas_atomicity_negative" + String dbName = "paimon_ctas_atomicity_negative_db" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + 'type'='paimon', + 'warehouse'='s3://warehouse/wh', + 's3.endpoint'='http://${externalEnvIp}:${minioPort}', + 's3.access_key'='admin', + 's3.secret_key'='password', + 's3.path.style.access'='true', + 'meta.cache.paimon.table.ttl-second'='0' + ) + """ + + try { + spark_paimon_multi """ + create database if not exists paimon.${dbName}; + drop table if exists paimon.${dbName}.ctas_target; + """ + + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + // A failed CTAS must not leave metadata that makes a retry fail with TABLE ALREADY EXISTS. + test { + sql """ + create table ctas_target engine=paimon + as select cast(1 as int) as id, cast('candidate' as string) as payload + """ + exception "PaimonExternalCatalog" + } + assertEquals(0, (sql """show tables like 'ctas_target'""").size()) + } finally { + spark_paimon """drop table if exists paimon.${dbName}.ctas_target""" + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_merge_engine_matrix.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_merge_engine_matrix.groovy new file mode 100644 index 00000000000000..266bc7c458b8ab --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_merge_engine_matrix.groovy @@ -0,0 +1,223 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_merge_engine_matrix", + "p0,external,paimon,external_docker,external_docker_paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test") + return + } + + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_paimon_merge_engine_matrix" + String dbName = "paimon_merge_engine_matrix_db" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + 'type'='paimon', + 'warehouse'='s3://warehouse/wh', + 's3.endpoint'='http://${externalEnvIp}:${minioPort}', + 's3.access_key'='admin', + 's3.secret_key'='password', + 's3.path.style.access'='true', + 'meta.cache.paimon.table.ttl-second'='0' + ) + """ + + try { + spark_paimon """create database if not exists paimon.${dbName}""" + + ["parquet", "orc"].each { String format -> + String deduplicateTable = "deduplicate_${format}" + String partialUpdateTable = "partial_update_${format}" + String aggregationTable = "aggregation_${format}" + String firstRowTable = "first_row_${format}" + + spark_paimon_multi """ + drop table if exists paimon.${dbName}.${deduplicateTable}; + create table paimon.${dbName}.${deduplicateTable} ( + id int, + score int, + note string + ) using paimon tblproperties ( + 'primary-key'='id', + 'bucket'='1', + 'merge-engine'='deduplicate', + 'file.format'='${format}' + ); + insert into paimon.${dbName}.${deduplicateTable} values + (1, 10, 'old-1'), + (2, 20, 'old-2'), + (3, 30, 'only-3'); + insert into paimon.${dbName}.${deduplicateTable} values + (1, 11, 'new-1'), + (2, 22, 'new-2'); + + drop table if exists paimon.${dbName}.${partialUpdateTable}; + create table paimon.${dbName}.${partialUpdateTable} ( + id int, + score int, + quantity int, + note string + ) using paimon tblproperties ( + 'primary-key'='id', + 'bucket'='1', + 'merge-engine'='partial-update', + 'file.format'='${format}' + ); + insert into paimon.${dbName}.${partialUpdateTable} values + (1, 10, 2, cast(null as string)), + (2, 30, cast(null as int), 'base-2'); + insert into paimon.${dbName}.${partialUpdateTable} values + (1, cast(null as int), cast(null as int), 'filled-1'), + (2, cast(null as int), 4, cast(null as string)); + insert into paimon.${dbName}.${partialUpdateTable} values + (1, 15, cast(null as int), cast(null as string)); + + drop table if exists paimon.${dbName}.${aggregationTable}; + create table paimon.${dbName}.${aggregationTable} ( + id int, + max_score int, + total bigint, + note string + ) using paimon tblproperties ( + 'primary-key'='id', + 'bucket'='1', + 'merge-engine'='aggregation', + 'fields.max_score.aggregate-function'='max', + 'fields.total.aggregate-function'='sum', + 'file.format'='${format}' + ); + insert into paimon.${dbName}.${aggregationTable} values + (1, 5, 10, 'old-1'), + (2, 9, 100, 'old-2'); + insert into paimon.${dbName}.${aggregationTable} values + (1, 7, 20, cast(null as string)), + (2, 8, 200, 'new-2'); + insert into paimon.${dbName}.${aggregationTable} values + (1, 6, 5, 'new-1'); + + drop table if exists paimon.${dbName}.${firstRowTable}; + create table paimon.${dbName}.${firstRowTable} ( + id int, + score int, + note string + ) using paimon tblproperties ( + 'primary-key'='id', + 'bucket'='1', + 'merge-engine'='first-row', + 'file.format'='${format}' + ); + insert into paimon.${dbName}.${firstRowTable} values + (1, 10, 'first-1'), + (2, 20, 'first-2'); + insert into paimon.${dbName}.${firstRowTable} values + (1, 99, 'ignored-1'), + (2, 88, 'ignored-2'), + (3, 30, 'first-3'); + """ + spark_paimon """ + call paimon.sys.compact( + table => '${dbName}.${firstRowTable}', + compact_strategy => 'full' + ) + """ + } + + // Dynamic buckets maintain a global key-to-partition mapping. Updating one key in a new + // partition must remove the old logical row instead of exposing both physical versions. + spark_paimon_multi """ + drop table if exists paimon.${dbName}.dynamic_cross_partition; + create table paimon.${dbName}.dynamic_cross_partition ( + id int, + part string, + score int + ) using paimon + partitioned by (part) + tblproperties ( + 'primary-key'='id', + 'bucket'='-1', + 'merge-engine'='deduplicate', + 'file.format'='parquet' + ); + insert into paimon.${dbName}.dynamic_cross_partition values + (1, 'old-part', 10), + (2, 'stable-part', 20); + insert into paimon.${dbName}.dynamic_cross_partition values + (1, 'new-part', 11); + """ + + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + [false, true].each { boolean forceJni -> + // MOR primary-key splits normally stay on Paimon's merge-aware reader. Keep both + // automatic routing and the explicit JNI override so a future routing change cannot + // silently bypass the merge-engine contract. + String reader = forceJni ? "forced_jni" : "auto" + sql """set force_jni_scanner=${forceJni}""" + + ["parquet", "orc"].each { String format -> + // PM-ME01: last-write-wins must merge overlapping LSM sorted runs. + "order_qt_${reader}_${format}_deduplicate" """ + select id, score, note from deduplicate_${format} order by id + """ + + // PM-ME02: NULL means "field not supplied" for partial-update, not "erase value". + "order_qt_${reader}_${format}_partial_update" """ + select id, score, quantity, note from partial_update_${format} order by id + """ + + // PM-ME03: aggregation functions apply across files while the default value field + // keeps last_non_null_value semantics. + "order_qt_${reader}_${format}_aggregation" """ + select id, max_score, total, note from aggregation_${format} order by id + """ + + // PM-ME04: first-row is intentionally different from deduplicate for duplicate keys. + "order_qt_${reader}_${format}_first_row" """ + select id, score, note from first_row_${format} order by id + """ + + "qt_${reader}_${format}_engine_aggregates" """ + select + (select sum(score) from deduplicate_${format}), + (select sum(score + quantity) from partial_update_${format}), + (select sum(max_score) from aggregation_${format}), + (select sum(score) from first_row_${format}) + """ + } + + // PM-DD01: cross-partition deduplication must expose one current row per primary key. + "order_qt_${reader}_dynamic_cross_partition" """ + select id, part, score from dynamic_cross_partition order by id + """ + "qt_${reader}_dynamic_old_partition" """ + select count(*) from dynamic_cross_partition where part = 'old-part' + """ + "qt_${reader}_dynamic_new_partition" """ + select count(*) from dynamic_cross_partition where part = 'new-part' + """ + } + } finally { + sql """set force_jni_scanner=false""" + sql """drop catalog if exists ${catalogName}""" + } +} diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_write_boundary.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_write_boundary.groovy new file mode 100644 index 00000000000000..2f41229debcc17 --- /dev/null +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_write_boundary.groovy @@ -0,0 +1,107 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_paimon_write_boundary", + "p0,external,paimon,external_docker,external_docker_paimon") { + String enabled = context.config.otherConfigs.get("enablePaimonTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable paimon test") + return + } + + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_paimon_write_boundary" + String dbName = "paimon_write_boundary_db" + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + 'type'='paimon', + 'warehouse'='s3://warehouse/wh', + 's3.endpoint'='http://${externalEnvIp}:${minioPort}', + 's3.access_key'='admin', + 's3.secret_key'='password', + 's3.path.style.access'='true', + 'meta.cache.paimon.table.ttl-second'='0' + ) + """ + + try { + spark_paimon_multi """ + create database if not exists paimon.${dbName}; + drop table if exists paimon.${dbName}.write_boundary; + create table paimon.${dbName}.write_boundary ( + id int, + score int, + note string + ) using paimon tblproperties ( + 'primary-key'='id', + 'bucket'='1', + 'file.format'='parquet' + ); + insert into paimon.${dbName}.write_boundary values + (1, 10, 'base-1'), + (2, 20, 'base-2'); + """ + + sql """switch ${catalogName}""" + sql """use ${dbName}""" + + qt_before_rows """select id, score, note from write_boundary order by id""" + qt_before_snapshots """select count(*) from write_boundary\$snapshots""" + + // WB01-WB06 preserve the documented data-write boundary at analysis time. The source table + // and its snapshot list must stay unchanged after every rejected write shape. + test { + sql """insert into write_boundary values (3, 30, 'insert-values')""" + exception "PaimonExternalCatalog" + } + test { + sql """insert into write_boundary select 3, 30, 'insert-select'""" + exception "PaimonExternalCatalog" + } + test { + sql """insert overwrite table write_boundary values (3, 30, 'overwrite')""" + exception "PaimonExternalCatalog" + } + test { + sql """update write_boundary set score = score + 1 where id = 1""" + exception "target table in update command should be an olapTable" + } + test { + sql """delete from write_boundary where id = 1""" + exception "delete command could be only used on olap table" + } + test { + sql """ + merge into write_boundary target + using (select 1 as id, 99 as score, 'merge' as note) source + on target.id = source.id + when matched then update set score = source.score, note = source.note + when not matched then insert (id, score, note) + values (source.id, source.score, source.note) + """ + exception "merge into command only support MOW unique key olapTable" + } + + sql """refresh table write_boundary""" + qt_after_rows """select id, score, note from write_boundary order by id""" + qt_after_snapshots """select count(*) from write_boundary\$snapshots""" + } finally { + sql """drop catalog if exists ${catalogName}""" + } +} From 5a280b2c285c4e6638ebc00a2d84a40601af6910 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 28 Jul 2026 09:55:52 +0800 Subject: [PATCH 3/5] [feature](paimon) Support query-level dynamic options (#65984) Paimon scans need a query-scoped way to pass dynamic table options to the Paimon SDK. This is especially useful for system tables such as `$files`, where options like `scan.snapshot-id` determine which snapshot is materialized. This PR: - adds `options` to Doris table scan parameters; - supports the existing relation parameter syntax `@options(...)` for both ordinary Paimon data tables and Paimon system tables; - applies query options through `Table.copy(options)`, so the cached base table is not mutated; - serializes the option-bearing table copy to the BE JNI reader, where lazy system-table splits are materialized; - keeps `@incr(...)` for incremental range reads. Paimon validates option names and values. Paimon system-table time travel with `FOR VERSION/TIME AS OF` and unsupported scan parameter types remain rejected. Query a Paimon system table at one snapshot: ```sql SELECT `partition`, bucket, file_path, file_format, schema_id, level, record_count, file_size_in_bytes, min_sequence_number, max_sequence_number, creation_time FROM paimon_catalog.db.`orders$files` @options('scan.snapshot-id'='12345'); ``` The same syntax works for ordinary data tables: ```sql SELECT * FROM paimon_catalog.db.orders @options( 'scan.mode'='from-snapshot', 'scan.snapshot-id'='12345' ); ``` Options are scoped to each relation, so different aliases can use different snapshots: ```sql SELECT left_orders.id, right_orders.id FROM paimon_catalog.db.orders @options('scan.snapshot-id'='1') left_orders JOIN paimon_catalog.db.orders @options('scan.snapshot-id'='2') right_orders ON left_orders.id = right_orders.id; ``` For incremental changes, use `@incr`: ```sql SELECT * FROM paimon_catalog.db.`orders$snapshots` @incr('startSnapshotId'=1, 'endSnapshotId'=2); ``` `@incr` maps the range to Paimon's `incremental-between` option. In contrast, `scan.snapshot-id` selects one table snapshot. A relation accepts one scan-parameter clause, so `@incr` and `@options` cannot be combined on the same relation. 1. Catalog `paimon.table-default.*` properties provide defaults when a table is created. 2. Explicit table properties take precedence over those catalog creation defaults. 3. Query-level `@options(...)` values override the same keys on the Paimon table copy for that query. 4. The cached table is not modified, so options do not leak to later queries. Support query-level Paimon dynamic options through the table scan parameter syntax `@options(...)` for data tables and system tables. - Test: - Unit Test - Regression test - Behavior changed: Yes. Paimon data-table and system-table scans can pass query-scoped options to the Paimon SDK. - Does this need documentation: Yes - FE build passed. - `NereidsParserTest`, `PaimonScanNodeTest`, and `AbstractPaimonPropertiesTest`: 98 tests passed, 0 failures, 0 errors. - Local regression: - `test_paimon_incr_read`: 1 suite passed, 0 failed, 0 fatal, 0 skipped. - `paimon_system_table`: 1 suite passed, 0 failed, 0 fatal, 0 skipped. - Data-table coverage includes native and JNI scanners, snapshots 1/2/3, relation-scoped aliases, query isolation, and invalid snapshot handling. - Multi-snapshot `$files` verification: - latest snapshot: 3 files; - `scan.snapshot-id=1`: 1 file; - `scan.snapshot-id=2`: 2 files; - `scan.snapshot-id=3`: 3 files. - Performance comparison on a Paimon table with 201 snapshots and 10,001 current files, using 10 alternating runs after warm-up: - latest `$files` mean: 5.704s; - `scan.snapshot-id=1` mean: 1.443s; - 3.95x faster, with 74.7% lower elapsed time. --- .../doris/analysis/TableScanParams.java | 46 +- .../doris/datasource/FileQueryScanNode.java | 22 +- .../PaimonLatestSnapshotProjectionLoader.java | 8 +- .../paimon/PaimonExternalTable.java | 28 +- .../datasource/paimon/PaimonScanParams.java | 391 ++++++++++++++ .../paimon/PaimonSysExternalTable.java | 24 + .../doris/datasource/paimon/PaimonUtil.java | 41 +- .../paimon/source/PaimonScanNode.java | 137 ++++- .../paimon/source/PaimonSource.java | 12 + .../nereids/ExternalTablePreloadInfo.java | 3 +- .../doris/nereids/StatementContext.java | 9 +- .../nereids/analyzer/UnboundRelation.java | 8 +- .../nereids/parser/LogicalPlanBuilder.java | 6 + .../nereids/rules/analysis/BindRelation.java | 42 ++ .../exploration/mv/MaterializedViewUtils.java | 9 +- .../trees/plans/commands/ExecuteCommand.java | 25 + .../trees/plans/logical/LogicalFileScan.java | 31 +- .../paimon/PaimonExternalMetaCacheTest.java | 41 +- .../paimon/PaimonExternalTableTest.java | 97 ++++ .../paimon/PaimonScanParamsTest.java | 355 +++++++++++++ .../datasource/paimon/PaimonUtilTest.java | 15 + .../paimon/source/PaimonScanNodeTest.java | 492 +++++++++++++++++- .../AbstractPaimonPropertiesTest.java | 14 + .../doris/nereids/StatementContextTest.java | 61 ++- .../nereids/parser/NereidsParserTest.java | 117 +++++ .../rules/analysis/BindRelationTest.java | 22 + .../mv/MaterializedViewUtilsTest.java | 19 + .../plans/commands/ExecuteCommandTest.java | 99 ++++ .../plans/logical/LogicalFileScanTest.java | 24 + .../paimon/paimon_data_system_table.out | 10 +- .../paimon/paimon_incr_read.out | 40 ++ .../paimon/paimon_system_table.out | 9 + .../paimon/paimon_time_travel.out | 6 + .../test_paimon_schema_time_travel_matrix.out | 27 + .../paimon/paimon_data_system_table.groovy | 21 + .../paimon/paimon_incr_read.groovy | 57 +- .../paimon/paimon_system_table.groovy | 71 ++- .../paimon/paimon_time_travel.groovy | 8 + ...st_paimon_schema_time_travel_matrix.groovy | 150 ++++++ 39 files changed, 2512 insertions(+), 85 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonScanParams.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonScanParamsTest.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommandTest.java create mode 100644 regression-test/data/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.out diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableScanParams.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableScanParams.java index a73b208a6112a7..08743f31c84f58 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableScanParams.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableScanParams.java @@ -17,23 +17,27 @@ package org.apache.doris.analysis; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import org.bouncycastle.util.Strings; import java.util.List; import java.util.Map; +import java.util.Optional; +import java.util.function.Function; public class TableScanParams { public static final String PARAMS_NAME = "name"; public static final String INCREMENTAL_READ = "incr"; public static final String BRANCH = "branch"; public static final String TAG = "tag"; + public static final String OPTIONS = "options"; private static final ImmutableSet VALID_PARAM_TYPES = ImmutableSet.of( INCREMENTAL_READ, BRANCH, - TAG); - + TAG, + OPTIONS); private final String paramType; // There are two ways to pass parameters to a function. // - One is in map form, where the data is stored in `mapParams`. @@ -42,18 +46,21 @@ public class TableScanParams { // such as: `listParams` is used for @func_name('value1', 'value2', 'value3') private final Map mapParams; private final List listParams; + private volatile Map resolvedMapParams; private void validate() { if (!VALID_PARAM_TYPES.contains(paramType)) { throw new IllegalArgumentException("Invalid param type: " + paramType); } - // TODO: validate mapParams and listParams for different param types + if (OPTIONS.equals(paramType) && (mapParams.isEmpty() || !listParams.isEmpty())) { + throw new IllegalArgumentException("OPTIONS requires a non-empty key/value map"); + } } public TableScanParams(String paramType, Map mapParams, List listParams) { this.paramType = Strings.toLowerCase(paramType); this.mapParams = mapParams == null ? ImmutableMap.of() : ImmutableMap.copyOf(mapParams); - this.listParams = listParams; + this.listParams = listParams == null ? ImmutableList.of() : ImmutableList.copyOf(listParams); validate(); } @@ -69,6 +76,34 @@ public Map getMapParams() { return mapParams; } + /** + * Resolve dynamic parameters once for the current statement execution. + */ + public Map getOrResolveMapParams( + Function, Map> resolver) { + Map resolved = resolvedMapParams; + if (resolved == null) { + synchronized (this) { + resolved = resolvedMapParams; + if (resolved == null) { + // Binding and scan initialization share this instance. Caching the resolved + // map prevents a mutable external selector from choosing two snapshots. + resolved = ImmutableMap.copyOf(resolver.apply(mapParams)); + resolvedMapParams = resolved; + } + } + } + return resolved; + } + + public Optional> getResolvedMapParams() { + return Optional.ofNullable(resolvedMapParams); + } + + public synchronized void resetResolvedMapParams() { + resolvedMapParams = null; + } + public boolean incrementalRead() { return INCREMENTAL_READ.equals(paramType); } @@ -80,4 +115,7 @@ public boolean isBranch() { public boolean isTag() { return TAG.equals(paramType); } + public boolean isOptions() { + return OPTIONS.equals(paramType); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java index bfe34e9a32325e..1cc71f863d27af 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java @@ -83,6 +83,7 @@ import java.util.Optional; import java.util.Set; import java.util.concurrent.atomic.AtomicReference; +import java.util.stream.Collectors; /** * FileQueryScanNode for querying the file access type of catalog, now only support @@ -281,10 +282,10 @@ private void setColumnPositionMapping() } // Pre-index columns into a Map for O(1) lookup - List columns = desc.getTable().getFullSchema(); - Map columnNameMap = new HashMap<>(columns.size()); - for (int i = 0; i < columns.size(); i++) { - columnNameMap.putIfAbsent(columns.get(i).getName(), i); + List columnNames = getFileColumnNames(); + Map columnNameMap = new HashMap<>(columnNames.size()); + for (int i = 0; i < columnNames.size(); i++) { + columnNameMap.putIfAbsent(columnNames.get(i), i); } for (TFileScanSlotInfo slot : params.getRequiredSlots()) { @@ -306,6 +307,12 @@ private void setColumnPositionMapping() params.setColumnIdxs(columnIdxs); } + protected List getFileColumnNames() { + return desc.getTable().getFullSchema().stream() + .map(Column::getName) + .collect(Collectors.toList()); + } + public TFileScanRangeParams getFileScanRangeParams() { return params; } @@ -714,7 +721,8 @@ public void setQueryTableSnapshot(TableSnapshot tableSnapshot) { } public TableSnapshot getQueryTableSnapshot() { - TableSnapshot snapshot = desc.getRef().getTableSnapShot(); + // Keep explicit snapshots available when a legacy or unit-built descriptor has no ref. + TableSnapshot snapshot = desc.getRef() == null ? null : desc.getRef().getTableSnapShot(); if (snapshot != null) { return snapshot; } @@ -726,7 +734,9 @@ public void setScanParams(TableScanParams scanParams) { } public TableScanParams getScanParams() { - TableScanParams scan = desc.getRef().getScanParams(); + // Unit-built and legacy descriptors may not carry a relation reference; explicit scan + // parameters must remain usable for those callers. + TableScanParams scan = desc.getRef() == null ? null : desc.getRef().getScanParams(); if (scan != null) { return scan; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java index 8e2c7a73b33901..2dc8c52165be59 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/metacache/paimon/PaimonLatestSnapshotProjectionLoader.java @@ -29,6 +29,7 @@ import org.apache.paimon.Snapshot; import org.apache.paimon.schema.TableSchema; import org.apache.paimon.table.DataTable; +import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; import java.util.Collections; @@ -73,8 +74,11 @@ private PaimonSnapshot resolveLatestSnapshot(Table paimonTable) { Optional optionalSnapshot = paimonTable.latestSnapshot(); if (optionalSnapshot.isPresent()) { latestSnapshotId = optionalSnapshot.get().id(); - snapshotTable = paimonTable.copy( - Collections.singletonMap(CoreOptions.SCAN_SNAPSHOT_ID.key(), String.valueOf(latestSnapshotId))); + // A schema-only change can be newer than the latest data snapshot. Preserve the + // snapshot pin while exposing the current schema so added columns read as null. + snapshotTable = ((FileStoreTable) paimonTable.copy( + Collections.singletonMap(CoreOptions.SCAN_SNAPSHOT_ID.key(), String.valueOf(latestSnapshotId)))) + .copyWithLatestSchema(); } DataTable dataTable = (DataTable) paimonTable; long latestSchemaId = dataTable.schemaManager().latest().map(TableSchema::id).orElse(0L); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java index 6a744f765e8e2f..6aeacc9058d115 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonExternalTable.java @@ -106,6 +106,32 @@ public Table getPaimonTable(Optional snapshot) { } } + public Table getPaimonTable(TableScanParams scanParams) { + if (scanParams != null && scanParams.isOptions()) { + Map options = scanParams.getMapParams(); + Table statementTable = getPaimonTable(MvccUtil.getSnapshotFromContext(this)); + Table resolutionTable = PaimonScanParams.usesStatementSnapshot(options) + ? statementTable + : getBasePaimonTable(); + Map resolvedOptions = scanParams.getOrResolveMapParams( + relationOptions -> PaimonScanParams.resolveOptions(resolutionTable, relationOptions)); + // Startup options are normalized to an immutable snapshot before schema binding. The + // scan phase reuses that exact resolution instead of consulting a mutable tag or clock. + Table table = PaimonScanParams.selectsSchema(resolvedOptions) + ? getBasePaimonTable() + : statementTable; + return PaimonScanParams.applyOptions(table, resolvedOptions); + } + return getPaimonTable(MvccUtil.getSnapshotFromContext(this)); + } + + public List getFullSchema(TableScanParams scanParams) { + Table table = getPaimonTable(scanParams); + return PaimonUtil.parseSchema(table, + getCatalog().getEnableMappingVarbinary(), + getCatalog().getEnableMappingTimestampTz()); + } + private PaimonSnapshotCacheValue getPaimonSnapshotCacheValue(Optional tableSnapshot, Optional scanParams) { makeSureInitialized(); @@ -423,7 +449,7 @@ public boolean isPartitionedTable() { return !getBasePaimonTable().partitionKeys().isEmpty(); } - private Table getBasePaimonTable() { + Table getBasePaimonTable() { return PaimonUtils.getPaimonTable(this); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonScanParams.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonScanParams.java new file mode 100644 index 00000000000000..1732b6ca90e952 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonScanParams.java @@ -0,0 +1,391 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.analysis.TableScanParams; + +import com.google.common.collect.ImmutableSet; +import org.apache.paimon.CoreOptions; +import org.apache.paimon.Snapshot; +import org.apache.paimon.options.ConfigOption; +import org.apache.paimon.options.FallbackKey; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.source.snapshot.FullCompactedStartingScanner; +import org.apache.paimon.table.source.snapshot.TimeTravelUtil; + +import java.util.Arrays; +import java.util.HashMap; +import java.util.Locale; +import java.util.Map; +import java.util.Optional; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Validation and application rules for relation-scoped Paimon scan parameters. + */ +public final class PaimonScanParams { + private static final String PINNED_FILE_CREATION_TIME = + "doris.internal.paimon.file-creation-time-millis"; + private static final String PINNED_EMPTY_SCAN = "doris.internal.paimon.empty-scan"; + + private static final Set QUERY_OPTION_KEYS = ImmutableSet.of( + CoreOptions.SCAN_MODE.key(), + CoreOptions.SCAN_TIMESTAMP.key(), + CoreOptions.SCAN_TIMESTAMP_MILLIS.key(), + CoreOptions.SCAN_WATERMARK.key(), + CoreOptions.SCAN_FILE_CREATION_TIME_MILLIS.key(), + CoreOptions.SCAN_CREATION_TIME_MILLIS.key(), + CoreOptions.SCAN_SNAPSHOT_ID.key(), + CoreOptions.SCAN_TAG_NAME.key(), + CoreOptions.SCAN_VERSION.key(), + CoreOptions.SCAN_MANIFEST_PARALLELISM.key(), + CoreOptions.SCAN_PLAN_SORT_PARTITION.key()); + + private static final Set STARTUP_POSITION_KEYS = ImmutableSet.of( + CoreOptions.SCAN_TIMESTAMP.key(), + CoreOptions.SCAN_TIMESTAMP_MILLIS.key(), + CoreOptions.SCAN_WATERMARK.key(), + CoreOptions.SCAN_FILE_CREATION_TIME_MILLIS.key(), + CoreOptions.SCAN_CREATION_TIME_MILLIS.key(), + CoreOptions.SCAN_SNAPSHOT_ID.key(), + CoreOptions.SCAN_TAG_NAME.key(), + CoreOptions.SCAN_VERSION.key()); + + private static final Set INHERITED_READ_STATE_KEYS = inheritedReadStateKeys(); + + // FilesScan enumerates the latest partitions before applying its range-aware per-partition scan, + // so it cannot safely read a range when a partition in that range has since been dropped. + private static final Set INCREMENTAL_SYSTEM_TABLES = ImmutableSet.of( + "audit_log", "binlog", "partitions", "ro", "row_tracking"); + + private static final Set PAIMON_READER_SYSTEM_TABLES = ImmutableSet.of( + "audit_log", "binlog", "row_tracking"); + + private static final Set OPTIONS_SYSTEM_TABLES = ImmutableSet.of( + // A system table may advertise OPTIONS only when every row-producing stage observes + // the selected snapshot; files and buckets still consult latest metadata internally. + "audit_log", "binlog", "manifests", "partitions", "ro", + "row_tracking", "table_indexes"); + + private PaimonScanParams() { + } + + public static void validateOptions(Map options) { + if (options.containsKey(CoreOptions.SCAN_FALLBACK_BRANCH.key())) { + throw new IllegalArgumentException("Paimon query option '" + + CoreOptions.SCAN_FALLBACK_BRANCH.key() + + "' is not supported because it requires rebuilding the table through the catalog factory."); + } + + Set unsupported = options.keySet().stream() + .filter(key -> !QUERY_OPTION_KEYS.contains(key)) + .collect(Collectors.toSet()); + if (!unsupported.isEmpty()) { + throw new IllegalArgumentException("Unsupported Paimon query option(s): " + unsupported); + } + + String scanMode = options.get(CoreOptions.SCAN_MODE.key()); + if ("from-creation-timestamp".equalsIgnoreCase(scanMode) + && options.get(CoreOptions.SCAN_CREATION_TIME_MILLIS.key()) == null) { + // Paimon 1.3.1 does not validate this newer mode, but its starting scanner + // requires the creation timestamp and otherwise fails after analysis. + throw new IllegalArgumentException("Paimon scan mode 'from-creation-timestamp' requires query option '" + + CoreOptions.SCAN_CREATION_TIME_MILLIS.key() + "'."); + } + + long positionCount = options.keySet().stream().filter(STARTUP_POSITION_KEYS::contains).count(); + if (positionCount > 1) { + throw new IllegalArgumentException( + "Only one Paimon startup position can be specified: " + STARTUP_POSITION_KEYS); + } + + if (options.containsKey(CoreOptions.SCAN_MODE.key()) && positionCount == 1) { + String position = options.keySet().stream() + .filter(STARTUP_POSITION_KEYS::contains) + .findFirst() + .get(); + String mode = scanMode.toLowerCase(Locale.ROOT); + if (!isCompatibleStartupMode(position, mode)) { + throw new IllegalArgumentException("Paimon scan mode '" + mode + + "' is incompatible with startup position '" + position + "'."); + } + } + } + + public static Table applyOptions(Table table, Map options) { + Map tableOptions = userOptions(options); + validateOptions(tableOptions); + Map isolatedOptions = new HashMap<>(tableOptions); + if (hasStartupOptions(tableOptions)) { + // Startup mode, position, and range form one inherited state family in Paimon. Clear + // absent members so one relation cannot accidentally reuse another relation's read state. + INHERITED_READ_STATE_KEYS.stream() + .filter(key -> !tableOptions.containsKey(key)) + .forEach(key -> isolatedOptions.put(key, null)); + } + return table.copy(isolatedOptions); + } + + private static Set inheritedReadStateKeys() { + ImmutableSet.Builder keys = ImmutableSet.builder(); + for (ConfigOption option : Arrays.asList( + CoreOptions.SCAN_TIMESTAMP, + CoreOptions.SCAN_TIMESTAMP_MILLIS, + CoreOptions.SCAN_WATERMARK, + CoreOptions.SCAN_FILE_CREATION_TIME_MILLIS, + CoreOptions.SCAN_CREATION_TIME_MILLIS, + CoreOptions.SCAN_SNAPSHOT_ID, + CoreOptions.SCAN_TAG_NAME, + CoreOptions.SCAN_VERSION, + CoreOptions.SCAN_MODE, + CoreOptions.SCAN_BOUNDED_WATERMARK, + CoreOptions.INCREMENTAL_BETWEEN, + CoreOptions.INCREMENTAL_BETWEEN_TIMESTAMP, + CoreOptions.INCREMENTAL_BETWEEN_SCAN_MODE, + CoreOptions.INCREMENTAL_TO_AUTO_TAG)) { + keys.add(option.key()); + for (FallbackKey fallbackKey : option.fallbackKeys()) { + keys.add(fallbackKey.getKey()); + } + } + return keys.build(); + } + + public static boolean hasStartupOptions(Map options) { + return options.containsKey(CoreOptions.SCAN_MODE.key()) + || options.keySet().stream().anyMatch(STARTUP_POSITION_KEYS::contains); + } + + public static boolean selectsSchema(Map options) { + return hasStartupOptions(options); + } + + public static boolean usesStatementSnapshot(Map options) { + if (options.keySet().stream().anyMatch(STARTUP_POSITION_KEYS::contains)) { + return false; + } + String mode = options.get(CoreOptions.SCAN_MODE.key()); + return mode == null + || "default".equalsIgnoreCase(mode) + || "latest".equalsIgnoreCase(mode) + || "latest-full".equalsIgnoreCase(mode) + || "full".equalsIgnoreCase(mode); + } + + public static Map resolveOptions(Table table, Map options) { + validateOptions(options); + if (!hasStartupOptions(options)) { + return options; + } + if (usesStatementSnapshot(options)) { + String pinnedSnapshotId = table.options().get(CoreOptions.SCAN_SNAPSHOT_ID.key()); + if (pinnedSnapshotId != null) { + return resolvedSnapshotOptions(options, pinnedSnapshotId); + } + } + if (!(table instanceof FileStoreTable)) { + throw new IllegalArgumentException("Paimon startup options require a file-store data table."); + } + FileStoreTable fileStoreTable = (FileStoreTable) table; + if (options.containsKey(CoreOptions.SCAN_FILE_CREATION_TIME_MILLIS.key())) { + return resolveFileCreationTime( + options, + fileStoreTable, + Long.parseLong(options.get(CoreOptions.SCAN_FILE_CREATION_TIME_MILLIS.key()))); + } + if (options.containsKey(CoreOptions.SCAN_CREATION_TIME_MILLIS.key())) { + long creationTime = Long.parseLong(options.get(CoreOptions.SCAN_CREATION_TIME_MILLIS.key())); + Long previousSnapshotId = TimeTravelUtil.earlierThanTimeMills( + fileStoreTable.snapshotManager(), + fileStoreTable.changelogManager(), + creationTime, + fileStoreTable.coreOptions().changelogLifecycleDecoupled(), + true); + if (previousSnapshotId != null + && fileStoreTable.snapshotManager().snapshotExists(previousSnapshotId + 1)) { + return resolvedSnapshotOptions(options, String.valueOf(previousSnapshotId + 1)); + } + return resolveFileCreationTime(options, fileStoreTable, creationTime); + } + + Table selectedTable = applyOptions(table, options); + if ("compacted-full".equalsIgnoreCase(options.get(CoreOptions.SCAN_MODE.key()))) { + Long snapshotId = compactedFullSnapshotId((FileStoreTable) selectedTable); + return snapshotId == null + ? resolvedEmptyOptions(options) + : resolvedSnapshotOptions(options, String.valueOf(snapshotId)); + } + Snapshot snapshot = TimeTravelUtil.tryTravelOrLatest((FileStoreTable) selectedTable); + if (snapshot == null) { + return resolvedEmptyOptions(options); + } + String tagName = selectedTagName(options, (FileStoreTable) selectedTable); + if (tagName != null) { + // A tag owns a retained Snapshot copy even after the ordinary snapshot file expires. + // Keep the canonical tag selector so planning reads that retained metadata path. + return resolvedTagOptions(options, tagName); + } + return resolvedSnapshotOptions(options, String.valueOf(snapshot.id())); + } + + private static String selectedTagName(Map options, FileStoreTable selectedTable) { + String tagName = options.get(CoreOptions.SCAN_TAG_NAME.key()); + if (tagName != null) { + return tagName; + } + // Paimon normalizes a tag-valued scan.version while copying the selected table. + return selectedTable.options().get(CoreOptions.SCAN_TAG_NAME.key()); + } + + private static Long compactedFullSnapshotId(FileStoreTable table) { + CoreOptions coreOptions = table.coreOptions(); + int deltaCommits = coreOptions.toConfiguration() + .getOptional(CoreOptions.FULL_COMPACTION_DELTA_COMMITS) + .orElse(1); + if (coreOptions.changelogProducer() == CoreOptions.ChangelogProducer.FULL_COMPACTION + || coreOptions.toConfiguration().contains(CoreOptions.FULL_COMPACTION_DELTA_COMMITS)) { + return table.snapshotManager().pickOrLatest(snapshot -> + snapshot.commitKind() == Snapshot.CommitKind.COMPACT + && FullCompactedStartingScanner.isFullCompactedIdentifier( + snapshot.commitIdentifier(), deltaCommits)); + } + // COMPACTED_FULL means the newest compact snapshot, which can be older than latest. + return table.snapshotManager().pickOrLatest( + snapshot -> snapshot.commitKind() == Snapshot.CommitKind.COMPACT); + } + + private static Map resolveFileCreationTime( + Map options, FileStoreTable table, long creationTime) { + Map resolved = new HashMap<>(options); + INHERITED_READ_STATE_KEYS.forEach(resolved::remove); + Optional latestSnapshot = table.latestSnapshot(); + if (latestSnapshot.isPresent()) { + resolved.put(CoreOptions.SCAN_SNAPSHOT_ID.key(), + String.valueOf(latestSnapshot.get().id())); + } else { + resolved.put(PINNED_EMPTY_SCAN, Boolean.TRUE.toString()); + } + // Paimon's file-creation scanner consults latest lazily. Keep its filter as internal + // metadata while replacing the live latest lookup with the snapshot fixed above. + resolved.put(PINNED_FILE_CREATION_TIME, String.valueOf(creationTime)); + return resolved; + } + + private static Map resolvedSnapshotOptions( + Map options, String snapshotId) { + Map resolved = new HashMap<>(options); + INHERITED_READ_STATE_KEYS.forEach(resolved::remove); + resolved.put(CoreOptions.SCAN_SNAPSHOT_ID.key(), snapshotId); + return resolved; + } + + private static Map resolvedTagOptions(Map options, String tagName) { + Map resolved = new HashMap<>(options); + INHERITED_READ_STATE_KEYS.forEach(resolved::remove); + resolved.put(CoreOptions.SCAN_TAG_NAME.key(), tagName); + return resolved; + } + + private static Map resolvedEmptyOptions(Map options) { + Map resolved = new HashMap<>(options); + INHERITED_READ_STATE_KEYS.forEach(resolved::remove); + // An empty table is also a statement state. Remember it explicitly so a commit between + // binding and split planning cannot turn the relation into a non-empty scan. + resolved.put(PINNED_EMPTY_SCAN, Boolean.TRUE.toString()); + return resolved; + } + + private static Map userOptions(Map options) { + return options.entrySet().stream() + .filter(entry -> !entry.getKey().startsWith("doris.internal.paimon.")) + .collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + } + + public static Optional getPinnedFileCreationTime(Map options) { + return Optional.ofNullable(options.get(PINNED_FILE_CREATION_TIME)).map(Long::parseLong); + } + + public static boolean isPinnedEmptyScan(Map options) { + return Boolean.parseBoolean(options.get(PINNED_EMPTY_SCAN)); + } + + public static Map isolateIncrementalRead(Map incrementalOptions) { + Map isolatedOptions = new HashMap<>(); + INHERITED_READ_STATE_KEYS.forEach(key -> isolatedOptions.put(key, null)); + isolatedOptions.putAll(incrementalOptions); + return isolatedOptions; + } + + private static boolean isCompatibleStartupMode(String position, String mode) { + if ("default".equals(mode)) { + return true; + } + if (CoreOptions.SCAN_TIMESTAMP.key().equals(position) + || CoreOptions.SCAN_TIMESTAMP_MILLIS.key().equals(position)) { + return "from-timestamp".equals(mode); + } + if (CoreOptions.SCAN_FILE_CREATION_TIME_MILLIS.key().equals(position)) { + return "from-file-creation-time".equals(mode); + } + if (CoreOptions.SCAN_CREATION_TIME_MILLIS.key().equals(position)) { + return "from-creation-timestamp".equals(mode); + } + return "from-snapshot".equals(mode) + || (CoreOptions.SCAN_SNAPSHOT_ID.key().equals(position) + && "from-snapshot-full".equals(mode)); + } + + public static boolean supportsIncrementalRead(String systemTableType) { + return systemTableType != null && INCREMENTAL_SYSTEM_TABLES.contains(systemTableType.toLowerCase()); + } + + public static boolean supportsOptions(String systemTableType) { + return systemTableType != null && OPTIONS_SYSTEM_TABLES.contains(systemTableType.toLowerCase()); + } + + public static boolean requiresPaimonReader(String systemTableType) { + // Range support and reader requirements are independent capabilities. File-backed system + // tables can honor INCR while still using Doris' native reader. Unknown mocked or legacy + // system-table types retain the previous behavior and do not force the Paimon reader. + return systemTableType != null && PAIMON_READER_SYSTEM_TABLES.contains(systemTableType.toLowerCase()); + } + + public static void validateSystemTable(String systemTableType, TableScanParams scanParams) { + if (scanParams == null) { + return; + } + if (scanParams.incrementalRead() && !supportsIncrementalRead(systemTableType)) { + throw new IllegalArgumentException( + "Paimon system table '" + systemTableType + "' does not support INCR scan params."); + } + if (scanParams.isOptions() && !supportsOptions(systemTableType)) { + throw new IllegalArgumentException( + "Paimon system table '" + systemTableType + "' does not support OPTIONS scan params."); + } + if (scanParams.isOptions() + && scanParams.getMapParams().containsKey(CoreOptions.SCAN_FILE_CREATION_TIME_MILLIS.key())) { + throw new IllegalArgumentException( + "Paimon system tables do not support scan.file-creation-time-millis OPTIONS."); + } + if (!scanParams.incrementalRead() && !scanParams.isOptions()) { + throw new IllegalArgumentException("Paimon system tables only support INCR or OPTIONS scan params."); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSysExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSysExternalTable.java index 59e5a946e8e392..2fe95ab2c14c70 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSysExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonSysExternalTable.java @@ -17,6 +17,7 @@ package org.apache.doris.datasource.paimon; +import org.apache.doris.analysis.TableScanParams; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.TableIf; import org.apache.doris.datasource.ExternalTable; @@ -133,6 +134,29 @@ public Table getSysPaimonTable() { return paimonSysTable; } + public Table getSysPaimonTable(TableScanParams scanParams) { + Table table = getSysPaimonTable(); + if (scanParams == null || !scanParams.isOptions()) { + return table; + } + Map resolvedOptions = scanParams.getOrResolveMapParams( + options -> PaimonScanParams.resolveOptions(sourceTable.getBasePaimonTable(), options)); + if (PaimonScanParams.getPinnedFileCreationTime(resolvedOptions).isPresent()) { + // Generic system-table wrappers cannot carry Paimon's manifest-entry predicate. + // Reject the fallback instead of silently widening it to the whole pinned snapshot. + throw new IllegalArgumentException( + "Paimon system tables cannot apply a creation-time file filter."); + } + return PaimonScanParams.applyOptions(table, resolvedOptions); + } + + public List getFullSchema(TableScanParams scanParams) { + Table table = getSysPaimonTable(scanParams); + return PaimonUtil.parseSchema(table, + getCatalog().getEnableMappingVarbinary(), + getCatalog().getEnableMappingTimestampTz()); + } + /** * Returns the schema of the system table. * The schema is derived from the system table's rowType. diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java index ea49e80a5b9478..0ec548992bd129 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/PaimonUtil.java @@ -349,6 +349,39 @@ public static void updatePaimonColumnUniqueId(Column column, DataField field) { updatePaimonColumnUniqueId(column, field.type()); } + public static void updatePaimonColumnMetadata(Column column, DataField field) { + updatePaimonColumnUniqueId(column, field); + updatePaimonColumnTimezone(column, field.type()); + } + + private static void updatePaimonColumnTimezone(Column column, DataType dataType) { + if (dataType.getTypeRoot() == org.apache.paimon.types.DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE) { + column.setWithTZExtraInfo(); + } + List children = column.getChildren(); + if (children == null) { + return; + } + switch (dataType.getTypeRoot()) { + case ARRAY: + updatePaimonColumnTimezone(children.get(0), ((ArrayType) dataType).getElementType()); + break; + case MAP: + MapType mapType = (MapType) dataType; + updatePaimonColumnTimezone(children.get(0), mapType.getKeyType()); + updatePaimonColumnTimezone(children.get(1), mapType.getValueType()); + break; + case ROW: + RowType rowType = (RowType) dataType; + for (int idx = 0; idx < children.size(); idx++) { + updatePaimonColumnTimezone(children.get(idx), rowType.getFields().get(idx).type()); + } + break; + default: + break; + } + } + public static TField getSchemaInfo(DataType dataType, boolean enableVarbinaryMapping, boolean enableTimestampTzMapping) { TField field = new TField(); @@ -507,14 +540,18 @@ public static List parseSchema(RowType rowType, List primaryKeys boolean enableTimestampTzMapping) { List resSchema = Lists.newArrayListWithCapacity(rowType.getFields().size()); rowType.getFields().forEach(field -> { - resSchema.add(new Column(field.name(), + Column column = new Column(field.name(), PaimonUtil.paimonTypeToDorisType(field.type(), enableVarbinaryMapping, enableTimestampTzMapping), primaryKeys.contains(field.name()), null, field.type().isNullable(), field.description(), true, - field.id())); + field.id()); + // Schema selected by relation-local options must expose the same recursive metadata + // as the normal schema cache, otherwise nested predicates bind to different field IDs. + updatePaimonColumnMetadata(column, field); + resSchema.add(column); }); return resSchema; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index c269d4ded4298b..c7b708e9742a36 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TupleDescriptor; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.TableIf; import org.apache.doris.common.DdlException; import org.apache.doris.common.MetaNotFoundException; @@ -31,6 +32,7 @@ import org.apache.doris.datasource.credentials.CredentialUtils; import org.apache.doris.datasource.credentials.VendedCredentialsFactory; import org.apache.doris.datasource.paimon.PaimonExternalCatalog; +import org.apache.doris.datasource.paimon.PaimonScanParams; import org.apache.doris.datasource.paimon.PaimonSysExternalTable; import org.apache.doris.datasource.paimon.PaimonUtil; import org.apache.doris.datasource.paimon.PaimonUtils; @@ -55,16 +57,21 @@ import com.google.common.base.Preconditions; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.paimon.CoreOptions; import org.apache.paimon.data.BinaryRow; import org.apache.paimon.predicate.Predicate; import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.BucketMode; +import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.table.source.DeletionFile; import org.apache.paimon.table.source.InnerTableScan; import org.apache.paimon.table.source.RawFile; import org.apache.paimon.table.source.ReadBuilder; +import org.apache.paimon.table.source.ScanMode; import org.apache.paimon.table.source.TableScan; +import org.apache.paimon.table.source.snapshot.SnapshotReader; import java.io.IOException; import java.util.ArrayList; @@ -83,8 +90,6 @@ public class PaimonScanNode extends FileQueryScanNode { private static final long COUNT_WITH_PARALLEL_SPLITS = 10000; // The keys of incremental read params for Paimon SDK - private static final String PAIMON_SCAN_SNAPSHOT_ID = "scan.snapshot-id"; - private static final String PAIMON_SCAN_MODE = "scan.mode"; private static final String PAIMON_INCREMENTAL_BETWEEN = "incremental-between"; private static final String PAIMON_INCREMENTAL_BETWEEN_SCAN_MODE = "incremental-between-scan-mode"; private static final String PAIMON_INCREMENTAL_BETWEEN_TIMESTAMP = "incremental-between-timestamp"; @@ -95,15 +100,15 @@ public class PaimonScanNode extends FileQueryScanNode { private static final String DORIS_END_TIMESTAMP = "endTimestamp"; private static final String DORIS_INCREMENTAL_BETWEEN_SCAN_MODE = "incrementalBetweenScanMode"; private static final String PAIMON_PROPERTY_PREFIX = "paimon."; + private static final String DORIS_ENABLE_FILE_READER_ASYNC = "jni.enable_file_reader_async"; private static final String DORIS_ENABLE_JNI_IO_MANAGER = "jni.enable_jni_io_manager"; private static final String DORIS_JNI_IO_MANAGER_TMP_DIR = "jni.io_manager.tmp_dir"; private static final String DORIS_JNI_IO_MANAGER_IMPL_CLASS = "jni.io_manager.impl_class"; private static final List BACKEND_PAIMON_OPTIONS = Arrays.asList( DORIS_ENABLE_JNI_IO_MANAGER, DORIS_JNI_IO_MANAGER_TMP_DIR, - DORIS_JNI_IO_MANAGER_IMPL_CLASS); - private static final String PAIMON_BINLOG_SYSTEM_TABLE_TYPE = "binlog"; - private static final String PAIMON_AUDIT_LOG_SYSTEM_TABLE_TYPE = "audit_log"; + DORIS_JNI_IO_MANAGER_IMPL_CLASS, + DORIS_ENABLE_FILE_READER_ASYNC); private enum SplitReadType { JNI, @@ -158,6 +163,7 @@ public String toString() { private Map storagePropertiesMap; private Map backendStorageProperties; private Map backendPaimonOptions = Collections.emptyMap(); + private Table processedTable; // The schema information involved in the current query process (including historical schema). protected ConcurrentHashMap currentQuerySchema = new ConcurrentHashMap<>(); @@ -169,16 +175,28 @@ public PaimonScanNode(PlanNodeId id, ScanContext scanContext) { super(id, desc, "PAIMON_SCAN_NODE", StatisticalType.PAIMON_SCAN_NODE, scanContext, needCheckColumnPriv, sv); + // Some branch-4.1 callers construct the scan node before attaching a table descriptor; + // defer source creation for them while preserving eager setup for normal planned scans. + if (desc.getTable() != null) { + source = new PaimonSource(desc); + } } @Override protected void doInitialize() throws UserException { + if (source == null) { + source = new PaimonSource(desc); + } + processedTable = getProcessedTable(); super.doInitialize(); long startTime = System.currentTimeMillis(); - source = new PaimonSource(desc); - serializedTable = PaimonUtil.encodeObjectToString(source.getPaimonTable()); + serializeProcessedTable(); + params.setNumOfColumnsFromFile(processedTable.rowType().getFieldCount() - getPathPartitionKeys().size()); + List queryColumns = desc.getSlots().stream() + .map(slot -> slot.getColumn()) + .collect(Collectors.toList()); // Todo: Get the current schema id of the table, instead of using -1. - ExternalUtil.initSchemaInfo(params, -1L, source.getTargetTable().getColumns()); + ExternalUtil.initSchemaInfo(params, -1L, queryColumns); PaimonExternalCatalog catalog = (PaimonExternalCatalog) source.getCatalog(); storagePropertiesMap = VendedCredentialsFactory.getStoragePropertiesMapWithVendedCredentials( catalog.getCatalogProperty().getMetastoreProperties(), @@ -192,6 +210,12 @@ protected void doInitialize() throws UserException { } } + private void serializeProcessedTable() throws UserException { + // System-table splits are materialized by the BE JNI reader, so it must receive the same + // option-bearing table copy that FE uses to plan the split. + serializedTable = PaimonUtil.encodeObjectToString(getProcessedTable()); + } + @VisibleForTesting public void setSource(PaimonSource source) { this.source = source; @@ -200,10 +224,22 @@ public void setSource(PaimonSource source) { @Override protected void convertPredicate() { PaimonPredicateConverter paimonPredicateConverter = new PaimonPredicateConverter( - source.getPaimonTable().rowType()); + processedTable.rowType()); predicates = paimonPredicateConverter.convertToPaimonExpr(conjuncts); } + @Override + protected List getFileColumnNames() { + if (scanParams != null && scanParams.isOptions()) { + // Relation-scoped options may select a historical schema, so its slots must be + // positioned against the same processed table that is serialized to the reader. + return processedTable.rowType().getFieldNames(); + } + // Normal scans must retain the refreshable descriptor schema; the cached Paimon table + // handle can still expose pre-refresh column names after an external schema change. + return super.getFileColumnNames(); + } + @Override protected void setScanParams(TFileRangeDesc rangeDesc, Split split) { if (split instanceof PaimonSplit) { @@ -409,7 +445,9 @@ public List getSplits(int numBackends) throws UserException { // Merged row counts contain only COUNT(*) semantics. COUNT(col) must keep every DataSplit // because BE will read the argument column to account for NULL and schema-mapping rules. - boolean applyCountPushdown = isTableLevelCountStarPushdown(); + // Incremental binlog readers pack an UPDATE_BEFORE/UPDATE_AFTER pair into one logical row, + // so DataSplit's physical merged count is not a valid COUNT(*) result for this relation. + boolean applyCountPushdown = isTableLevelCountStarPushdown() && !isIncrementalBinlogScan(); // Used to avoid repeatedly calculating partition info map for the same // partition data. // And for counting the number of selected partitions for this paimon table. @@ -559,8 +597,17 @@ boolean shouldForceJniForSystemTable() { } PaimonSysExternalTable paimonSysExternalTable = (PaimonSysExternalTable) externalTable; String sysTableType = paimonSysExternalTable.getSysTableType(); - return PAIMON_BINLOG_SYSTEM_TABLE_TYPE.equalsIgnoreCase(sysTableType) - || PAIMON_AUDIT_LOG_SYSTEM_TABLE_TYPE.equalsIgnoreCase(sysTableType); + return PaimonScanParams.requiresPaimonReader(sysTableType); + } + + private boolean isIncrementalBinlogScan() { + TableScanParams params = getScanParams(); + if (params == null || !params.incrementalRead() || source == null) { + return false; + } + ExternalTable externalTable = source.getExternalTable(); + return externalTable instanceof PaimonSysExternalTable + && "binlog".equalsIgnoreCase(((PaimonSysExternalTable) externalTable).getSysTableType()); } private long determineTargetFileSplitSize(List dataSplits, @@ -610,11 +657,37 @@ public List getPaimonSplitFromAPI() throws long startTime = System.currentTimeMillis(); try { Table paimonTable = getProcessedTable(); + Map resolvedOptions = scanParams == null + ? Collections.emptyMap() + : scanParams.getResolvedMapParams().orElse(Collections.emptyMap()); + if (PaimonScanParams.isPinnedEmptyScan(resolvedOptions)) { + return Collections.emptyList(); + } + Optional fileCreationTime = PaimonScanParams.getPinnedFileCreationTime(resolvedOptions); + if (fileCreationTime.isPresent()) { + if (!(paimonTable instanceof FileStoreTable)) { + throw new UserException("Paimon file-creation OPTIONS require a data table."); + } + FileStoreTable fileStoreTable = (FileStoreTable) paimonTable; + SnapshotReader snapshotReader = fileStoreTable.newSnapshotReader() + .withMode(ScanMode.ALL) + .withSnapshot(Long.parseLong( + paimonTable.options().get(CoreOptions.SCAN_SNAPSHOT_ID.key()))) + .withManifestEntryFilter(entry -> + entry.file().creationTimeEpochMillis() >= fileCreationTime.get()); + preserveBatchScanFilters(fileStoreTable, snapshotReader); + if (predicates != null) { + predicates.forEach(snapshotReader::withFilter); + } + return snapshotReader.read().splits(); + } List fieldNames = paimonTable.rowType().getFieldNames(); int[] projected = desc.getSlots().stream().mapToInt( slot -> getFieldIndex(fieldNames, slot.getColumn().getName())) - .filter(i -> i >= 0) .toArray(); + if (Arrays.stream(projected).anyMatch(index -> index < 0)) { + throw new UserException("Paimon scan schema does not contain all bound Doris columns."); + } ReadBuilder readBuilder = paimonTable.newReadBuilder(); TableScan scan = readBuilder.withFilter(predicates) .withProjection(projected) @@ -636,6 +709,20 @@ public List getPaimonSplitFromAPI() throws } } + private void preserveBatchScanFilters(FileStoreTable table, SnapshotReader snapshotReader) { + CoreOptions options = table.coreOptions(); + // This direct reader bypasses DataTableBatchScan, so preserve its correctness filters for + // deletion-vector/first-row tables and postponed buckets before reading the pinned plan. + if (!table.primaryKeys().isEmpty() + && options.batchScanSkipLevel0() + && options.toConfiguration().get(CoreOptions.BATCH_SCAN_MODE) == CoreOptions.BatchScanMode.NONE) { + snapshotReader.withLevelFilter(level -> level > 0).enableValueFilter(); + } + if (options.bucket() == BucketMode.POSTPONE_BUCKET) { + snapshotReader.onlyReadRealBuckets(); + } + } + @VisibleForTesting static int getFieldIndex(List fieldNames, String columnName) { for (int i = 0; i < fieldNames.size(); i++) { @@ -873,11 +960,8 @@ public static Map validateIncrementalReadParams(Map paimonScanParams = new HashMap<>(); - paimonScanParams.put(PAIMON_SCAN_SNAPSHOT_ID, null); - paimonScanParams.put(PAIMON_SCAN_MODE, null); if (hasSnapshotParams) { - paimonScanParams.put(PAIMON_SCAN_MODE, null); if (hasStartSnapshotId && !hasEndSnapshotId) { // Only startSnapshotId is specified throw new UserException("endSnapshotId is required when using snapshot-based incremental read"); @@ -908,15 +992,21 @@ public static Map validateIncrementalReadParams(Map new ExternalTablePreloadInfo((ExternalTable) table)); - if (tableSnapshot.isPresent() || scanParams.isPresent()) { + boolean selectorFreePaimonOptions = scanParams.isPresent() + && scanParams.get().isOptions() + && (table instanceof PaimonExternalTable || table instanceof PaimonSysExternalTable) + && PaimonScanParams.usesStatementSnapshot(scanParams.get().getMapParams()); + if (tableSnapshot.isPresent() || (scanParams.isPresent() && !selectorFreePaimonOptions)) { preloadInfo.markNonLatestRelation(); } else { preloadInfo.markLatestRelation(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java index 093af6ca6b300e..717c55c8eb8c63 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/analyzer/UnboundRelation.java @@ -148,21 +148,23 @@ public LogicalProperties computeLogicalProperties() { public Plan withGroupExpression(Optional groupExpression) { return new UnboundRelation(relationId, nameParts, groupExpression, Optional.of(getLogicalProperties()), - partNames, isTempPart, tabletIds, hints, tableSample, indexName, null, indexInSqlString, tableSnapshot); + partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, + indexInSqlString, tableSnapshot); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new UnboundRelation(relationId, nameParts, groupExpression, - logicalProperties, partNames, isTempPart, tabletIds, hints, tableSample, indexName, null, + logicalProperties, partNames, isTempPart, tabletIds, hints, tableSample, indexName, scanParams, indexInSqlString, tableSnapshot); } public UnboundRelation withIndexInSql(Pair index) { + // SQL source indexing annotates relation identity only; it must not erase scan semantics. return new UnboundRelation(relationId, nameParts, groupExpression, Optional.of(getLogicalProperties()), partNames, isTempPart, tabletIds, hints, tableSample, indexName, - null, Optional.of(index), tableSnapshot); + scanParams, Optional.of(index), tableSnapshot); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java index 25f58441f8c35c..153799232c606c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java @@ -6937,6 +6937,12 @@ private List visitTabletListContext(TabletListContext ctx) { private TableRefInfo visitBaseTableRefContext(BaseTableRefContext ctx) { List nameParts = visitMultipartIdentifier(ctx.multipartIdentifier()); TableScanParams scanParams = visitOptScanParamsContext(ctx.optScanParams()); + if (scanParams != null) { + // Command table references do not consume relation scan parameters, so reject them + // at the parser boundary instead of silently changing the command's meaning. + throw new ParseException(scanParams.getParamType().toUpperCase(Locale.ROOT) + + " scan params are only supported in query relations.", ctx); + } TableSnapshot tableSnapShot = visitTableSnapshotContext(ctx.tableSnapshot()); PartitionNamesInfo partitionNameInfo = visitSpecifiedPartitionContext(ctx.specifiedPartition()); List tabletIdList = visitTabletListContext(ctx.tabletList()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index 81a54f8c67e1e4..cd4eb0d328bccf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.rules.analysis; +import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.catalog.AggStateType; import org.apache.doris.catalog.AggregateType; @@ -42,6 +43,9 @@ import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; import org.apache.doris.datasource.iceberg.IcebergExternalTable; +import org.apache.doris.datasource.paimon.PaimonExternalTable; +import org.apache.doris.datasource.paimon.PaimonScanParams; +import org.apache.doris.datasource.paimon.PaimonSysExternalTable; import org.apache.doris.datasource.systable.SysTableResolver; import org.apache.doris.nereids.CTEContext; import org.apache.doris.nereids.CascadesContext; @@ -161,6 +165,7 @@ private LogicalPlan bindWithCurrentDb(CascadesContext cascadesContext, UnboundRe // check if it is a CTE's name CTEContext cteContext = cascadesContext.getCteContext().findCTEContext(tableName).orElse(null); if (cteContext != null) { + rejectScanParamsOnCte(unboundRelation); Optional analyzedCte = cteContext.getAnalyzedCTEPlan(tableName); if (analyzedCte.isPresent()) { LogicalCTEConsumer consumer = new LogicalCTEConsumer(unboundRelation.getRelationId(), @@ -181,6 +186,7 @@ private LogicalPlan bindWithCurrentDb(CascadesContext cascadesContext, UnboundRe // check if it is a recursive CTE's name if (cascadesContext.getRecursiveCteContext().isPresent() && cascadesContext.getRecursiveCteContext().get().findCTEContext(tableName).isPresent()) { + rejectScanParamsOnCte(unboundRelation); if (cascadesContext.isAnalyzingRecursiveCteAnchorChild()) { throw new AnalysisException( String.format("recursive reference to query %s must not appear within its non-recursive term", @@ -205,6 +211,13 @@ private LogicalPlan bindWithCurrentDb(CascadesContext cascadesContext, UnboundRe return plan; } + private void rejectScanParamsOnCte(UnboundRelation unboundRelation) { + if (unboundRelation.getScanParams() != null) { + // A CTE reference has no physical table handle on which scan parameters can be applied. + throw new AnalysisException("Table scan parameters are not supported on CTE references."); + } + } + private LogicalPlan bind(CascadesContext cascadesContext, UnboundRelation unboundRelation) { List tableQualifier = RelationUtil.getQualifierName(cascadesContext.getConnectContext(), unboundRelation.getNameParts()); @@ -408,6 +421,10 @@ private Optional handleMetaTable(TableIf table, UnboundRelation unb if (sysTablePlan.isNative()) { List qualifierWithoutTableName = qualifiedTableName.subList(0, qualifiedTableName.size() - 1); ExternalTable sysExternalTable = sysTablePlan.getSysExternalTable(); + if (sysExternalTable instanceof PaimonSysExternalTable) { + validatePaimonSystemTableScanParams( + (PaimonSysExternalTable) sysExternalTable, unboundRelation.getScanParams()); + } return Optional.of(new LogicalFileScan( unboundRelation.getRelationId(), sysExternalTable, @@ -433,6 +450,8 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio Utils.qualifiedNameWithBackquote(qualifiedTableName)); }); + validateOptionsTarget(table, unboundRelation.getScanParams()); + // Handle meta table like "table_name$partitions" // qualifiedTableName should be like "ctl.db.tbl$partitions" Optional logicalPlan = handleMetaTable(table, unboundRelation, qualifiedTableName); @@ -610,6 +629,29 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio } } + private void validateOptionsTarget(TableIf table, TableScanParams scanParams) { + if (scanParams == null || !scanParams.isOptions()) { + return; + } + if (!(table instanceof PaimonExternalTable)) { + throw new AnalysisException("OPTIONS scan params are only supported for Paimon tables."); + } + try { + PaimonScanParams.validateOptions(scanParams.getMapParams()); + } catch (IllegalArgumentException e) { + throw new AnalysisException(e.getMessage(), e); + } + } + + private void validatePaimonSystemTableScanParams( + PaimonSysExternalTable table, TableScanParams scanParams) { + try { + PaimonScanParams.validateSystemTable(table.getSysTableType(), scanParams); + } catch (IllegalArgumentException e) { + throw new AnalysisException(e.getMessage(), e); + } + } + private Plan parseAndAnalyzeExternalView( ExternalTable table, String externalCatalog, String externalDb, String ddlSql, CascadesContext cascadesContext) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 4c3194703f5153..bab7db768faab0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -617,8 +617,13 @@ public static final class TableQueryOperatorChecker extends DefaultPlanVisitorcollectToList( + UnboundRelation.class::isInstance)) { + TableScanParams scanParams = relation.getScanParams(); + if (scanParams != null) { + scanParams.resetResolvedMapParams(); + } + } + } if (logicalPlan instanceof LogicalSqlCache) { throw new AnalysisException("Unsupported sql cache for server prepared statement"); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java index 81d882d8fc5fb5..d8cac61939aebd 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java @@ -19,6 +19,7 @@ import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TableSnapshot; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.common.IdGenerator; import org.apache.doris.datasource.ExternalTable; @@ -26,6 +27,8 @@ import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergSysExternalTable; import org.apache.doris.datasource.mvcc.MvccUtil; +import org.apache.doris.datasource.paimon.PaimonExternalTable; +import org.apache.doris.datasource.paimon.PaimonSysExternalTable; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.TableSample; @@ -72,7 +75,7 @@ public LogicalFileScan(RelationId id, ExternalTable table, List qualifie Optional tableSample, Optional tableSnapshot, Optional scanParams, Optional> cachedOutputs) { this(id, table, qualifier, - table.initSelectedPartitions(MvccUtil.getSnapshotFromContext(table)), + initialSelectedPartitions(table, scanParams), operativeSlots, ImmutableList.of(), tableSample, tableSnapshot, scanParams, Optional.empty(), Optional.empty(), @@ -97,6 +100,16 @@ protected LogicalFileScan(RelationId id, ExternalTable table, List quali this.cachedOutputs = cachedSlots; } + private static SelectedPartitions initialSelectedPartitions( + ExternalTable table, Optional scanParams) { + if ((table instanceof PaimonExternalTable || table instanceof PaimonSysExternalTable) + && scanParams.isPresent() && scanParams.get().isOptions()) { + // A relation-scoped historical snapshot cannot reuse partitions cached for the + // statement-level latest snapshot; Paimon will prune its selected snapshot instead. + return SelectedPartitions.NOT_PRUNED; + } + return table.initSelectedPartitions(MvccUtil.getSnapshotFromContext(table)); + } public SelectedPartitions getSelectedPartitions() { return selectedPartitions; } @@ -188,16 +201,22 @@ public List computeOutput() { if (table instanceof IcebergExternalTable) { // iceberg v3 need append row lineage columns - return computeIcebergOutput((IcebergExternalTable) table); + return computeIcebergOutput(); + } else if (scanParams.isPresent() && scanParams.get().isOptions() + && (table instanceof PaimonExternalTable || table instanceof PaimonSysExternalTable)) { + List schema = table instanceof PaimonSysExternalTable + ? ((PaimonSysExternalTable) table).getFullSchema(scanParams.get()) + : ((PaimonExternalTable) table).getFullSchema(scanParams.get()); + return computeOutput(schema); } else { return super.computeOutput(); } } - private List computeIcebergOutput(IcebergExternalTable iceTable) { + private List computeOutput(List schema) { IdGenerator exprIdGenerator = StatementScopeIdGenerator.getExprIdGenerator(); Builder slots = ImmutableList.builder(); - table.getFullSchema() + schema .stream() .map(col -> SlotReference.fromColumn(exprIdGenerator.getNextId(), table, col, qualified())) .forEach(slots::add); @@ -208,6 +227,10 @@ private List computeIcebergOutput(IcebergExternalTable iceTable) { return slots.build(); } + private List computeIcebergOutput() { + return computeOutput(table.getFullSchema()); + } + @Override public List computeAsteriskOutput() { return super.computeAsteriskOutput(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java index dc33354f6f93fd..3389eb23918b07 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalMetaCacheTest.java @@ -21,17 +21,56 @@ import org.apache.doris.datasource.NameMapping; import org.apache.doris.datasource.SchemaCacheValue; import org.apache.doris.datasource.metacache.MetaCacheEntryStats; - +import org.apache.doris.datasource.metacache.paimon.PaimonLatestSnapshotProjectionLoader; +import org.apache.doris.datasource.metacache.paimon.PaimonPartitionInfoLoader; + +import org.apache.paimon.CoreOptions; +import org.apache.paimon.Snapshot; +import org.apache.paimon.schema.SchemaManager; +import org.apache.paimon.schema.TableSchema; +import org.apache.paimon.table.FileStoreTable; import org.junit.Assert; import org.junit.Test; +import org.mockito.Mockito; import java.util.Collections; import java.util.Map; +import java.util.Optional; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; public class PaimonExternalMetaCacheTest { + @Test + public void testLatestSnapshotUsesLatestSchemaForPinnedRead() { + PaimonLatestSnapshotProjectionLoader loader = new PaimonLatestSnapshotProjectionLoader( + new PaimonPartitionInfoLoader(null), + (nameMapping, schemaId) -> new PaimonSchemaCacheValue( + Collections.emptyList(), Collections.emptyList(), null)); + NameMapping nameMapping = new NameMapping(1L, "db", "table", "remote_db", "remote_table"); + FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); + FileStoreTable pinnedTable = Mockito.mock(FileStoreTable.class); + FileStoreTable latestSchemaTable = Mockito.mock(FileStoreTable.class); + Snapshot snapshot = Mockito.mock(Snapshot.class); + SchemaManager schemaManager = Mockito.mock(SchemaManager.class); + TableSchema latestSchema = Mockito.mock(TableSchema.class); + Mockito.when(snapshot.id()).thenReturn(12L); + Mockito.when(baseTable.latestSnapshot()).thenReturn(Optional.of(snapshot)); + Mockito.when(baseTable.copy(Collections.singletonMap( + CoreOptions.SCAN_SNAPSHOT_ID.key(), "12"))).thenReturn(pinnedTable); + Mockito.when(pinnedTable.copyWithLatestSchema()).thenReturn(latestSchemaTable); + Mockito.when(baseTable.schemaManager()).thenReturn(schemaManager); + Mockito.when(schemaManager.latest()).thenReturn(Optional.of(latestSchema)); + Mockito.when(latestSchema.id()).thenReturn(4L); + + PaimonSnapshotCacheValue value = loader.load(nameMapping, baseTable); + + Assert.assertEquals(12L, value.getSnapshot().getSnapshotId()); + Assert.assertEquals(4L, value.getSnapshot().getSchemaId()); + Assert.assertSame(latestSchemaTable, value.getSnapshot().getTable()); + Mockito.verify(pinnedTable).copyWithLatestSchema(); + } + @Test public void testInvalidateTablePrecise() { ExecutorService executor = Executors.newSingleThreadExecutor(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java new file mode 100644 index 00000000000000..6b0116433514f9 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonExternalTableTest.java @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.analysis.TableScanParams; +import org.apache.doris.datasource.mvcc.MvccSnapshot; +import org.apache.doris.datasource.mvcc.MvccUtil; + +import com.google.common.collect.ImmutableMap; +import org.apache.paimon.table.Table; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.Optional; + +public class PaimonExternalTableTest { + + @Test + public void testSelectorFreeOptionsPreserveStatementSnapshot() { + PaimonExternalTable externalTable = Mockito.mock( + PaimonExternalTable.class, Mockito.CALLS_REAL_METHODS); + MvccSnapshot snapshot = Mockito.mock(MvccSnapshot.class); + Optional statementSnapshot = Optional.of(snapshot); + Table statementTable = Mockito.mock(Table.class); + Table statementCopy = Mockito.mock(Table.class); + Table baseTable = Mockito.mock(Table.class); + Table baseCopy = Mockito.mock(Table.class); + TableScanParams scanParams = new TableScanParams( + TableScanParams.OPTIONS, + ImmutableMap.of("scan.plan-sort-partition", "true"), + Collections.emptyList()); + + Mockito.doReturn(statementTable).when(externalTable).getPaimonTable(statementSnapshot); + Mockito.when(statementTable.copy(scanParams.getMapParams())).thenReturn(statementCopy); + Mockito.when(baseTable.copy(scanParams.getMapParams())).thenReturn(baseCopy); + + try (MockedStatic mvccUtil = Mockito.mockStatic(MvccUtil.class); + MockedStatic paimonUtils = Mockito.mockStatic(PaimonUtils.class)) { + mvccUtil.when(() -> MvccUtil.getSnapshotFromContext(externalTable)).thenReturn(statementSnapshot); + paimonUtils.when(() -> PaimonUtils.getPaimonTable(externalTable)).thenReturn(baseTable); + + Assert.assertSame(statementCopy, externalTable.getPaimonTable(scanParams)); + } + } + + @Test + public void testModeOnlyLatestUsesStatementSnapshotAcrossPhases() { + PaimonExternalTable externalTable = Mockito.mock( + PaimonExternalTable.class, Mockito.CALLS_REAL_METHODS); + MvccSnapshot snapshot = Mockito.mock(MvccSnapshot.class); + Optional statementSnapshot = Optional.of(snapshot); + Table statementTable = Mockito.mock(Table.class); + Table pinnedCopy = Mockito.mock(Table.class); + Table baseTable = Mockito.mock(Table.class); + TableScanParams scanParams = new TableScanParams( + TableScanParams.OPTIONS, + ImmutableMap.of("scan.mode", "latest"), + Collections.emptyList()); + + Mockito.doReturn(statementTable).when(externalTable).getPaimonTable(statementSnapshot); + Mockito.when(statementTable.options()).thenReturn(ImmutableMap.of("scan.snapshot-id", "7")); + Mockito.when(baseTable.copy(ArgumentMatchers.anyMap())).thenReturn(pinnedCopy); + + try (MockedStatic mvccUtil = Mockito.mockStatic(MvccUtil.class); + MockedStatic paimonUtils = Mockito.mockStatic(PaimonUtils.class)) { + mvccUtil.when(() -> MvccUtil.getSnapshotFromContext(externalTable)).thenReturn(statementSnapshot); + paimonUtils.when(() -> PaimonUtils.getPaimonTable(externalTable)).thenReturn(baseTable); + + Assert.assertSame(pinnedCopy, externalTable.getPaimonTable(scanParams)); + Assert.assertSame(pinnedCopy, externalTable.getPaimonTable(scanParams)); + } + + Mockito.verify(baseTable, Mockito.times(2)).copy(ArgumentMatchers.argThat(options -> + "7".equals(options.get("scan.snapshot-id")) + && options.containsKey("scan.mode") + && options.get("scan.mode") == null)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonScanParamsTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonScanParamsTest.java new file mode 100644 index 00000000000000..b003acac955fa7 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonScanParamsTest.java @@ -0,0 +1,355 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.datasource.paimon; + +import org.apache.doris.analysis.TableScanParams; + +import com.google.common.collect.ImmutableMap; +import org.apache.paimon.CoreOptions; +import org.apache.paimon.Snapshot; +import org.apache.paimon.table.FileStoreTable; +import org.apache.paimon.table.Table; +import org.apache.paimon.table.source.snapshot.TimeTravelUtil; +import org.apache.paimon.utils.SnapshotManager; +import org.junit.Assert; +import org.junit.Test; +import org.mockito.ArgumentMatchers; +import org.mockito.MockedStatic; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.Map; + +public class PaimonScanParamsTest { + + @Test + public void testValidateKnownScanOptions() { + PaimonScanParams.validateOptions(ImmutableMap.of( + "scan.snapshot-id", "1", + "scan.plan-sort-partition", "true")); + } + + @Test + public void testRejectUnknownAndConflictingOptions() { + IllegalArgumentException typo = Assert.assertThrows( + IllegalArgumentException.class, + () -> PaimonScanParams.validateOptions( + ImmutableMap.of("scan.snapsh0t-id", "1"))); + Assert.assertTrue(typo.getMessage().contains("scan.snapsh0t-id")); + + IllegalArgumentException conflict = Assert.assertThrows( + IllegalArgumentException.class, + () -> PaimonScanParams.validateOptions(ImmutableMap.of( + "scan.snapshot-id", "1", + "scan.tag-name", "tag1"))); + Assert.assertTrue(conflict.getMessage().contains("Only one")); + } + + @Test + public void testRejectIncompatibleStartupOptionsAndFallbackBranch() { + Assert.assertThrows(IllegalArgumentException.class, + () -> PaimonScanParams.validateOptions(ImmutableMap.of( + "scan.snapshot-id", "1", + "scan.creation-time-millis", "1000"))); + Assert.assertThrows(IllegalArgumentException.class, + () -> PaimonScanParams.validateOptions(ImmutableMap.of( + "scan.mode", "latest", + "scan.snapshot-id", "1"))); + IllegalArgumentException fallback = Assert.assertThrows( + IllegalArgumentException.class, + () -> PaimonScanParams.validateOptions( + ImmutableMap.of("scan.fallback-branch", "archive"))); + Assert.assertTrue(fallback.getMessage().contains("scan.fallback-branch")); + } + + @Test + public void testRejectMissingCreationTimeAndNonBatchOptions() { + IllegalArgumentException missingTime = Assert.assertThrows( + IllegalArgumentException.class, + () -> PaimonScanParams.validateOptions( + ImmutableMap.of("scan.mode", "from-creation-timestamp"))); + Assert.assertTrue(missingTime.getMessage().contains("scan.creation-time-millis")); + + for (String option : new String[] {"scan.bounded.watermark", "scan.max-splits-per-task"}) { + IllegalArgumentException unsupported = Assert.assertThrows( + IllegalArgumentException.class, + () -> PaimonScanParams.validateOptions(ImmutableMap.of(option, "1"))); + Assert.assertTrue(unsupported.getMessage().contains(option)); + } + } + + @Test + public void testApplyPositionClearsInheritedStartupState() { + Table table = Mockito.mock(Table.class); + Table copied = Mockito.mock(Table.class); + Mockito.when(table.copy(ArgumentMatchers.anyMap())).thenReturn(copied); + + Assert.assertSame(copied, PaimonScanParams.applyOptions( + table, ImmutableMap.of("scan.creation-time-millis", "1000"))); + + Mockito.verify(table).copy(ArgumentMatchers.argThat(applied -> + "1000".equals(applied.get("scan.creation-time-millis")) + && containsNull(applied, "scan.mode") + && containsNull(applied, "scan.snapshot-id") + && containsNull(applied, "scan.tag-name") + && containsNull(applied, "scan.timestamp") + && containsNull(applied, "scan.timestamp-millis") + && containsNull(applied, "scan.watermark") + && containsNull(applied, "scan.version") + && containsNull(applied, "scan.file-creation-time-millis") + && containsNull(applied, "scan.bounded.watermark") + && containsNull(applied, "incremental-between") + && containsNull(applied, "incremental-between-timestamp") + && containsNull(applied, "incremental-between-scan-mode") + && containsNull(applied, "incremental-to-auto-tag"))); + } + + @Test + public void testApplyModeClearsInheritedPositions() { + Table table = Mockito.mock(Table.class); + Mockito.when(table.copy(ArgumentMatchers.anyMap())).thenReturn(Mockito.mock(Table.class)); + + PaimonScanParams.applyOptions(table, ImmutableMap.of("scan.mode", "latest")); + + Mockito.verify(table).copy(ArgumentMatchers.argThat(applied -> + "latest".equals(applied.get("scan.mode")) + && containsNull(applied, "scan.snapshot-id") + && containsNull(applied, "scan.tag-name") + && containsNull(applied, "scan.timestamp") + && containsNull(applied, "scan.timestamp-millis") + && containsNull(applied, "scan.watermark") + && containsNull(applied, "scan.version") + && containsNull(applied, "scan.file-creation-time-millis") + && containsNull(applied, "scan.creation-time-millis"))); + } + + @Test + public void testIsolationClearsFallbackReadStateKeys() { + Table table = Mockito.mock(Table.class); + Mockito.when(table.copy(ArgumentMatchers.anyMap())).thenReturn(Mockito.mock(Table.class)); + + PaimonScanParams.applyOptions(table, ImmutableMap.of("scan.snapshot-id", "1")); + + Mockito.verify(table).copy(ArgumentMatchers.argThat(applied -> + containsNull(applied, "log.scan") + && containsNull(applied, "log.scan.timestamp-millis"))); + + Map incremental = PaimonScanParams.isolateIncrementalRead( + ImmutableMap.of("incremental-between", "1,2")); + Assert.assertTrue(containsNull(incremental, "log.scan")); + Assert.assertTrue(containsNull(incremental, "log.scan.timestamp-millis")); + } + + @Test + public void testSystemTableCapabilityMatrixIncludesRangeAwareReaders() { + Assert.assertFalse(PaimonScanParams.supportsIncrementalRead("files")); + for (String type : new String[] {"partitions", "ro"}) { + Assert.assertTrue(type, PaimonScanParams.supportsIncrementalRead(type)); + Assert.assertFalse(type, PaimonScanParams.requiresPaimonReader(type)); + } + Assert.assertFalse(PaimonScanParams.supportsOptions("buckets")); + Assert.assertFalse(PaimonScanParams.supportsOptions("files")); + Assert.assertTrue(PaimonScanParams.supportsOptions("table_indexes")); + Assert.assertTrue(PaimonScanParams.requiresPaimonReader("audit_log")); + + Assert.assertThrows(IllegalArgumentException.class, + () -> PaimonScanParams.validateSystemTable("files", new TableScanParams( + TableScanParams.OPTIONS, + ImmutableMap.of("scan.creation-time-millis", "1234"), + Collections.emptyList()))); + Assert.assertThrows(IllegalArgumentException.class, + () -> PaimonScanParams.validateSystemTable("files", new TableScanParams( + TableScanParams.OPTIONS, + ImmutableMap.of("scan.file-creation-time-millis", "1234"), + Collections.emptyList()))); + } + + @Test + public void testSchemaSelectingOptionsRequirePinnedReaderSchema() { + Assert.assertTrue(PaimonScanParams.selectsSchema( + ImmutableMap.of("scan.snapshot-id", "1"))); + Assert.assertTrue(PaimonScanParams.selectsSchema( + ImmutableMap.of("scan.mode", "latest"))); + Assert.assertFalse(PaimonScanParams.selectsSchema( + ImmutableMap.of("scan.plan-sort-partition", "true"))); + } + + @Test + public void testMutableSelectorResolvesOnlyOncePerRelation() { + FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); + FileStoreTable selectedTable = Mockito.mock(FileStoreTable.class); + Snapshot firstSnapshot = Mockito.mock(Snapshot.class); + Mockito.when(baseTable.copy(ArgumentMatchers.anyMap())).thenReturn(selectedTable); + TableScanParams scanParams = new TableScanParams( + TableScanParams.OPTIONS, + ImmutableMap.of("scan.tag-name", "mutable_tag"), + Collections.emptyList()); + + try (MockedStatic timeTravel = Mockito.mockStatic(TimeTravelUtil.class)) { + timeTravel.when(() -> TimeTravelUtil.tryTravelOrLatest(selectedTable)) + .thenReturn(firstSnapshot); + Map first = scanParams.getOrResolveMapParams( + options -> PaimonScanParams.resolveOptions(baseTable, options)); + Map second = scanParams.getOrResolveMapParams( + options -> PaimonScanParams.resolveOptions(baseTable, options)); + + Assert.assertSame(first, second); + Assert.assertEquals("mutable_tag", second.get("scan.tag-name")); + Assert.assertFalse(second.containsKey("scan.snapshot-id")); + timeTravel.verify(() -> TimeTravelUtil.tryTravelOrLatest(selectedTable), Mockito.times(1)); + } + } + + @Test + public void testTagSelectorRetainsTagMetadataInsteadOfExpiredSnapshotPath() { + FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); + FileStoreTable selectedTable = Mockito.mock(FileStoreTable.class); + Snapshot taggedSnapshot = Mockito.mock(Snapshot.class); + Mockito.when(baseTable.copy(ArgumentMatchers.anyMap())).thenReturn(selectedTable); + + try (MockedStatic timeTravel = Mockito.mockStatic(TimeTravelUtil.class)) { + timeTravel.when(() -> TimeTravelUtil.tryTravelOrLatest(selectedTable)) + .thenReturn(taggedSnapshot); + + Map resolved = PaimonScanParams.resolveOptions( + baseTable, ImmutableMap.of("scan.tag-name", "retained_tag")); + + Assert.assertEquals("retained_tag", resolved.get("scan.tag-name")); + Assert.assertFalse(resolved.containsKey("scan.snapshot-id")); + } + } + + @Test + public void testTagValuedVersionRetainsCanonicalTagMetadata() { + FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); + FileStoreTable selectedTable = Mockito.mock(FileStoreTable.class); + Snapshot taggedSnapshot = Mockito.mock(Snapshot.class); + Mockito.when(baseTable.copy(ArgumentMatchers.anyMap())).thenReturn(selectedTable); + Mockito.when(selectedTable.options()).thenReturn(ImmutableMap.of("scan.tag-name", "retained_tag")); + + try (MockedStatic timeTravel = Mockito.mockStatic(TimeTravelUtil.class)) { + timeTravel.when(() -> TimeTravelUtil.tryTravelOrLatest(selectedTable)) + .thenReturn(taggedSnapshot); + + Map resolved = PaimonScanParams.resolveOptions( + baseTable, ImmutableMap.of("scan.version", "retained_tag")); + + Assert.assertEquals("retained_tag", resolved.get("scan.tag-name")); + Assert.assertFalse(resolved.containsKey("scan.version")); + Assert.assertFalse(resolved.containsKey("scan.snapshot-id")); + } + } + + @Test + public void testFileCreationTimePinsLatestSnapshotAndKeepsFilter() { + FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); + Snapshot latestSnapshot = Mockito.mock(Snapshot.class); + Mockito.when(latestSnapshot.id()).thenReturn(19L); + Mockito.when(baseTable.latestSnapshot()).thenReturn(java.util.Optional.of(latestSnapshot)); + + Map resolved = PaimonScanParams.resolveOptions( + baseTable, + ImmutableMap.of("scan.file-creation-time-millis", "1234")); + + Assert.assertEquals("19", resolved.get("scan.snapshot-id")); + Assert.assertEquals(Long.valueOf(1234L), + PaimonScanParams.getPinnedFileCreationTime(resolved).orElse(null)); + Assert.assertFalse(resolved.containsKey("scan.file-creation-time-millis")); + } + + @Test + public void testModeOnlyLatestPinsEmptyStatementState() { + FileStoreTable emptyTable = Mockito.mock(FileStoreTable.class); + FileStoreTable selectedTable = Mockito.mock(FileStoreTable.class); + Mockito.when(emptyTable.options()).thenReturn(Collections.emptyMap()); + Mockito.when(emptyTable.copy(ArgumentMatchers.anyMap())).thenReturn(selectedTable); + + try (MockedStatic timeTravel = Mockito.mockStatic(TimeTravelUtil.class)) { + timeTravel.when(() -> TimeTravelUtil.tryTravelOrLatest(selectedTable)).thenReturn(null); + Map resolved = PaimonScanParams.resolveOptions( + emptyTable, ImmutableMap.of("scan.mode", "latest")); + + Assert.assertTrue(PaimonScanParams.isPinnedEmptyScan(resolved)); + Assert.assertFalse(resolved.containsKey("scan.mode")); + } + } + + @Test + public void testCompactedFullPinsCompactedSnapshotInsteadOfLatest() { + FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); + FileStoreTable selectedTable = Mockito.mock(FileStoreTable.class); + Snapshot latestSnapshot = Mockito.mock(Snapshot.class); + Snapshot appendSnapshot = Mockito.mock(Snapshot.class); + Snapshot compactSnapshot = Mockito.mock(Snapshot.class); + SnapshotManager snapshotManager = Mockito.mock(SnapshotManager.class); + Mockito.when(latestSnapshot.id()).thenReturn(19L); + Mockito.when(appendSnapshot.commitKind()).thenReturn(Snapshot.CommitKind.APPEND); + Mockito.when(compactSnapshot.commitKind()).thenReturn(Snapshot.CommitKind.COMPACT); + Mockito.when(baseTable.copy(ArgumentMatchers.anyMap())).thenReturn(selectedTable); + Mockito.when(selectedTable.coreOptions()).thenReturn(new CoreOptions(Collections.emptyMap())); + Mockito.when(selectedTable.snapshotManager()).thenReturn(snapshotManager); + Mockito.when(snapshotManager.pickOrLatest(ArgumentMatchers.any())).thenAnswer(invocation -> { + java.util.function.Predicate selector = invocation.getArgument(0); + Assert.assertFalse(selector.test(appendSnapshot)); + Assert.assertTrue(selector.test(compactSnapshot)); + return 17L; + }); + + try (MockedStatic timeTravel = Mockito.mockStatic(TimeTravelUtil.class)) { + timeTravel.when(() -> TimeTravelUtil.tryTravelOrLatest(selectedTable)).thenReturn(latestSnapshot); + + Map resolved = PaimonScanParams.resolveOptions( + baseTable, ImmutableMap.of("scan.mode", "compacted-full")); + + Assert.assertEquals("17", resolved.get("scan.snapshot-id")); + } + } + + @Test + public void testCompactedFullHonorsFullCompactionDeltaCommits() { + FileStoreTable baseTable = Mockito.mock(FileStoreTable.class); + FileStoreTable selectedTable = Mockito.mock(FileStoreTable.class); + Snapshot nonFullCompaction = Mockito.mock(Snapshot.class); + Snapshot fullCompaction = Mockito.mock(Snapshot.class); + SnapshotManager snapshotManager = Mockito.mock(SnapshotManager.class); + Mockito.when(nonFullCompaction.commitKind()).thenReturn(Snapshot.CommitKind.COMPACT); + Mockito.when(nonFullCompaction.commitIdentifier()).thenReturn(4L); + Mockito.when(fullCompaction.commitKind()).thenReturn(Snapshot.CommitKind.COMPACT); + Mockito.when(fullCompaction.commitIdentifier()).thenReturn(6L); + Mockito.when(baseTable.copy(ArgumentMatchers.anyMap())).thenReturn(selectedTable); + Mockito.when(selectedTable.coreOptions()).thenReturn(new CoreOptions(ImmutableMap.of( + "changelog-producer", "full-compaction", + "full-compaction.delta-commits", "3"))); + Mockito.when(selectedTable.snapshotManager()).thenReturn(snapshotManager); + Mockito.when(snapshotManager.pickOrLatest(ArgumentMatchers.any())).thenAnswer(invocation -> { + java.util.function.Predicate selector = invocation.getArgument(0); + Assert.assertFalse(selector.test(nonFullCompaction)); + Assert.assertTrue(selector.test(fullCompaction)); + return 17L; + }); + + Map resolved = PaimonScanParams.resolveOptions( + baseTable, ImmutableMap.of("scan.mode", "compacted-full")); + + Assert.assertEquals("17", resolved.get("scan.snapshot-id")); + } + + private static boolean containsNull(Map options, String key) { + return options.containsKey(key) && options.get(key) == null; + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java index fe4c94f2f1beba..423c9ce2e270a4 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonUtilTest.java @@ -116,6 +116,21 @@ public void testSystemTableSchemaPreservesNonLowercaseColumnNames() { Assert.assertEquals("_SEQUENCE_NUMBER", columns.get(1).getName()); } + @Test + public void testParseSchemaPreservesNestedFieldMetadata() { + DataField eventTime = DataTypes.FIELD( + 17, "event_time", DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(3)); + RowType rowType = DataTypes.ROW( + DataTypes.FIELD(5, "payload", DataTypes.ROW(eventTime))); + + List columns = PaimonUtil.parseSchema(rowType, Collections.emptyList(), false, true); + + Assert.assertEquals(5, columns.get(0).getUniqueId()); + Column nested = columns.get(0).getChildren().get(0); + Assert.assertEquals(17, nested.getUniqueId()); + Assert.assertEquals("WITH_TIMEZONE", nested.getExtraInfo()); + } + @Test public void testGetPartitionInfoMapPreservesNonLowercaseKeys() { DataField mixedCasePartition = DataTypes.FIELD(0, "Dt", DataTypes.STRING()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java index 303b4d7d36c270..6b2d32f085f1b2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/source/PaimonScanNodeTest.java @@ -18,16 +18,21 @@ package org.apache.doris.datasource.paimon.source; import org.apache.doris.analysis.SlotId; +import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; +import org.apache.doris.catalog.Column; import org.apache.doris.common.ExceptionChecker; import org.apache.doris.common.UserException; import org.apache.doris.datasource.CatalogProperty; import org.apache.doris.datasource.FileQueryScanNode; import org.apache.doris.datasource.FileSplitter; import org.apache.doris.datasource.paimon.PaimonExternalCatalog; +import org.apache.doris.datasource.paimon.PaimonExternalTable; import org.apache.doris.datasource.paimon.PaimonFileExternalCatalog; +import org.apache.doris.datasource.paimon.PaimonScanParams; import org.apache.doris.datasource.paimon.PaimonSysExternalTable; +import org.apache.doris.datasource.paimon.PaimonUtil; import org.apache.doris.datasource.property.metastore.MetastoreProperties; import org.apache.doris.datasource.property.metastore.PaimonJdbcMetaStoreProperties; import org.apache.doris.planner.PlanNodeId; @@ -38,14 +43,29 @@ import org.apache.doris.thrift.TPaimonReaderType; import org.apache.doris.thrift.TPushAggOp; +import com.google.common.collect.ImmutableMap; +import org.apache.paimon.CoreOptions; +import org.apache.paimon.Snapshot; import org.apache.paimon.data.BinaryRow; +import org.apache.paimon.fs.FileIO; +import org.apache.paimon.fs.Path; import org.apache.paimon.io.DataFileMeta; import org.apache.paimon.manifest.FileSource; +import org.apache.paimon.schema.TableSchema; import org.apache.paimon.stats.SimpleStats; +import org.apache.paimon.table.AppendOnlyFileStoreTable; +import org.apache.paimon.table.BucketMode; +import org.apache.paimon.table.CatalogEnvironment; +import org.apache.paimon.table.FileStoreTable; import org.apache.paimon.table.Table; import org.apache.paimon.table.source.DataSplit; import org.apache.paimon.table.source.RawFile; +import org.apache.paimon.table.source.ScanMode; +import org.apache.paimon.table.source.snapshot.SnapshotReader; +import org.apache.paimon.types.DataField; import org.apache.paimon.types.DataTypes; +import org.apache.paimon.types.IntType; +import org.apache.paimon.utils.InstantiationUtil; import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; @@ -57,6 +77,7 @@ import java.lang.reflect.Method; import java.util.ArrayList; import java.util.Arrays; +import java.util.Base64; import java.util.Collections; import java.util.HashMap; import java.util.List; @@ -71,14 +92,6 @@ public class PaimonScanNodeTest { @Mock private PaimonFileExternalCatalog paimonFileExternalCatalog; - private PaimonSource mockPaimonSourceWithPartitionKeys(List partitionKeys) { - PaimonSource source = Mockito.mock(PaimonSource.class); - Table paimonTable = Mockito.mock(Table.class); - Mockito.when(paimonTable.partitionKeys()).thenReturn(partitionKeys); - Mockito.when(source.getPaimonTable()).thenReturn(paimonTable); - return source; - } - @Test public void testCountColumnKeepsAllSplitsWhileCountStarUsesMergedRowCount() throws UserException { PaimonScanNode node = Mockito.spy(newTestNode(new PlanNodeId(1), new TupleId(3), sv)); @@ -111,6 +124,35 @@ public void testCountColumnKeepsAllSplitsWhileCountStarUsesMergedRowCount() thro Assert.assertEquals(Optional.of(15_000L), ((PaimonSplit) countStarSplits.get(0)).getRowCount()); } + @Test + public void testIncrementalBinlogCountStarDoesNotUsePhysicalRowCount() throws UserException { + PaimonScanNode node = Mockito.spy(newTestNode(new PlanNodeId(1), new TupleId(3), sv)); + PaimonSource source = mockPaimonSourceWithPartitionKeys(Collections.emptyList()); + PaimonSysExternalTable binlogTable = Mockito.mock(PaimonSysExternalTable.class); + Mockito.when(binlogTable.getSysTableType()).thenReturn("binlog"); + Mockito.when(source.getExternalTable()).thenReturn(binlogTable); + node.setSource(source); + node.setScanParams(new TableScanParams( + TableScanParams.INCREMENTAL_READ, + ImmutableMap.of("startSnapshotId", "1", "endSnapshotId", "2"), + Collections.emptyList())); + Mockito.doReturn(Arrays.asList( + mockCountDataSplit("before.parquet", 1), + mockCountDataSplit("after.parquet", 1))) + .when(node).getPaimonSplitFromAPI(); + Mockito.when(sv.isForceJniScanner()).thenReturn(false); + Mockito.when(sv.getIgnoreSplitType()).thenReturn("NONE"); + + node.setPushDownAggNoGrouping(TPushAggOp.COUNT); + node.setPushDownCountSlotIds(Collections.emptyList()); + List splits = node.getSplits(1); + + Assert.assertEquals(2, splits.size()); + for (org.apache.doris.spi.Split split : splits) { + Assert.assertFalse(((PaimonSplit) split).getRowCount().isPresent()); + } + } + @Test public void testSplitWeight() throws UserException { @@ -224,7 +266,7 @@ public void testValidateIncrementalReadParams() throws UserException { Map result = PaimonScanNode.validateIncrementalReadParams(params); Assert.assertEquals("1,5", result.get("incremental-between")); Assert.assertTrue(result.containsKey("scan.mode") && result.get("scan.mode") == null); - Assert.assertEquals(3, result.size()); + Assert.assertEquals(16, result.size()); // 3. startSnapshotId + endSnapshotId + incrementalBetweenScanMode params.clear(); @@ -235,7 +277,7 @@ public void testValidateIncrementalReadParams() throws UserException { Assert.assertEquals("2,8", result.get("incremental-between")); Assert.assertEquals("diff", result.get("incremental-between-scan-mode")); Assert.assertTrue(result.containsKey("scan.mode") && result.get("scan.mode") == null); - Assert.assertEquals(4, result.size()); + Assert.assertEquals(16, result.size()); // 4. Only startTimestamp params.clear(); @@ -244,7 +286,7 @@ public void testValidateIncrementalReadParams() throws UserException { Assert.assertEquals("1000," + Long.MAX_VALUE, result.get("incremental-between-timestamp")); Assert.assertTrue(result.containsKey("scan.mode") && result.get("scan.mode") == null); Assert.assertTrue(result.containsKey("scan.snapshot-id") && result.get("scan.snapshot-id") == null); - Assert.assertEquals(3, result.size()); + Assert.assertEquals(16, result.size()); // 5. Both startTimestamp and endTimestamp params.clear(); @@ -254,7 +296,7 @@ public void testValidateIncrementalReadParams() throws UserException { Assert.assertEquals("1000,2000", result.get("incremental-between-timestamp")); Assert.assertTrue(result.containsKey("scan.mode") && result.get("scan.mode") == null); Assert.assertTrue(result.containsKey("scan.snapshot-id") && result.get("scan.snapshot-id") == null); - Assert.assertEquals(3, result.size()); + Assert.assertEquals(16, result.size()); // Test invalid parameter combinations @@ -349,7 +391,7 @@ public void testValidateIncrementalReadParams() throws UserException { result = PaimonScanNode.validateIncrementalReadParams(params); Assert.assertEquals("5,5", result.get("incremental-between")); Assert.assertTrue(result.containsKey("scan.mode") && result.get("scan.mode") == null); - Assert.assertEquals(3, result.size()); + Assert.assertEquals(16, result.size()); // 13. Test invalid timestamp values (< 0) params.clear(); @@ -435,8 +477,7 @@ public void testValidateIncrementalReadParams() throws UserException { Assert.assertEquals("1,5", result.get("incremental-between")); Assert.assertEquals(mode, result.get("incremental-between-scan-mode")); Assert.assertTrue(result.containsKey("scan.mode") && result.get("scan.mode") == null); - Assert.assertTrue(result.containsKey("scan.mode") && result.get("scan.mode") == null); - Assert.assertEquals(4, result.size()); + Assert.assertEquals(16, result.size()); } // 18. Test no parameters at all @@ -504,6 +545,399 @@ public void testPaimonDataSystemTableForceJniEvenWhenNativeSupported() throws Us List auditLogSplits = spyPaimonScanNode.getSplits(1); Assert.assertEquals(1, auditLogSplits.size()); Assert.assertNotNull(((PaimonSplit) auditLogSplits.get(0)).getSplit()); + + PaimonSysExternalTable rowTrackingTable = Mockito.mock(PaimonSysExternalTable.class); + Mockito.when(rowTrackingTable.getSysTableType()).thenReturn("row_tracking"); + Mockito.when(source.getExternalTable()).thenReturn(rowTrackingTable); + + Assert.assertTrue(spyPaimonScanNode.shouldForceJniForSystemTable()); + List rowTrackingSplits = spyPaimonScanNode.getSplits(1); + Assert.assertEquals(1, rowTrackingSplits.size()); + Assert.assertNotNull(((PaimonSplit) rowTrackingSplits.get(0)).getSplit()); + } + + @Test + public void testPaimonDataSystemTablesBypassCppReader() throws Exception { + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + PaimonSource source = Mockito.mock(PaimonSource.class); + PaimonSysExternalTable systemTable = Mockito.mock(PaimonSysExternalTable.class); + Mockito.when(source.getExternalTable()).thenReturn(systemTable); + node.setSource(source); + setField(PaimonScanNode.class, node, "storagePropertiesMap", Collections.emptyMap()); + + for (String type : Arrays.asList("audit_log", "binlog", "row_tracking")) { + Mockito.when(systemTable.getSysTableType()).thenReturn(type); + TFileRangeDesc rangeDesc = new TFileRangeDesc(); + invokePrivateMethod(node, "setPaimonParams", + new Class[] {TFileRangeDesc.class, PaimonSplit.class}, + rangeDesc, new PaimonSplit(createDataSplit(type + ".parquet"))); + Assert.assertEquals(TPaimonReaderType.PAIMON_JNI, + rangeDesc.getTableFormatParams().getPaimonParams().getReaderType()); + } + } + + @Test + public void testSchemaSelectingOptionsBypassCppReader() throws Exception { + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + PaimonSource source = Mockito.mock(PaimonSource.class); + PaimonExternalTable table = Mockito.mock(PaimonExternalTable.class); + Mockito.when(source.getExternalTable()).thenReturn(table); + Table baseTable = Mockito.mock(Table.class); + Mockito.when(baseTable.partitionKeys()).thenReturn(Collections.emptyList()); + Mockito.when(source.getPaimonTable()).thenReturn(baseTable); + node.setSource(source); + node.setScanParams(new TableScanParams( + TableScanParams.OPTIONS, + ImmutableMap.of("scan.snapshot-id", "1"), + Collections.emptyList())); + setField(PaimonScanNode.class, node, "storagePropertiesMap", Collections.emptyMap()); + + TFileRangeDesc rangeDesc = new TFileRangeDesc(); + invokePrivateMethod(node, "setPaimonParams", + new Class[] {TFileRangeDesc.class, PaimonSplit.class}, + rangeDesc, new PaimonSplit(createDataSplit("historical.parquet"))); + + Assert.assertEquals(TPaimonReaderType.PAIMON_JNI, + rangeDesc.getTableFormatParams().getPaimonParams().getReaderType()); + } + + @Test + public void testSystemTablePassesIncrementalOptionsToPaimonTable() throws Exception { + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + PaimonSource source = Mockito.mock(PaimonSource.class); + PaimonSysExternalTable systemTable = Mockito.mock(PaimonSysExternalTable.class); + Mockito.when(systemTable.getSysTableType()).thenReturn("audit_log"); + Table baseTable = Mockito.mock(Table.class); + Table copiedTable = Mockito.mock(Table.class); + Mockito.when(source.getExternalTable()).thenReturn(systemTable); + Mockito.when(source.getPaimonTable()).thenReturn(baseTable); + node.setSource(source); + + Map params = new HashMap<>(); + params.put("startSnapshotId", "1"); + params.put("endSnapshotId", "2"); + node.setScanParams(new TableScanParams( + TableScanParams.INCREMENTAL_READ, params, Collections.emptyList())); + + Map expectedOptions = new HashMap<>(); + expectedOptions.put("scan.timestamp", null); + expectedOptions.put("scan.timestamp-millis", null); + expectedOptions.put("scan.watermark", null); + expectedOptions.put("scan.file-creation-time-millis", null); + expectedOptions.put("scan.creation-time-millis", null); + expectedOptions.put("scan.snapshot-id", null); + expectedOptions.put("scan.tag-name", null); + expectedOptions.put("scan.version", null); + expectedOptions.put("scan.bounded.watermark", null); + expectedOptions.put("scan.mode", null); + expectedOptions.put("log.scan", null); + expectedOptions.put("log.scan.timestamp-millis", null); + expectedOptions.put("incremental-between-timestamp", null); + expectedOptions.put("incremental-between-scan-mode", null); + expectedOptions.put("incremental-to-auto-tag", null); + expectedOptions.put("incremental-between", "1,2"); + Mockito.when(baseTable.copy(expectedOptions)).thenReturn(copiedTable); + + try { + Assert.assertSame(copiedTable, invokePrivateMethod(node, "getProcessedTable")); + } catch (java.lang.reflect.InvocationTargetException e) { + Assert.fail("Paimon system table should accept incremental options, but got: " + + e.getTargetException().getMessage()); + } + Mockito.verify(baseTable).copy(expectedOptions); + } + + @Test + public void testPinnedFileCreationScanPreservesBatchReaderFilters() throws Exception { + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + PaimonSource source = Mockito.mock(PaimonSource.class); + PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class); + FileStoreTable table = Mockito.mock(FileStoreTable.class); + Snapshot snapshot = Mockito.mock(Snapshot.class); + SnapshotReader reader = Mockito.mock(SnapshotReader.class); + SnapshotReader.Plan plan = Mockito.mock(SnapshotReader.Plan.class); + CoreOptions coreOptions = Mockito.mock(CoreOptions.class); + org.apache.paimon.options.Options configuration = new org.apache.paimon.options.Options(); + configuration.set(CoreOptions.BATCH_SCAN_MODE, CoreOptions.BatchScanMode.NONE); + + Mockito.when(source.getExternalTable()).thenReturn(externalTable); + Mockito.when(source.getPaimonTable()).thenReturn(table); + Mockito.when(source.getPaimonTable(ArgumentMatchers.any(TableScanParams.class))).thenReturn(table); + Mockito.when(snapshot.id()).thenReturn(23L); + Mockito.when(table.latestSnapshot()).thenReturn(Optional.of(snapshot)); + Mockito.when(table.options()).thenReturn(ImmutableMap.of("scan.snapshot-id", "23")); + Mockito.when(table.primaryKeys()).thenReturn(Collections.singletonList("id")); + Mockito.when(table.coreOptions()).thenReturn(coreOptions); + Mockito.when(coreOptions.batchScanSkipLevel0()).thenReturn(true); + Mockito.when(coreOptions.toConfiguration()).thenReturn(configuration); + Mockito.when(coreOptions.bucket()).thenReturn(BucketMode.POSTPONE_BUCKET); + Mockito.when(table.newSnapshotReader()).thenReturn(reader); + Mockito.when(reader.withMode(ScanMode.ALL)).thenReturn(reader); + Mockito.when(reader.withSnapshot(23L)).thenReturn(reader); + Mockito.when(reader.withManifestEntryFilter(ArgumentMatchers.any())).thenReturn(reader); + Mockito.when(reader.withLevelFilter(ArgumentMatchers.any())).thenReturn(reader); + Mockito.when(reader.enableValueFilter()).thenReturn(reader); + Mockito.when(reader.onlyReadRealBuckets()).thenReturn(reader); + Mockito.when(reader.read()).thenReturn(plan); + Mockito.when(plan.splits()).thenReturn(Collections.emptyList()); + node.setSource(source); + TableScanParams scanParams = new TableScanParams( + TableScanParams.OPTIONS, + ImmutableMap.of("scan.file-creation-time-millis", "1234"), + Collections.emptyList()); + scanParams.getOrResolveMapParams(options -> PaimonScanParams.resolveOptions(table, options)); + node.setScanParams(scanParams); + + Assert.assertTrue(node.getPaimonSplitFromAPI().isEmpty()); + + Mockito.verify(reader).withLevelFilter(ArgumentMatchers.any()); + Mockito.verify(reader).enableValueFilter(); + Mockito.verify(reader).onlyReadRealBuckets(); + } + + @Test + public void testSystemTableRejectsIncrementalReadWhenReaderIgnoresRange() throws Exception { + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + PaimonSource source = Mockito.mock(PaimonSource.class); + PaimonSysExternalTable systemTable = Mockito.mock(PaimonSysExternalTable.class); + Mockito.when(systemTable.getSysTableType()).thenReturn("snapshots"); + Mockito.when(source.getExternalTable()).thenReturn(systemTable); + Mockito.when(source.getPaimonTable()).thenReturn(Mockito.mock(Table.class)); + node.setSource(source); + node.setScanParams(new TableScanParams( + TableScanParams.INCREMENTAL_READ, + ImmutableMap.of("startSnapshotId", "1", "endSnapshotId", "2"), + Collections.emptyList())); + + try { + invokePrivateMethod(node, "getProcessedTable"); + Assert.fail("snapshots must reject an incremental range it does not consume"); + } catch (java.lang.reflect.InvocationTargetException e) { + Assert.assertTrue(e.getTargetException().getMessage() + .contains("does not support INCR")); + } + } + + @Test + public void testSystemTablePassesDynamicOptionsToPaimonTable() throws Exception { + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + PaimonSource source = Mockito.mock(PaimonSource.class); + PaimonSysExternalTable systemTable = Mockito.mock(PaimonSysExternalTable.class); + Mockito.when(systemTable.getSysTableType()).thenReturn("table_indexes"); + Table baseTable = Mockito.mock(Table.class); + Table copiedTable = Mockito.mock(Table.class); + Mockito.when(source.getExternalTable()).thenReturn(systemTable); + Mockito.when(source.getPaimonTable()).thenReturn(baseTable); + Mockito.when(source.getPaimonTable(ArgumentMatchers.any(TableScanParams.class))) + .thenAnswer(invocation -> PaimonScanParams.applyOptions( + baseTable, invocation.getArgument(0).getMapParams())); + node.setSource(source); + + Map options = ImmutableMap.of( + "scan.snapshot-id", "12345", + "scan.mode", "from-snapshot"); + node.setScanParams(new TableScanParams( + TableScanParams.OPTIONS, options, Collections.emptyList())); + Mockito.when(baseTable.copy(ArgumentMatchers.anyMap())).thenReturn(copiedTable); + + try { + Assert.assertSame(copiedTable, invokePrivateMethod(node, "getProcessedTable")); + } catch (java.lang.reflect.InvocationTargetException e) { + Assert.fail("Paimon system table should accept dynamic options, but got: " + + e.getTargetException().getMessage()); + } + Mockito.verify(baseTable).copy(ArgumentMatchers.argThat(applied -> + "12345".equals(applied.get("scan.snapshot-id")) + && "from-snapshot".equals(applied.get("scan.mode")) + && applied.containsKey("scan.tag-name") + && applied.get("scan.tag-name") == null)); + } + + @Test + public void testDataTableQueryOptionsOverrideDefaultsWithoutMutation() throws Exception { + Map defaultOptions = new HashMap<>(); + defaultOptions.put("scan.mode", "latest"); + TableSchema schema = new TableSchema( + 0, + Collections.singletonList(new DataField(0, "id", new IntType())), + 0, + Collections.emptyList(), + Collections.emptyList(), + defaultOptions, + null); + Table baseTable = new AppendOnlyFileStoreTable( + Mockito.mock(FileIO.class), + new Path("memory://paimon_dynamic_options"), + schema, + CatalogEnvironment.empty()); + + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + PaimonSource source = Mockito.mock(PaimonSource.class); + Mockito.when(source.getExternalTable()).thenReturn(Mockito.mock(PaimonExternalTable.class)); + Mockito.when(source.getPaimonTable()).thenReturn(baseTable); + Mockito.when(source.getPaimonTable(ArgumentMatchers.any(TableScanParams.class))) + .thenAnswer(invocation -> PaimonScanParams.applyOptions( + baseTable, invocation.getArgument(0).getMapParams())); + node.setSource(source); + + Map queryOptions = ImmutableMap.of( + "scan.mode", "from-snapshot", + "scan.snapshot-id", "2"); + node.setScanParams(new TableScanParams( + TableScanParams.OPTIONS, queryOptions, Collections.emptyList())); + + Table processedTable = (Table) invokePrivateMethod(node, "getProcessedTable"); + Assert.assertEquals("from-snapshot", processedTable.options().get("scan.mode")); + Assert.assertEquals("2", processedTable.options().get("scan.snapshot-id")); + Assert.assertEquals("latest", baseTable.options().get("scan.mode")); + Assert.assertFalse(baseTable.options().containsKey("scan.snapshot-id")); + } + + @Test + public void testDataTableOptionsUseRelationScopedCatalogHandle() throws Exception { + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + PaimonSource source = Mockito.mock(PaimonSource.class); + PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class); + Table statementSnapshotTable = Mockito.mock(Table.class); + Table relationScopedTable = Mockito.mock(Table.class); + Mockito.when(source.getExternalTable()).thenReturn(externalTable); + Mockito.when(source.getPaimonTable()).thenReturn(statementSnapshotTable); + node.setSource(source); + + TableScanParams scanParams = new TableScanParams( + TableScanParams.OPTIONS, + ImmutableMap.of("scan.snapshot-id", "1"), + Collections.emptyList()); + node.setScanParams(scanParams); + Mockito.when(source.getPaimonTable(scanParams)).thenReturn(relationScopedTable); + + Assert.assertSame(relationScopedTable, invokePrivateMethod(node, "getProcessedTable")); + Mockito.verify(source).getPaimonTable(scanParams); + Mockito.verify(statementSnapshotTable, Mockito.never()).copy(ArgumentMatchers.anyMap()); + } + + @Test + public void testFileColumnPositionsUseProcessedHistoricalSchema() throws Exception { + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + node.setScanParams(new TableScanParams( + TableScanParams.OPTIONS, + ImmutableMap.of("scan.snapshot-id", "1"), + Collections.emptyList())); + Table historicalTable = Mockito.mock(Table.class); + Mockito.when(historicalTable.rowType()).thenReturn(new org.apache.paimon.types.RowType(Arrays.asList( + new DataField(0, "id", new IntType()), + new DataField(1, "old_name", new org.apache.paimon.types.VarCharType())))); + setField(PaimonScanNode.class, node, "processedTable", historicalTable); + + Assert.assertEquals(Arrays.asList("id", "old_name"), node.getFileColumnNames()); + } + + @Test + public void testLatestScanUsesRefreshedDescriptorColumnPositions() throws Exception { + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + Column latestColumn = Mockito.mock(Column.class); + Mockito.when(latestColumn.getName()).thenReturn("renamed_name"); + Mockito.when(node.getTupleDesc().getTable().getFullSchema()) + .thenReturn(Collections.singletonList(latestColumn)); + + Table staleTableHandle = Mockito.mock(Table.class); + setField(PaimonScanNode.class, node, "processedTable", staleTableHandle); + + Assert.assertEquals(Collections.singletonList("renamed_name"), node.getFileColumnNames()); + } + + @Test + public void testDataTableQueryOptionsReplaceInheritedSnapshotSelector() throws Exception { + Map defaultOptions = new HashMap<>(); + defaultOptions.put("scan.snapshot-id", "9"); + TableSchema schema = new TableSchema( + 0, + Collections.singletonList(new DataField(0, "id", new IntType())), + 0, + Collections.emptyList(), + Collections.emptyList(), + defaultOptions, + null); + Table pinnedLatestTable = new AppendOnlyFileStoreTable( + Mockito.mock(FileIO.class), + new Path("memory://paimon_dynamic_tag"), + schema, + CatalogEnvironment.empty()); + + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + PaimonSource source = Mockito.mock(PaimonSource.class); + Mockito.when(source.getExternalTable()).thenReturn(Mockito.mock(PaimonExternalTable.class)); + Mockito.when(source.getPaimonTable()).thenReturn(pinnedLatestTable); + Mockito.when(source.getPaimonTable(ArgumentMatchers.any(TableScanParams.class))) + .thenAnswer(invocation -> PaimonScanParams.applyOptions( + pinnedLatestTable, invocation.getArgument(0).getMapParams())); + node.setSource(source); + node.setScanParams(new TableScanParams( + TableScanParams.OPTIONS, + ImmutableMap.of("scan.tag-name", "tag1"), + Collections.emptyList())); + + Table processedTable = (Table) invokePrivateMethod(node, "getProcessedTable"); + Assert.assertEquals("tag1", processedTable.options().get("scan.tag-name")); + Assert.assertFalse(processedTable.options().containsKey("scan.snapshot-id")); + } + + @Test + public void testBackendSerializationUsesDynamicOptionsTable() throws Exception { + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + PaimonSource source = Mockito.mock(PaimonSource.class); + PaimonSysExternalTable systemTable = Mockito.mock(PaimonSysExternalTable.class); + Mockito.when(systemTable.getSysTableType()).thenReturn("table_indexes"); + Table baseTable = Mockito.mock(Table.class); + Table copiedTable = Mockito.mock(Table.class, Mockito.withSettings().serializable()); + Mockito.when(source.getExternalTable()).thenReturn(systemTable); + Mockito.when(source.getPaimonTable()).thenReturn(baseTable); + Mockito.when(source.getPaimonTable(ArgumentMatchers.any(TableScanParams.class))) + .thenAnswer(invocation -> PaimonScanParams.applyOptions( + baseTable, invocation.getArgument(0).getMapParams())); + // The invocation happens on the deserialized mock copy, so Mockito cannot record it + // against this test instance when checking strict stubbings. + Mockito.lenient().when(copiedTable.name()).thenReturn("files-at-snapshot"); + node.setSource(source); + + Map options = ImmutableMap.of("scan.snapshot-id", "1"); + node.setScanParams(new TableScanParams( + TableScanParams.OPTIONS, options, Collections.emptyList())); + Mockito.when(baseTable.copy(ArgumentMatchers.anyMap())).thenReturn(copiedTable); + + try { + invokePrivateMethod(node, "serializeProcessedTable"); + } catch (NoSuchMethodException e) { + Assert.fail("PaimonScanNode must serialize the processed table for backend JNI reads"); + } + + java.lang.reflect.Field field = PaimonScanNode.class.getDeclaredField("serializedTable"); + field.setAccessible(true); + String encoded = (String) field.get(node); + Table decoded = InstantiationUtil.deserializeObject( + Base64.getUrlDecoder().decode(encoded), PaimonUtil.class.getClassLoader()); + Assert.assertEquals("files-at-snapshot", decoded.name()); + } + + @Test + public void testSystemTableRejectsNonIncrementalScanParams() throws Exception { + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); + PaimonSource source = Mockito.mock(PaimonSource.class); + Mockito.when(source.getExternalTable()).thenReturn(Mockito.mock(PaimonSysExternalTable.class)); + Mockito.when(source.getPaimonTable()).thenReturn(Mockito.mock(Table.class)); + node.setSource(source); + node.setScanParams(new TableScanParams( + TableScanParams.BRANCH, + Collections.singletonMap(TableScanParams.PARAMS_NAME, "branch1"), + Collections.emptyList())); + + try { + invokePrivateMethod(node, "getProcessedTable"); + Assert.fail("Paimon system table should reject non-incremental scan params"); + } catch (java.lang.reflect.InvocationTargetException e) { + Assert.assertTrue(e.getTargetException().getMessage() + .contains("only support INCR or OPTIONS scan params")); + } } @Test @@ -592,13 +1026,10 @@ public void testGetBackendPaimonOptionsForJniIOManager() { @Test public void testApplyBackendPaimonOptionsAtScanNodeLevel() throws Exception { - PaimonScanNode node = new PaimonScanNode(new PlanNodeId(0), new TupleDescriptor(new TupleId(0)), - false, sv, ScanContext.EMPTY); + PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); PaimonSource source = Mockito.mock(PaimonSource.class); - Table paimonTable = Mockito.mock(Table.class); - Mockito.when(source.getTableLocation()).thenReturn("file:///warehouse"); + Table paimonTable = mockPaimonTableWithPartitionKeys(Collections.emptyList()); Mockito.when(source.getPaimonTable()).thenReturn(paimonTable); - Mockito.when(paimonTable.partitionKeys()).thenReturn(Collections.emptyList()); node.setSource(source); Map backendOptions = new HashMap<>(); @@ -677,7 +1108,6 @@ public void testSetPaimonParamsUsesOrderedPartitionKeys() throws Exception { Table table = Mockito.mock(Table.class); PaimonSysExternalTable sysTable = Mockito.mock(PaimonSysExternalTable.class); Mockito.when(source.getPaimonTable()).thenReturn(table); - Mockito.when(source.getTableLocation()).thenReturn("file:///warehouse"); Mockito.when(source.getExternalTable()).thenReturn(sysTable); Mockito.when(sysTable.isDataTable()).thenReturn(true); Mockito.when(table.partitionKeys()).thenReturn(Arrays.asList("Pt", "Dt")); @@ -738,11 +1168,9 @@ public void testNativeSplitCarriesPartitionMetadataWithoutRuntimeFilterPruning() } @Test - public void testSetPaimonParamsUsesJniWhenCppOptionEnabled() throws Exception { - Mockito.when(sv.isEnablePaimonCppReader()).thenReturn(true); + public void testSetPaimonParamsUsesJniForDataSplit() throws Exception { PaimonScanNode node = newTestNode(new PlanNodeId(0), new TupleId(0), sv); PaimonSource source = Mockito.mock(PaimonSource.class); - Mockito.when(source.getTableLocation()).thenReturn("file:///warehouse"); Table paimonTable = mockPaimonTableWithPartitionKeys(Collections.emptyList()); Mockito.when(source.getPaimonTable()).thenReturn(paimonTable); node.setSource(source); @@ -770,8 +1198,20 @@ private void mockJniReader(PaimonScanNode spyNode) { Mockito.doReturn(false).when(spyNode).supportNativeReader(ArgumentMatchers.any(Optional.class)); } - private PaimonScanNode newTestNode(PlanNodeId id, TupleId tupleId, SessionVariable sessionVariable) { - return new PaimonScanNode(id, new TupleDescriptor(tupleId), false, sessionVariable, ScanContext.EMPTY); + private PaimonScanNode newTestNode(PlanNodeId planNodeId, TupleId tupleId, SessionVariable sessionVariable) { + TupleDescriptor desc = new TupleDescriptor(tupleId); + PaimonExternalTable externalTable = Mockito.mock(PaimonExternalTable.class); + Table paimonTable = mockPaimonTableWithPartitionKeys(Collections.emptyList()); + Mockito.when(externalTable.getPaimonTable(ArgumentMatchers.any(Optional.class))).thenReturn(paimonTable); + desc.setTable(externalTable); + return new PaimonScanNode(planNodeId, desc, false, sessionVariable, ScanContext.EMPTY); + } + + private PaimonSource mockPaimonSourceWithPartitionKeys(List partitionKeys) { + PaimonSource source = Mockito.mock(PaimonSource.class); + Table paimonTable = mockPaimonTableWithPartitionKeys(partitionKeys); + Mockito.when(source.getPaimonTable()).thenReturn(paimonTable); + return source; } private Table mockPaimonTableWithPartitionKeys(List partitionKeys) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractPaimonPropertiesTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractPaimonPropertiesTest.java index 9cac43830ff75b..095dcfa8f1513f 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractPaimonPropertiesTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/property/metastore/AbstractPaimonPropertiesTest.java @@ -183,4 +183,18 @@ void testRejectInvalidTableOptionValue() { Assertions.assertTrue(exception.getMessage().contains("read.batch-size")); } + @Test + void testForwardTableDefaultOptionsToPaimonCatalog() { + Map input = new HashMap<>(); + input.put("paimon.table-default.scan.mode", "latest"); + input.put("paimon.table-default.scan.snapshot-id", "7"); + TestPaimonProperties testProps = new TestPaimonProperties(input); + + testProps.buildCatalogOptions(); + + Assertions.assertEquals( + "latest", testProps.getCatalogOptionsMap().get("table-default.scan.mode")); + Assertions.assertEquals( + "7", testProps.getCatalogOptionsMap().get("table-default.scan.snapshot-id")); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java index 6df0d4693cbef9..87d31f533cfa9b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/StatementContextTest.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids; +import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.TableIf; @@ -35,6 +36,7 @@ import org.apache.doris.qe.SessionVariable; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Test; import org.mockito.InOrder; import org.mockito.Mockito; @@ -167,7 +169,7 @@ public void testSkipPreloadWhenSessionVariableDisabled() { } @Test - public void testSkipLatestPreloadWhenExplicitSnapshotExists() { + public void testPreloadLatestRelationWhenExplicitSnapshotAliasExists() { ConnectContext connectContext = Mockito.mock(ConnectContext.class); TableIf internalTable = Mockito.mock(TableIf.class); HMSExternalTable hmsExternalTable = Mockito.mock(HMSExternalTable.class); @@ -176,6 +178,7 @@ public void testSkipLatestPreloadWhenExplicitSnapshotExists() { SessionVariable sessionVariable = new SessionVariable(); sessionVariable.setEnablePreloadExternalMetadata(true); + // A historical alias must not cancel the metadata warmup required by a latest alias. Mockito.when(connectContext.getSessionVariable()).thenReturn(sessionVariable); Mockito.when(connectContext.getQueryIdentifier()).thenReturn("query-2"); Mockito.when(internalTable.needReadLockWhenPlan()).thenReturn(true); @@ -201,11 +204,10 @@ public void testSkipLatestPreloadWhenExplicitSnapshotExists() { org.junit.jupiter.api.Assertions.assertTrue(result.isExecuted()); org.junit.jupiter.api.Assertions.assertEquals(1, result.getCandidateTableCount()); - org.junit.jupiter.api.Assertions.assertEquals(0, result.getPreloadedTableCount()); - Mockito.verify(hmsExternalTable, Mockito.never()) + org.junit.jupiter.api.Assertions.assertEquals(1, result.getPreloadedTableCount()); + Mockito.verify(hmsExternalTable, Mockito.times(1)) .loadSnapshot(Mockito.>any(), Mockito.any()); - Mockito.verify(hmsExternalTable, Mockito.never()).getBaseSchema(); - Mockito.verify(hmsExternalTable, Mockito.never()).initSelectedPartitions(Mockito.any()); + Mockito.verify(hmsExternalTable, Mockito.times(1)).getBaseSchema(); } finally { statementContext.close(); } @@ -529,6 +531,55 @@ public void testPreloadPaimonLatestSnapshotBeforeLock() { } } + @Test + public void testSelectorFreePaimonOptionsPreloadLatestSnapshotBeforeLock() { + ConnectContext connectContext = Mockito.mock(ConnectContext.class); + TableIf internalTable = Mockito.mock(TableIf.class); + PaimonExternalTable paimonExternalTable = Mockito.mock(PaimonExternalTable.class); + DatabaseIf database = mockDatabase(); + CatalogIf catalog = mockCatalog(); + MvccSnapshot mvccSnapshot = Mockito.mock(MvccSnapshot.class); + SessionVariable sessionVariable = new SessionVariable(); + sessionVariable.setEnablePreloadExternalMetadata(true); + + Mockito.when(connectContext.getSessionVariable()).thenReturn(sessionVariable); + Mockito.when(internalTable.needReadLockWhenPlan()).thenReturn(true); + Mockito.when(paimonExternalTable.getId()).thenReturn(18L); + Mockito.when(paimonExternalTable.getName()).thenReturn("paimon_tbl"); + Mockito.when(paimonExternalTable.getDatabase()).thenReturn(database); + Mockito.when(database.getFullName()).thenReturn("db"); + Mockito.when(database.getCatalog()).thenReturn(catalog); + Mockito.when(catalog.getName()).thenReturn("ctl"); + Mockito.when(paimonExternalTable.supportsExternalMetadataPreload()).thenReturn(true); + Mockito.when(paimonExternalTable.supportsLatestSnapshotPreload()).thenReturn(true); + Mockito.when(paimonExternalTable.loadSnapshot(Mockito.>any(), Mockito.any())) + .thenReturn(mvccSnapshot); + Mockito.when(paimonExternalTable.getBaseSchema()).thenReturn(Collections.emptyList()); + Mockito.when(paimonExternalTable.supportInternalPartitionPruned()).thenReturn(true); + Mockito.when(paimonExternalTable.initSelectedPartitions(Mockito.any())) + .thenReturn(SelectedPartitions.NOT_PRUNED); + + StatementContext statementContext = new StatementContext(connectContext, new OriginStatement("select 1", 0)); + try { + statementContext.getTables().put(ImmutableList.of("ctl", "db", "internal"), internalTable); + statementContext.registerExternalTableForPreload( + paimonExternalTable, + Optional.empty(), + Optional.of(new TableScanParams( + TableScanParams.OPTIONS, + ImmutableMap.of("scan.plan-sort-partition", "true"), + Collections.emptyList()))); + + executePreload(statementContext); + + Mockito.verify(paimonExternalTable, Mockito.times(1)) + .loadSnapshot(Mockito.>any(), Mockito.any()); + Mockito.verify(paimonExternalTable, Mockito.times(1)).getBaseSchema(); + } finally { + statementContext.close(); + } + } + @SuppressWarnings("unchecked") private DatabaseIf mockDatabase() { return Mockito.mock(DatabaseIf.class); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java index 3b0c669049e7df..bc9e68dd06b98b 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/parser/NereidsParserTest.java @@ -24,6 +24,7 @@ import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundFunction; import org.apache.doris.nereids.analyzer.UnboundOneRowRelation; +import org.apache.doris.nereids.analyzer.UnboundRelation; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.exceptions.NotSupportedException; import org.apache.doris.nereids.exceptions.ParseException; @@ -118,6 +119,100 @@ public void testSingle() { Assertions.assertNull(exceptionOccurred); } + @Test + public void testParseTableOptionsParams() { + NereidsParser nereidsParser = new NereidsParser(); + UnboundRelation relation = findFirstUnboundRelation(nereidsParser.parseSingle( + "select * from paimon_catalog.test_db.`orders$files`" + + "@options('scan.snapshot-id'='12345', 'scan.mode'='from-snapshot')")); + + Assertions.assertNotNull(relation); + Assertions.assertNotNull(relation.getScanParams()); + Assertions.assertEquals("options", relation.getScanParams().getParamType()); + Assertions.assertEquals( + ImmutableMap.of("scan.snapshot-id", "12345", "scan.mode", "from-snapshot"), + relation.getScanParams().getMapParams()); + } + + @Test + public void testRejectOptionsWithoutKeyValuePairs() { + NereidsParser nereidsParser = new NereidsParser(); + Assertions.assertThrows(IllegalArgumentException.class, + () -> nereidsParser.parseSingle("select * from t@options()")); + Assertions.assertThrows(IllegalArgumentException.class, + () -> nereidsParser.parseSingle("select * from t@options(foo, bar)")); + } + + @Test + public void testCreateViewParserPreservesTableOptions() { + NereidsParser nereidsParser = new NereidsParser(); + UnboundRelation relation = findFirstUnboundRelation(nereidsParser.parseForCreateView( + "select * from paimon_catalog.test_db.orders" + + "@options('scan.snapshot-id'='1')")); + + Assertions.assertNotNull(relation); + Assertions.assertNotNull(relation.getScanParams()); + Assertions.assertEquals( + ImmutableMap.of("scan.snapshot-id", "1"), + relation.getScanParams().getMapParams()); + } + + @Test + public void testRejectOptionsInBaseTableRefCommand() { + NereidsParser nereidsParser = new NereidsParser(); + ParseException exception = Assertions.assertThrows(ParseException.class, + () -> nereidsParser.parseSingle( + "show replica distribution from db1.t" + + "@options('scan.snapshot-id'='1')")); + Assertions.assertTrue(exception.getMessage().contains( + "OPTIONS scan params are only supported in query relations")); + } + + @Test + public void testParseIndependentDataTableOptionsParams() { + NereidsParser nereidsParser = new NereidsParser(); + Plan plan = nereidsParser.parseSingle( + "select * from paimon_catalog.test_db.orders" + + "@options('scan.snapshot-id'='1') left_orders " + + "join paimon_catalog.test_db.orders" + + "@options('scan.snapshot-id'='2') right_orders " + + "on left_orders.id = right_orders.id"); + + List relations = new ArrayList<>(); + collectUnboundRelations(plan, relations); + Assertions.assertEquals(2, relations.size()); + Assertions.assertEquals( + ImmutableMap.of("scan.snapshot-id", "1"), + relations.get(0).getScanParams().getMapParams()); + Assertions.assertEquals( + ImmutableMap.of("scan.snapshot-id", "2"), + relations.get(1).getScanParams().getMapParams()); + } + + @Test + public void testRejectConflictingTableScanParams() { + NereidsParser nereidsParser = new NereidsParser(); + Assertions.assertThrows(ParseException.class, () -> nereidsParser.parseSingle( + "select * from paimon_catalog.test_db.orders" + + "@options('scan.snapshot-id'='1')" + + "@options('scan.snapshot-id'='2')")); + Assertions.assertThrows(ParseException.class, () -> nereidsParser.parseSingle( + "select * from paimon_catalog.test_db.orders@incr(" + + "'startSnapshotId'=1, 'endSnapshotId'=2)" + + "@options('scan.snapshot-id'='1')")); + } + + @Test + public void testOptionsHintIsNotTableScanParams() { + NereidsParser nereidsParser = new NereidsParser(); + UnboundRelation relation = findFirstUnboundRelation(nereidsParser.parseSingle( + "select * from paimon_catalog.test_db.orders " + + "/*+ OPTIONS('scan.snapshot-id'='1') */")); + + Assertions.assertNotNull(relation); + Assertions.assertNull(relation.getScanParams()); + } + @Test public void testErrorListener() { parsePlan("select * from t1 where a = 1 illegal_symbol") @@ -931,6 +1026,28 @@ private void checkQueryTopPlanClass(String sql, NereidsParser parser, Class c } } + private UnboundRelation findFirstUnboundRelation(Plan plan) { + if (plan instanceof UnboundRelation) { + return (UnboundRelation) plan; + } + for (Plan child : plan.children()) { + UnboundRelation relation = findFirstUnboundRelation(child); + if (relation != null) { + return relation; + } + } + return null; + } + + private void collectUnboundRelations(Plan plan, List relations) { + if (plan instanceof UnboundRelation) { + relations.add((UnboundRelation) plan); + } + for (Plan child : plan.children()) { + collectUnboundRelations(child, relations); + } + } + @Test public void testBlockSqlAst() { String sql = "plan replayer dump select `AD``D` from t1 where a = 1"; diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java index e877989f41e8a8..a03b1386077295 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/analysis/BindRelationTest.java @@ -18,6 +18,7 @@ package org.apache.doris.nereids.rules.analysis; import org.apache.doris.nereids.analyzer.UnboundRelation; +import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.pattern.GeneratedPlanPatterns; import org.apache.doris.nereids.rules.RulePromise; import org.apache.doris.nereids.trees.expressions.Alias; @@ -90,6 +91,27 @@ void bindByDbQualifier() { ((LogicalOlapScan) plan).qualified()); } + @Test + void rejectOptionsOnUnsupportedTableType() { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> PlanChecker.from(connectContext) + .analyze("SELECT * FROM db1.t@options('scan.snapshot-id'='1')")); + + Assertions.assertEquals( + "OPTIONS scan params are only supported for Paimon tables.", exception.getMessage()); + } + + @Test + void rejectOptionsOnCteReference() { + AnalysisException exception = Assertions.assertThrows(AnalysisException.class, + () -> PlanChecker.from(connectContext) + .analyze("WITH c AS (SELECT * FROM db1.t) " + + "SELECT * FROM c@options('scan.snapshot-id'='1')")); + + Assertions.assertEquals( + "Table scan parameters are not supported on CTE references.", exception.getMessage()); + } + @Test void bindSchemaTable() { boolean originValue = connectContext.getSessionVariable().isFetchAllFeForSystemTable(); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java index 1c6bbcfc25ef09..03563d5f89271a 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtilsTest.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.rules.exploration.mv; +import org.apache.doris.analysis.TableScanParams; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.TableIf; import org.apache.doris.mtmv.BaseTableInfo; @@ -24,14 +25,19 @@ import org.apache.doris.nereids.rules.exploration.mv.RelatedTableInfo.RelatedTableColumnInfo; import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; import org.apache.doris.nereids.util.PlanChecker; import org.apache.doris.utframe.TestWithFeService; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; +import java.util.Collections; import java.util.List; +import java.util.Optional; /** * Test for materialized view util @@ -921,6 +927,19 @@ public void containTableQueryOperatorWithoutOperatorTest() { }); } + @Test + public void containTableQueryOperatorWithFileScanParamsTest() { + LogicalFileScan fileScan = Mockito.mock(LogicalFileScan.class); + Mockito.when(fileScan.getTableSample()).thenReturn(Optional.empty()); + Mockito.when(fileScan.getScanParams()).thenReturn(Optional.of(new TableScanParams( + TableScanParams.OPTIONS, + ImmutableMap.of("scan.snapshot-id", "1"), + Collections.emptyList()))); + + Assertions.assertTrue(MaterializedViewUtils.TableQueryOperatorChecker.INSTANCE + .visitLogicalRelation(fileScan, null)); + } + @Test public void getRelatedTableInfoWhenMultiPartitionExprs() { PlanChecker.from(connectContext) diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommandTest.java new file mode 100644 index 00000000000000..eed86c07b5a6ee --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/ExecuteCommandTest.java @@ -0,0 +1,99 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans.commands; + +import org.apache.doris.analysis.TableScanParams; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.analyzer.UnboundRelation; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.OriginStatement; +import org.apache.doris.qe.PreparedStatementContext; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.qe.StmtExecutor; + +import com.google.common.collect.ImmutableMap; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +import java.util.Collections; +import java.util.concurrent.atomic.AtomicInteger; + +public class ExecuteCommandTest { + + @Test + public void testResolvedScanOptionsAreResetForEveryExecute() throws Exception { + String sql = "select * from p@options('scan.mode'='latest')"; + LogicalPlan logicalPlan = new NereidsParser().parseSingle(sql); + UnboundRelation relation = logicalPlan.collectToList( + UnboundRelation.class::isInstance).get(0); + TableScanParams scanParams = relation.getScanParams(); + AtomicInteger snapshotId = new AtomicInteger(); + + Assertions.assertEquals("1", resolveNextSnapshot(scanParams, snapshotId)); + + ConnectContext connectContext = Mockito.mock(ConnectContext.class); + StatementContext statementContext = new StatementContext(); + PrepareCommand prepareCommand = new PrepareCommand( + "stmt", logicalPlan, Collections.emptyList(), new OriginStatement(sql, 0)); + PreparedStatementContext preparedStatement = new PreparedStatementContext( + prepareCommand, connectContext, statementContext, "stmt"); + StmtExecutor executor = Mockito.mock(StmtExecutor.class); + Mockito.when(connectContext.getPreparedStementContext("stmt")).thenReturn(preparedStatement); + Mockito.when(connectContext.getSessionVariable()).thenReturn(new SessionVariable()); + Mockito.when(connectContext.getStatementContext()).thenReturn(statementContext); + Mockito.when(executor.getContext()).thenReturn(connectContext); + + new ExecuteCommand("stmt", prepareCommand, statementContext).run(connectContext, executor); + Assertions.assertEquals("2", resolveNextSnapshot(scanParams, snapshotId)); + + new ExecuteCommand("stmt", prepareCommand, statementContext).run(connectContext, executor); + Assertions.assertEquals("3", resolveNextSnapshot(scanParams, snapshotId)); + Mockito.verify(executor, Mockito.times(2)).execute(); + } + + @Test + public void testExecutePreparedCommandWithoutPlanChildren() throws Exception { + String sql = "show variables"; + LogicalPlan logicalPlan = new NereidsParser().parseSingle(sql); + + ConnectContext connectContext = Mockito.mock(ConnectContext.class); + StatementContext statementContext = new StatementContext(); + PrepareCommand prepareCommand = new PrepareCommand( + "stmt", logicalPlan, Collections.emptyList(), new OriginStatement(sql, 0)); + PreparedStatementContext preparedStatement = new PreparedStatementContext( + prepareCommand, connectContext, statementContext, "stmt"); + StmtExecutor executor = Mockito.mock(StmtExecutor.class); + Mockito.when(connectContext.getPreparedStementContext("stmt")).thenReturn(preparedStatement); + Mockito.when(connectContext.getSessionVariable()).thenReturn(new SessionVariable()); + Mockito.when(connectContext.getStatementContext()).thenReturn(statementContext); + Mockito.when(executor.getContext()).thenReturn(connectContext); + + new ExecuteCommand("stmt", prepareCommand, statementContext).run(connectContext, executor); + + Mockito.verify(executor).execute(); + } + + private String resolveNextSnapshot(TableScanParams scanParams, AtomicInteger snapshotId) { + return scanParams.getOrResolveMapParams(ignored -> ImmutableMap.of( + "scan.snapshot-id", String.valueOf(snapshotId.incrementAndGet()))) + .get("scan.snapshot-id"); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScanTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScanTest.java index 865bba61e1f3ad..b8199ea63b8467 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScanTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScanTest.java @@ -17,10 +17,12 @@ package org.apache.doris.nereids.trees.plans.logical; +import org.apache.doris.analysis.TableScanParams; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.Type; import org.apache.doris.datasource.iceberg.IcebergExternalTable; import org.apache.doris.datasource.iceberg.IcebergUtils; +import org.apache.doris.datasource.paimon.PaimonExternalTable; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions; @@ -31,6 +33,7 @@ import java.util.Arrays; import java.util.Collections; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.stream.Collectors; @@ -64,4 +67,25 @@ public void testComputeOutputIncludesInvisibleRowLineageColumnsForIcebergTable() IcebergUtils.ICEBERG_ROW_ID_COL, IcebergUtils.ICEBERG_LAST_UPDATED_SEQUENCE_NUMBER_COL), outputNames); } + + @Test + public void testPaimonOptionsBindRelationScopedSnapshotSchema() { + PaimonExternalTable table = Mockito.mock(PaimonExternalTable.class); + Mockito.when(table.getName()).thenReturn("paimon_tbl"); + Map options = Collections.singletonMap("scan.snapshot-id", "1"); + TableScanParams scanParams = new TableScanParams( + TableScanParams.OPTIONS, options, Collections.emptyList()); + Mockito.when(table.getFullSchema(scanParams)).thenReturn(Arrays.asList( + new Column("id", Type.INT, true), + new Column("old_name", Type.STRING, true))); + + LogicalFileScan scan = new LogicalFileScan(new RelationId(1), table, + Collections.singletonList("db"), Collections.emptyList(), + Optional.empty(), Optional.empty(), Optional.of(scanParams), Optional.empty()); + + Assertions.assertSame(SelectedPartitions.NOT_PRUNED, scan.getSelectedPartitions()); + Assertions.assertEquals(Arrays.asList("id", "old_name"), + scan.computeOutput().stream().map(slot -> slot.getName()).collect(Collectors.toList())); + Mockito.verify(table, Mockito.never()).initSelectedPartitions(Mockito.any()); + } } diff --git a/regression-test/data/external_table_p0/paimon/paimon_data_system_table.out b/regression-test/data/external_table_p0/paimon/paimon_data_system_table.out index 08ad3e823be19a..621b5e32313436 100644 --- a/regression-test/data/external_table_p0/paimon/paimon_data_system_table.out +++ b/regression-test/data/external_table_p0/paimon/paimon_data_system_table.out @@ -129,6 +129,15 @@ name text Yes true \N NONE 20 epsilon 30 zeta +-- !audit_log_incr -- ++U 2 beta_v2 + +-- !binlog_incr -- ++U 2 \N beta_v2 \N + +-- !binlog_incr_count -- +1 + -- !jni_binlog_rows -- +I 1 \N alpha \N +U 2 \N beta_v2 \N @@ -152,4 +161,3 @@ name text Yes true \N NONE +I 2 beta +I 3 gamma +I 4 delta - diff --git a/regression-test/data/external_table_p0/paimon/paimon_incr_read.out b/regression-test/data/external_table_p0/paimon/paimon_incr_read.out index ad49163cc36eb3..07e264fd6695de 100644 --- a/regression-test/data/external_table_p0/paimon/paimon_incr_read.out +++ b/regression-test/data/external_table_p0/paimon/paimon_incr_read.out @@ -1,4 +1,24 @@ -- This file is automatically generated. You should know what you did if you want to edit this +-- !options_snapshot1 -- +1 Alice 30 + +-- !options_snapshot2 -- +1 Alice 30 +2 Bob 25 + +-- !options_snapshot3 -- +1 Alice 30 +2 Bob 25 +3 Charlie 28 + +-- !options_isolation -- +1 Alice 30 +2 Bob 25 +3 Charlie 28 + +-- !relation_scoped_options -- +2 + -- !snapshot_incr3 -- 2 Bob 25 @@ -47,6 +67,26 @@ Alice 30 -- !join -- 2 Bob 25 2 Bob 25 +-- !options_snapshot1 -- +1 Alice 30 + +-- !options_snapshot2 -- +1 Alice 30 +2 Bob 25 + +-- !options_snapshot3 -- +1 Alice 30 +2 Bob 25 +3 Charlie 28 + +-- !options_isolation -- +1 Alice 30 +2 Bob 25 +3 Charlie 28 + +-- !relation_scoped_options -- +2 + -- !snapshot_incr3 -- 2 Bob 25 diff --git a/regression-test/data/external_table_p0/paimon/paimon_system_table.out b/regression-test/data/external_table_p0/paimon/paimon_system_table.out index d7223d69fb8348..c2363693014fd4 100644 --- a/regression-test/data/external_table_p0/paimon/paimon_system_table.out +++ b/regression-test/data/external_table_p0/paimon/paimon_system_table.out @@ -65,3 +65,12 @@ delta_record_count bigint Yes true \N NONE changelog_record_count bigint Yes true \N NONE watermark bigint Yes true \N NONE +-- !files_at_first_snapshot -- +1 + +-- !files_at_latest_snapshot -- +3 + +-- !files_without_options -- +3 + diff --git a/regression-test/data/external_table_p0/paimon/paimon_time_travel.out b/regression-test/data/external_table_p0/paimon/paimon_time_travel.out index effd42e0b15efa..0023cf71d86fcf 100644 --- a/regression-test/data/external_table_p0/paimon/paimon_time_travel.out +++ b/regression-test/data/external_table_p0/paimon/paimon_time_travel.out @@ -538,6 +538,12 @@ true 7 -- !expired_tag_count -- 2 +-- !expired_tag_options_count -- +2 + +-- !expired_version_options_count -- +2 + -- !branch_1_count_list -- 10 diff --git a/regression-test/data/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.out b/regression-test/data/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.out new file mode 100644 index 00000000000000..c6a8fa8ff608d1 --- /dev/null +++ b/regression-test/data/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.out @@ -0,0 +1,27 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !row_tracking_first_snapshot_incr -- +1 alpha 0 1 +2 beta 1 1 + +-- !options_snapshot_schema -- +1 alpha old-v1 10 + +-- !options_tag_schema -- +1 alpha old-v1 10 + +-- !audit_log_options_tag_schema -- ++I 1 alpha old-v1 10 + +-- !options_behavioral_latest_schema -- +1 alpha +2 beta +3 gamma +4 delta +5 epsilon +6 zeta + +-- !nested_options_snapshot_schema -- +1 10 20 30 + +-- !create_view_options_schema -- +1 alpha old-v1 10 diff --git a/regression-test/suites/external_table_p0/paimon/paimon_data_system_table.groovy b/regression-test/suites/external_table_p0/paimon/paimon_data_system_table.groovy index a9dc74aa16614e..5244a210c015e2 100644 --- a/regression-test/suites/external_table_p0/paimon/paimon_data_system_table.groovy +++ b/regression-test/suites/external_table_p0/paimon/paimon_data_system_table.groovy @@ -138,6 +138,7 @@ suite("paimon_data_system_table", "p0,external,doris,external_docker,external_do } sql """set force_jni_scanner=false""" + sql """set enable_paimon_cpp_reader=true""" // Paimon data system tables need Paimon-side semantics: // - binlog: pack/merge + array materialization // - audit_log: rowkind / sequence-number projection @@ -146,6 +147,25 @@ suite("paimon_data_system_table", "p0,external,doris,external_docker,external_do assertJniPath("select rowkind, id[1], name[1] from ${nativeTableName}\$binlog", "${nativeTableName}\$binlog") assertJniPath("select rowkind, id, name from ${tableName}\$audit_log", "${tableName}\$audit_log") assertJniPath("select rowkind, id, name from ${nativeTableName}\$audit_log", "${nativeTableName}\$audit_log") + assertJniPath( + "select rowkind, id, name from ${tableName}\$audit_log" + + "@incr('startSnapshotId'=1, 'endSnapshotId'=2)", + "${tableName}\$audit_log incremental") + + order_qt_audit_log_incr """ + select rowkind, id, name from ${tableName}\$audit_log + @incr('startSnapshotId'=1, 'endSnapshotId'=2) + order by id + """ + order_qt_binlog_incr """ + select rowkind, id[1], id[2], name[1], name[2] from ${tableName}\$binlog + @incr('startSnapshotId'=1, 'endSnapshotId'=2) + order by id[1] + """ + qt_binlog_incr_count """ + select count(*) from ${tableName}\$binlog + @incr('startSnapshotId'=1, 'endSnapshotId'=2) + """ assertCountStarPushdown("select count(*) from ${tableName}\$binlog", "${tableName}\$binlog") assertCountStarPushdown("select count(*) from ${tableName}\$audit_log", "${tableName}\$audit_log") @@ -163,6 +183,7 @@ suite("paimon_data_system_table", "p0,external,doris,external_docker,external_do qt_jni_native_binlog_rows """select rowkind, id[1], id[2], name[1], name[2] from ${nativeTableName}\$binlog order by id[1]""" qt_jni_native_audit_log_rows """select rowkind, id, name from ${nativeTableName}\$audit_log order by id""" } finally { + sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=false""" } } diff --git a/regression-test/suites/external_table_p0/paimon/paimon_incr_read.groovy b/regression-test/suites/external_table_p0/paimon/paimon_incr_read.groovy index 824d38f1cda40f..97094d4cf591dd 100644 --- a/regression-test/suites/external_table_p0/paimon/paimon_incr_read.groovy +++ b/regression-test/suites/external_table_p0/paimon/paimon_incr_read.groovy @@ -43,6 +43,61 @@ suite("test_paimon_incr_read", "p0,external,doris,external_docker,external_docke def test_incr_read = { String force -> sql """ set force_jni_scanner=${force} """ + + // Query-level OPTIONS params apply independently for both native and JNI readers. + order_qt_options_snapshot1 """ + select id, name, age from paimon_incr + @options('scan.snapshot-id'='1') order by id + """ + order_qt_options_snapshot2 """ + select id, name, age from paimon_incr + @options('scan.snapshot-id'='2') order by id + """ + order_qt_options_snapshot3 """ + select id, name, age from paimon_incr + @options('scan.snapshot-id'='3') order by id + """ + // The latest query proves that a prior relation's dynamic option did not leak. + order_qt_options_isolation """select id, name, age from paimon_incr order by id""" + + order_qt_relation_scoped_options """ + select right_orders.id + from paimon_incr + @options('scan.snapshot-id'='2') right_orders + left anti join paimon_incr + @options('scan.snapshot-id'='1') left_orders + on right_orders.id = left_orders.id + order by right_orders.id + """ + + test { + sql """ + select * from paimon_incr + @options('scan.snapshot-id'='999999') + """ + exception "snapshot" + } + test { + sql """select * from paimon_incr@options()""" + exception "OPTIONS requires a non-empty key/value map" + } + test { + sql """select * from paimon_incr@options(foo, bar)""" + exception "OPTIONS requires a non-empty key/value map" + } + test { + sql """select * from paimon_incr@options('scan.snapsh0t-id'='1')""" + exception "Unsupported Paimon query option" + } + test { + sql """select * from paimon_incr@options('scan.mode'='from-creation-timestamp')""" + exception "scan.creation-time-millis" + } + test { + sql """select * from paimon_incr@options('scan.bounded.watermark'='1')""" + exception "Unsupported Paimon query option" + } + order_qt_snapshot_incr3 """select * from paimon_incr@incr('startSnapshotId'=1, 'endSnapshotId'=2)""" order_qt_snapshot_incr4 """select * from paimon_incr@incr('startSnapshotId'=1, 'endSnapshotId'=3)""" order_qt_snapshot_incr5 """select * from paimon_incr@incr('startSnapshotId'=2, 'endSnapshotId'=3)""" @@ -107,5 +162,3 @@ suite("test_paimon_incr_read", "p0,external,doris,external_docker,external_docke // sql """drop catalog if exists ${catalog_name}""" } } - - diff --git a/regression-test/suites/external_table_p0/paimon/paimon_system_table.groovy b/regression-test/suites/external_table_p0/paimon/paimon_system_table.groovy index cbe138c71162b4..6eb1ae4e312cc1 100644 --- a/regression-test/suites/external_table_p0/paimon/paimon_system_table.groovy +++ b/regression-test/suites/external_table_p0/paimon/paimon_system_table.groovy @@ -151,7 +151,72 @@ suite("paimon_system_table", "p0,external,doris,external_docker,external_docker_ desc ${tableName}\$snapshots """ - // 2.6 system table does not support time travel + // 2.6 only system tables whose complete reader path observes a selected snapshot support OPTIONS. + String multiSnapshotTable = "test_paimon_incr_read_db.paimon_incr" + List> snapshotRows = sql """ + select snapshot_id from ${multiSnapshotTable}\$snapshots order by snapshot_id + """ + assertTrue(snapshotRows.size() > 1, "The regression table should contain multiple snapshots") + String firstSnapshotId = String.valueOf(snapshotRows.first()[0]) + String latestSnapshotId = String.valueOf(snapshotRows.last()[0]) + order_qt_files_without_options """ + select count(*) from ${multiSnapshotTable}\$files + """ + + for (String systemTable : ["files", "buckets"]) { + test { + sql """ + select count(*) from ${multiSnapshotTable}\$${systemTable} + @options('scan.snapshot-id'='${firstSnapshotId}') + """ + exception "does not support OPTIONS" + } + } + + // The fixture has no deletion vectors, but both reads must reach Paimon's index table + // instead of being rejected by Doris' OPTIONS capability gate. + assertNotNull(sql(""" + select count(*) from ${multiSnapshotTable}\$table_indexes + @options('scan.snapshot-id'='${firstSnapshotId}') + """)) + assertNotNull(sql(""" + select count(*) from ${multiSnapshotTable}\$table_indexes + @options('scan.snapshot-id'='${latestSnapshotId}') + """)) + + test { + sql """ + select count(*) from ${multiSnapshotTable}\$files + @incr('startSnapshotId'='${firstSnapshotId}', + 'endSnapshotId'='${latestSnapshotId}') + """ + exception "does not support INCR" + } + + List> incrementalPartitions = sql """ + select sum(record_count) from ${multiSnapshotTable}\$partitions + @incr('startSnapshotId'='${firstSnapshotId}', + 'endSnapshotId'='${latestSnapshotId}') + """ + assertEquals(2L, ((Number) incrementalPartitions[0][0]).longValue()) + + List incrementalRoIds = sql(""" + select id from ${multiSnapshotTable}\$ro + @incr('startSnapshotId'='${firstSnapshotId}', + 'endSnapshotId'='${latestSnapshotId}') + order by id + """).collect { row -> ((Number) row[0]).longValue() } + assertEquals([2L, 3L], incrementalRoIds) + + test { + sql """select * from ${tableName}\$snapshots@incr('startSnapshotId'=1, 'endSnapshotId'=2)""" + exception "does not support INCR" + } + test { + sql """select * from ${tableName}\$snapshots@options('scan.snapshot-id'='1')""" + exception "does not support OPTIONS" + } + test { sql """select * from ${tableName}\$snapshots FOR VERSION AS OF 1""" exception "Paimon system tables do not support time travel" @@ -160,10 +225,6 @@ suite("paimon_system_table", "p0,external,doris,external_docker,external_docker_ sql """select * from ${tableName}\$snapshots FOR TIME AS OF "2024-07-11 16:01:57.425" """ exception "Paimon system tables do not support time travel" } - test { - sql """select * from ${tableName}\$snapshots@incr('startSnapshotId'=1, 'endSnapshotId'=2)""" - exception "Paimon system tables do not support scan params" - } } catch (Exception e) { logger.error("Paimon system table test failed: " + e.getMessage()) diff --git a/regression-test/suites/external_table_p0/paimon/paimon_time_travel.groovy b/regression-test/suites/external_table_p0/paimon/paimon_time_travel.groovy index d91cc75abe57db..0b1705ee7929c2 100644 --- a/regression-test/suites/external_table_p0/paimon/paimon_time_travel.groovy +++ b/regression-test/suites/external_table_p0/paimon/paimon_time_travel.groovy @@ -155,6 +155,14 @@ suite("paimon_time_travel", "p0,external,doris,external_docker,external_docker_d // tag on expired snapshot should still be readable qt_expired_tag_count """select count(*) from ${tableName}_expired_tag FOR VERSION AS OF 't_exp_1';""" + qt_expired_tag_options_count """ + select count(*) from ${tableName}_expired_tag + @options('scan.tag-name'='t_exp_1'); + """ + qt_expired_version_options_count """ + select count(*) from ${tableName}_expired_tag + @options('scan.version'='t_exp_1'); + """ List> branchesResult = sql """ select branch_name from ${tableName}\$branches order by branch_name;""" logger.info("Query result from ${tableName}\$branches: ${branchesResult}") diff --git a/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy index 7ec5a571a211df..0783d60392813d 100644 --- a/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy +++ b/regression-test/suites/external_table_p0/paimon/test_paimon_schema_time_travel_matrix.groovy @@ -31,6 +31,9 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { String nestedTable = "nested_timeline" String pkTable = "pk_dv_timeline" String partitionTable = "partition_timeline" + String rowTrackingTable = "row_tracking_timeline" + String viewDb = "paimon_scan_options_view_db" + String historicalView = "historical_top_view" def latestSnapshotId = { String tableName -> List> rows = spark_paimon """ @@ -53,6 +56,20 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { """ } + def snapshotCommitTimeMillis = { String tableName, String snapshotId -> + List> rows = spark_paimon """ + -- Paimon's TIMESTAMP_NTZ commit time is a UTC wall clock; convert it to the + -- Spark session zone before casting so unix_millis keeps the actual instant. + SELECT unix_millis(CAST( + convert_timezone('UTC', current_timezone(), commit_time) AS TIMESTAMP + )) + FROM paimon.${dbName}.`${tableName}\$snapshots` + WHERE snapshot_id = ${snapshotId} + """ + assertEquals(1, rows.size()) + return rows[0][0].toString() + } + def assertUnknownColumn = { String query, String columnName -> test { sql query @@ -87,6 +104,17 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { try { spark_paimon_multi """ CREATE DATABASE IF NOT EXISTS paimon.${dbName}; + DROP TABLE IF EXISTS paimon.${dbName}.${rowTrackingTable}; + CREATE TABLE paimon.${dbName}.${rowTrackingTable} ( + id INT, + name STRING + ) USING paimon + TBLPROPERTIES ( + 'bucket'='-1', + 'row-tracking.enabled'='true' + ); + INSERT INTO paimon.${dbName}.${rowTrackingTable} + VALUES (1, 'alpha'), (2, 'beta'); DROP TABLE IF EXISTS paimon.${dbName}.${topTable}; CREATE TABLE paimon.${dbName}.${topTable} ( id INT, @@ -119,6 +147,7 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { VALUES (2, 'beta', 'old-v2', 20, 'added-v2', 200); """ String topCpAdd = latestSnapshotId(topTable) + String topCpAddTimeMillis = snapshotCommitTimeMillis(topTable, topCpAdd) createTag(topTable, "top_cp_add", topCpAdd) Thread.sleep(1100) @@ -337,6 +366,28 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { sql """use ${dbName}""" sql """refresh catalog ${catalogName}""" + // row_tracking exposes Paimon-generated hidden columns, so its incremental scan must + // bypass the C++ reader even when the session otherwise enables native Paimon scans. + sql """set force_jni_scanner=false""" + sql """set enable_paimon_cpp_reader=true""" + String rowTrackingExplain = sql(""" + explain verbose + select id, name, _ROW_ID, _SEQUENCE_NUMBER + from ${rowTrackingTable}\$row_tracking + @incr('startSnapshotId'=0, 'endSnapshotId'=1) + """).collect { row -> row[0].toString() }.join("\n") + assertTrue(rowTrackingExplain.contains("paimonNativeReadSplits=0/"), + "row_tracking incremental scan must not select native Paimon splits") + assertTrue(rowTrackingExplain.contains("SplitStat [type=JNI"), + "row_tracking incremental scan must use JNI splits") + order_qt_row_tracking_first_snapshot_incr """ + select id, name, _ROW_ID, _SEQUENCE_NUMBER + from ${rowTrackingTable}\$row_tracking + @incr('startSnapshotId'=0, 'endSnapshotId'=1) + order by id + """ + sql """set enable_paimon_cpp_reader=false""" + // Scenario TC01: validate latest schema/data, explicit new binding, predicate and aggregate. assertEquals([[1, null], [2, null], [3, null], [4, null], [5, 5000L], [6, 6000L]], sql("""select id, victim from ${topTable} order by id""")) @@ -363,6 +414,34 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { from ${topTable}@tag(top_cp0) order by id """)) + order_qt_options_snapshot_schema """ + select id, old_name, victim, metric + from ${topTable}@options('scan.snapshot-id'='${topCp0}') + order by id + """ + order_qt_options_tag_schema """ + select id, old_name, victim, metric + from ${topTable}@options('scan.tag-name'='top_cp0') + order by id + """ + assertEquals([[1, "alpha", "old-v1", 10], [2, "beta", "old-v2", 20]], + sql(""" + select id, old_name, victim, metric + from ${topTable}@options( + 'scan.creation-time-millis'='${topCpAddTimeMillis}' + ) + order by id + """)) + order_qt_audit_log_options_tag_schema """ + select rowkind, id, old_name, victim, metric + from ${topTable}\$audit_log@options('scan.tag-name'='top_cp0') + order by id + """ + order_qt_options_behavioral_latest_schema """ + select id, MixedName + from ${topTable}@options('scan.plan-sort-partition'='true') + order by id + """ assertEquals(topCp0Rows, sql(""" select id, old_name, victim, metric from ${topTable}@branch(top_cp0_branch) @@ -433,6 +512,13 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { from ${nestedTable} for version as of ${nestedCp0} where payload.old_child = 10 """)) + order_qt_nested_options_snapshot_schema """ + select id, payload.old_child, + element_at(attributes, 'a').old_child, + events[1].old_child + from ${nestedTable}@options('scan.snapshot-id'='${nestedCp0}') + where payload.old_child = 10 + """ assertEquals([[1, null, null, null], [2, 112, 122, 132]], sql(""" select id, payload.added_child, @@ -540,6 +626,61 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { assertEquals([[1, "p1", "old"], [2, "p2", "old"], [3, "p3", "new"]], sql("""select id, old_partition, new_payload from ${partitionTable} order by id""")) + // OPTIONS must survive CREATE VIEW rewriting, while unsupported consumers fail explicitly. + sql """create database if not exists internal.${viewDb}""" + sql """drop view if exists internal.${viewDb}.${historicalView}""" + sql """ + create view internal.${viewDb}.${historicalView} as + select id, old_name, victim, metric + from ${catalogName}.${dbName}.${topTable} + @options('scan.snapshot-id'='${topCp0}') + """ + order_qt_create_view_options_schema """ + select id, old_name, victim, metric + from internal.${viewDb}.${historicalView} + order by id + """ + test { + sql """ + with historical as ( + select id from ${topTable} + ) + select * from historical@options('scan.snapshot-id'='${topCp0}') + """ + exception "Table scan parameters are not supported on CTE references" + } + test { + sql """ + show replica distribution from ${catalogName}.${dbName}.${topTable} + @options('scan.snapshot-id'='${topCp0}') + """ + exception "OPTIONS scan params are only supported in query relations" + } + test { + sql """ + select * from ${topTable}@options( + 'scan.snapshot-id'='${topCp0}', + 'scan.creation-time-millis'='0' + ) + """ + exception "Only one Paimon startup position can be specified" + } + test { + sql """ + select * from ${topTable}@options( + 'scan.mode'='latest', + 'scan.snapshot-id'='${topCp0}' + ) + """ + exception "is incompatible with startup position" + } + test { + sql """ + select * from ${topTable}@options('scan.fallback-branch'='archive') + """ + exception "scan.fallback-branch" + } + // Scenario TC09/R13/R17: cache, JNI and CPP paths return the same historical schema/data. sql """switch ${noCacheCatalogName}""" sql """use ${dbName}""" @@ -565,6 +706,14 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { order by id """) assertEquals(forcedJniRows, cppRows) + // Schema-selecting OPTIONS must bypass paimon-cpp, whose table handle always uses the + // latest schema, even when native Paimon scans are enabled for the session. + List> cppOptionsRows = sql(""" + select id, old_name, victim, metric + from ${topTable}@options('scan.snapshot-id'='${topCp0}') + order by id + """) + assertEquals(forcedJniRows, cppOptionsRows) // Scenario T10/T11: retained tags survive expiration; missing refs never fall back to latest. spark_paimon """ @@ -594,6 +743,7 @@ suite("test_paimon_schema_time_travel_matrix", "p0,external,paimon") { } finally { sql """set enable_paimon_cpp_reader=false""" sql """set force_jni_scanner=false""" + sql """drop database if exists internal.${viewDb} force""" sql """drop catalog if exists ${catalogName}""" sql """drop catalog if exists ${noCacheCatalogName}""" } From 10f40b377b29af74aea768a73277abed909d2748 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 28 Jul 2026 10:49:34 +0800 Subject: [PATCH 4/5] [test](iceberg) Add partition evolution schema coverage (#65870) ### What problem does this PR solve? Related PR: #65502 Problem Summary: #65502 already fixed the Iceberg scan correctness issue by sending the complete current table schema to readers. Although that change was introduced for equality-delete dependencies, the same all-column schema invariant is also required for partition evolution: a column classified as an identity partition column by a newer partition spec can still be stored physically in files written by an older spec. This follow-up does not change scan behavior. It documents the partition-evolution invariant, extracts the existing schema initialization into a testable helper, and adds a focused FE unit test verifying that a non-projected evolved partition column remains in the reader schema. ### Release note None ### Check List (For Author) - Test - [ ] Regression test - [x] Unit Test - [ ] Manual test (add detailed scripts or steps below) - [ ] No need to test or manual test. Explain why: - [ ] This is a refactor/code format and no logic has been changed. - [ ] Previous test can cover this change. - [ ] No code files have been changed. - [ ] Other reason - Behavior changed: - [x] No. - [ ] Yes. - Does this need documentation? - [x] No. - [ ] Yes. ### Check List (For Reviewer who merge this PR) - [ ] Confirm the release note - [ ] Confirm test cases - [ ] Confirm document - [ ] Add branch pick label --- .../iceberg/source/IcebergScanNode.java | 7 +++- .../iceberg/source/IcebergScanNodeTest.java | 42 +++++++++++++++++++ 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java index d5ee31ace104d7..0eae2b93736550 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergScanNode.java @@ -489,11 +489,16 @@ public void createScanRangeLocations() throws UserException { super.createScanRangeLocations(); enableCurrentIcebergScanSemantics(); // Extract name mapping from Iceberg table properties - Optional>> nameMapping = extractNameMapping(); + initializeIcebergSchemaInfo(extractNameMapping()); + } + @VisibleForTesting + void initializeIcebergSchemaInfo(Optional>> nameMapping) throws UserException { // Equality-delete keys are hidden scan dependencies and need not appear in the query // projection. Both scanners need the complete current schema to resolve field ids, // historical names, types, and initial defaults when an old data file lacks such a key. + // An identity partition column can also be a physical field in files written by an older + // partition spec, so preserving the complete schema is required for partition evolution. ExternalUtil.initSchemaInfoForAllColumn(params, -1L, source.getTargetTable().getColumns(), nameMapping.orElse(Collections.emptyMap()), nameMapping.isPresent(), getBase64EncodedInitialDefaultsForScan()); diff --git a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java index 504f0f48d090f8..bab7e8085fbbe3 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/datasource/iceberg/source/IcebergScanNodeTest.java @@ -17,13 +17,16 @@ package org.apache.doris.datasource.iceberg.source; +import org.apache.doris.analysis.SlotDescriptor; import org.apache.doris.analysis.SlotId; import org.apache.doris.analysis.TableScanParams; import org.apache.doris.analysis.TableSnapshot; import org.apache.doris.analysis.TupleDescriptor; import org.apache.doris.analysis.TupleId; +import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.TableIf; +import org.apache.doris.catalog.Type; import org.apache.doris.common.UserException; import org.apache.doris.common.util.LocationPath; import org.apache.doris.datasource.CatalogIf; @@ -126,6 +129,12 @@ public List getPathPartitionKeys() { return Collections.emptyList(); } + void addSlot(int slotId, Column column) { + SlotDescriptor slot = new SlotDescriptor(new SlotId(slotId), desc); + slot.setColumn(column); + desc.addSlot(slot); + } + @Override public TableSnapshot getQueryTableSnapshot() { return tableSnapshot; @@ -141,6 +150,12 @@ int enableAndGetIcebergScanSemanticsVersion() { enableCurrentIcebergScanSemantics(); return params.getIcebergScanSemanticsVersion(); } + + TFileScanRangeParams initializeAndGetIcebergSchemaInfo() throws UserException { + params = new TFileScanRangeParams(); + initializeIcebergSchemaInfo(Optional.empty()); + return params; + } } @Test @@ -151,6 +166,33 @@ public void testEmitsCurrentIcebergScanSemanticsCapability() { node.enableAndGetIcebergScanSemanticsVersion()); } + @Test + public void testPartitionEvolutionKeepsNonFileSlotInReaderSchema() throws Exception { + Column evolvedIdentityColumn = new Column("int_col", Type.BIGINT, true); + evolvedIdentityColumn.setUniqueId(1); + Column projectedColumn = new Column("payload", Type.STRING, true); + projectedColumn.setUniqueId(2); + + IcebergExternalTable targetTable = Mockito.mock(IcebergExternalTable.class); + Mockito.when(targetTable.getColumns()).thenReturn( + ImmutableList.of(evolvedIdentityColumn, projectedColumn)); + IcebergSource source = Mockito.mock(IcebergSource.class); + Mockito.when(source.getTargetTable()).thenReturn(targetTable); + + TestIcebergScanNode node = Mockito.spy(new TestIcebergScanNode(new SessionVariable())); + node.addSlot(1, projectedColumn); + setIcebergSource(node, source); + Mockito.doReturn(Collections.emptyMap()).when(node).getBase64EncodedInitialDefaultsForScan(); + + TFileScanRangeParams scanParams = node.initializeAndGetIcebergSchemaInfo(); + + Assert.assertEquals(2, scanParams.getHistorySchemaInfo().get(0).getRootField().getFieldsSize()); + Assert.assertEquals("int_col", scanParams.getHistorySchemaInfo().get(0).getRootField() + .getFields().get(0).getFieldPtr().getName()); + Assert.assertEquals("payload", scanParams.getHistorySchemaInfo().get(0).getRootField() + .getFields().get(1).getFieldPtr().getName()); + } + @Test public void testExtractNameMappingDistinguishesAbsentAndEmpty() throws Exception { TestIcebergScanNode node = new TestIcebergScanNode(new SessionVariable()); From ea7a992bd759ae6ca6f7489d7bfcbf5e414f38e5 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Tue, 28 Jul 2026 21:17:48 +0800 Subject: [PATCH 5/5] [fix](style) Separate method definitions in Paimon backport --- .../src/main/java/org/apache/doris/analysis/TableScanParams.java | 1 + .../doris/nereids/trees/plans/logical/LogicalFileScan.java | 1 + 2 files changed, 2 insertions(+) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableScanParams.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableScanParams.java index 08743f31c84f58..89aca0e9daa132 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/TableScanParams.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/TableScanParams.java @@ -115,6 +115,7 @@ public boolean isBranch() { public boolean isTag() { return TAG.equals(paramType); } + public boolean isOptions() { return OPTIONS.equals(paramType); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java index d8cac61939aebd..a3397b8c6823f9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java @@ -110,6 +110,7 @@ private static SelectedPartitions initialSelectedPartitions( } return table.initSelectedPartitions(MvccUtil.getSnapshotFromContext(table)); } + public SelectedPartitions getSelectedPartitions() { return selectedPartitions; }