From 58a6c27e9473b7fece7099cdd02fe44b93ba0f94 Mon Sep 17 00:00:00 2001 From: Gabriel Date: Sat, 25 Jul 2026 16:50:30 +0800 Subject: [PATCH] [test](regression) Expand Iceberg write evolution coverage (#66021) ### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Iceberg write P0 coverage did not systematically combine schema evolution, partition evolution, snapshot/tag/branch references, row-level DML, overwrite, Doris source models, partition transforms, file formats, distribution properties, complex types, concurrency, and NULL semantics. This PR adds independent suites so dimensions can run concurrently, compares Doris and Spark results on the same Doris-written Iceberg tables, and isolates confirmed negative reproductions behind explicit opt-in switches. ### What changed? - Cover schema and partition evolution with snapshot, tag, branch, overwrite, and historical reads. - Cover ARRAY, MAP, and STRUCT evolution with nested NULL values. - Cover MOR and COW DELETE, UPDATE, and MERGE behavior, including duplicate-source cardinality and STRING truncate boundaries. - Cover current-spec and branch overwrite, delete-file interactions, failed-write atomicity, and retries. - Cover Duplicate, Unique MOW, Unique MOR, and Aggregate sources with RANGE, LIST, unpartitioned, HASH, RANDOM, and AUTO bucket layouts. - Cover STRING and numeric partition transforms, UTF-8 edge values, NULL partitions, and transform metadata. - Cover CTAS, Parquet and ORC writes, Avro rejection, sort order, distribution mode, multi-file flush, and concurrent commits. - Add a P0 coverage matrix and deterministic expected outputs. - Modify regression tests and test documentation only; no production code is changed. ### Release note None ### Check List (For Author) - Test - [x] Regression test - Built FE and BE from the tested master commit. - Validated with one FE and two BEs. - Final validation batch: 11 suites, 0 failed, 0 fatal. - Cross-checked Doris and Spark queries on the same Iceberg tables for all positive scenarios. - Confirmed defect reproductions are isolated behind explicit switches so default P0 remains safe. - Behavior changed: - [x] No. - Does this need documentation? - [x] No. --- ...test_iceberg_write_branch_dml_boundary.out | 12 + .../test_iceberg_write_complex_evolution.out | 30 ++ ...berg_write_concurrent_merge_invariants.out | 4 + ...est_iceberg_write_ctas_format_boundary.out | 16 + ...test_iceberg_write_dml_modes_evolution.out | 32 ++ .../test_iceberg_write_evolution_refs.out | 75 +++++ ..._write_merge_duplicate_source_negative.out | 3 + .../test_iceberg_write_merge_semantics.out | 13 + ..._iceberg_write_merge_truncate_negative.out | 9 + ...st_iceberg_write_nullability_atomicity.out | 7 + ...eberg_write_nullable_truncate_negative.out | 13 + ...rg_write_order_distribution_properties.out | 14 + ...test_iceberg_write_overwrite_atomicity.out | 23 ++ ...t_iceberg_write_overwrite_delete_files.out | 40 +++ ...test_iceberg_write_overwrite_evolution.out | 45 +++ ...est_iceberg_write_partition_types_null.out | 64 ++++ .../test_iceberg_write_source_models.out | 26 ++ ...ceberg_write_string_transform_metadata.out | 40 +++ .../write/ICEBERG_WRITE_P0_COVERAGE.md | 120 ++++++++ ...t_iceberg_write_branch_dml_boundary.groovy | 169 +++++++++++ ...est_iceberg_write_complex_evolution.groovy | 178 +++++++++++ ...g_write_concurrent_merge_invariants.groovy | 209 +++++++++++++ ..._iceberg_write_ctas_format_boundary.groovy | 180 +++++++++++ ...t_iceberg_write_dml_modes_evolution.groovy | 269 ++++++++++++++++ .../test_iceberg_write_evolution_refs.groovy | 253 +++++++++++++++ ...ite_merge_duplicate_source_negative.groovy | 102 +++++++ .../test_iceberg_write_merge_semantics.groovy | 207 +++++++++++++ ...eberg_write_merge_truncate_negative.groovy | 100 ++++++ ...iceberg_write_nullability_atomicity.groovy | 133 ++++++++ ...rg_write_nullable_truncate_negative.groovy | 97 ++++++ ...write_order_distribution_properties.groovy | 215 +++++++++++++ ...t_iceberg_write_overwrite_atomicity.groovy | 163 ++++++++++ ...ceberg_write_overwrite_delete_files.groovy | 197 ++++++++++++ ...t_iceberg_write_overwrite_evolution.groovy | 180 +++++++++++ ..._iceberg_write_partition_types_null.groovy | 287 ++++++++++++++++++ ...write_required_null_select_negative.groovy | 103 +++++++ ...write_required_null_values_negative.groovy | 81 +++++ .../test_iceberg_write_source_models.groovy | 220 ++++++++++++++ ...erg_write_string_transform_metadata.groovy | 212 +++++++++++++ 39 files changed, 4141 insertions(+) create mode 100644 regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_branch_dml_boundary.out create mode 100644 regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_complex_evolution.out create mode 100644 regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_concurrent_merge_invariants.out create mode 100644 regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_ctas_format_boundary.out create mode 100644 regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_dml_modes_evolution.out create mode 100644 regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_evolution_refs.out create mode 100644 regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_merge_duplicate_source_negative.out create mode 100644 regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_merge_semantics.out create mode 100644 regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_merge_truncate_negative.out create mode 100644 regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_nullability_atomicity.out create mode 100644 regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_nullable_truncate_negative.out create mode 100644 regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_order_distribution_properties.out create mode 100644 regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_overwrite_atomicity.out create mode 100644 regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_overwrite_delete_files.out create mode 100644 regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_overwrite_evolution.out create mode 100644 regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_partition_types_null.out create mode 100644 regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_source_models.out create mode 100644 regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_string_transform_metadata.out create mode 100644 regression-test/suites/external_table_p0/iceberg/write/ICEBERG_WRITE_P0_COVERAGE.md create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_branch_dml_boundary.groovy create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_complex_evolution.groovy create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_concurrent_merge_invariants.groovy create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_ctas_format_boundary.groovy create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_dml_modes_evolution.groovy create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_evolution_refs.groovy create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_duplicate_source_negative.groovy create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_semantics.groovy create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_truncate_negative.groovy create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_nullability_atomicity.groovy create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_nullable_truncate_negative.groovy create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_order_distribution_properties.groovy create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_overwrite_atomicity.groovy create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_overwrite_delete_files.groovy create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_overwrite_evolution.groovy create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_partition_types_null.groovy create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_required_null_select_negative.groovy create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_required_null_values_negative.groovy create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_source_models.groovy create mode 100644 regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_string_transform_metadata.groovy diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_branch_dml_boundary.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_branch_dml_boundary.out new file mode 100644 index 00000000000000..5795463b78bf0c --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_branch_dml_boundary.out @@ -0,0 +1,12 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !branch_write -- +1 A main +2 B branch-insert +3 C branch-overwrite +-- !main_after_branch_write -- +1 A main + +-- !branch_after_rejected_dml -- +1 A main +2 B branch-insert +3 C branch-overwrite diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_complex_evolution.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_complex_evolution.out new file mode 100644 index 00000000000000..76bede82d3f5fb --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_complex_evolution.out @@ -0,0 +1,30 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !complex_current -- +1 A [1, null, 3] {"x":10, "null-value":null} {"metric":10, "label":"old-a", "nested":{"count":1, "comment":null, "score":null}, "tags":null, "attributes":null} +2 N \N {"x":null} {"metric":20, "label":null, "nested":{"count":null, "comment":"old-null", "score":null}, "tags":null, "attributes":null} +3 B [] {} \N +4 A1 [4000000000, null] {"large":5000000000, "null-value":null} {"metric":6000000000, "label":"new-a", "nested":{"count":7000000000, "comment":"nested-new", "score":7.5}, "tags":["x", null, "z"], "attributes":{"a":8000000000, "b":null}} +5 N2 [null] \N {"metric":50, "label":null, "nested":{"count":5, "comment":null, "score":null}, "tags":null, "attributes":{"null-value":null}} + +-- !complex_children -- +1 10 1 \N \N \N +2 20 \N \N \N \N +3 \N \N \N \N \N +4 6000000000 7000000000 7.5 ["x", null, "z"] {"a":8000000000, "b":null} +5 50 5 \N \N {"null-value":null} + +-- !complex_nulls -- +1 +2 +3 +5 + +-- !complex_partition_specs -- +0 3 +2 2 + +-- !complex_base_tag -- +1 [1, null, 3] {"x":10, "null-value":null} 10 old-a 1 \N +2 \N {"x":null} 20 \N \N old-null +3 [] {} \N \N \N \N + diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_concurrent_merge_invariants.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_concurrent_merge_invariants.out new file mode 100644 index 00000000000000..2ab2c582fe3762 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_concurrent_merge_invariants.out @@ -0,0 +1,4 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !concurrent_append_counts -- +append-one 128 128 +append-two 128 128 diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_ctas_format_boundary.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_ctas_format_boundary.out new file mode 100644 index 00000000000000..4a418a9b8cd01e --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_ctas_format_boundary.out @@ -0,0 +1,16 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !ctas_complex_rows -- +1 A ["x", null] {"k":"v"} {"score":10, "note":"one"} +2 \N [] {"null-value":null} {"score":null, "note":"two"} +3 A ["😀"] {} {"score":30, "note":null} + +-- !ctas_complex_files -- +orc 3 + +-- !ctas_complex_partitions -- +0 3 3 + +-- !ctas_complex_physical_partitions -- +\N 0 1 +A 0 1 +A 3 1 diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_dml_modes_evolution.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_dml_modes_evolution.out new file mode 100644 index 00000000000000..98c60ec0eb83ee --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_dml_modes_evolution.out @@ -0,0 +1,32 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !mor_current -- +1 A-updated alpha 2026-01-01T01:00 110 updated +2 B-merged beta-merged 2026-04-02T02:00 220 merged +4 A-updated delta 2026-01-04T04:00 140 updated +7 C golf 2026-03-01T07:00 70 \N +8 D hotel 2026-05-01T08:00 80 inserted + +-- !mor_base_tag -- +1 A alpha 2026-01-01T01:00 10 +2 B beta 2026-01-02T02:00 20 +3 \N null-key \N 30 +4 A delta 2026-01-04T04:00 40 + +-- !mor_before_dml_tag -- +1 A alpha 2026-01-01T01:00 10 \N +2 B beta 2026-01-02T02:00 20 \N +3 \N null-key \N 30 \N +4 A delta 2026-01-04T04:00 40 \N +5 B echo 2026-02-01T05:00 50 new-spec +6 \N foxtrot \N 60 new-null +7 C golf 2026-03-01T07:00 70 \N + +-- !mor_delete_files -- +0 4 4 +2 2 2 + +-- !cow_after_rejections -- +1 A alpha 2026-01-01T01:00 10 base +2 \N null-key \N 20 null-partition +3 B beta 2026-02-01T03:00 30 new-spec + diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_evolution_refs.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_evolution_refs.out new file mode 100644 index 00000000000000..f7380c14c7ded5 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_evolution_refs.out @@ -0,0 +1,75 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !current_rows -- +1 CN alpha 2026-01-01T08:00 10.10 10 \N \N +2 US beta 2026-01-02T09:00 20.20 20 \N \N +3 \N null-key \N 30.30 30 \N \N +4 CN-east gamma 2026-02-01T10:00 40.40 4000000000 after-evolution new-spec +5 DE-west delta 2026-03-02T11:00 50.50 50 \N \N +6 \N epsilon \N 60.60 60 null-partition null-zone + +-- !cross_spec_zone_filter -- +1 +4 + +-- !cross_spec_time_filter -- +3 +4 +5 +6 + +-- !partition_specs -- +0 3 +4 3 + +-- !base_snapshot -- +1 CN +2 US +3 \N + +-- !base_tag -- +1 CN +2 US +3 \N + +-- !evolved_tag -- +1 CN \N +2 US \N +3 \N \N +4 CN-east new-spec +5 DE-west \N +6 \N null-zone + +-- !branch_after_insert -- +1 CN \N +2 US \N +3 \N \N +7 JP-east branch-insert +8 FR-west branch-overwrite-seed + +-- !main_unchanged_after_branch_insert -- +1 +2 +3 +4 +5 +6 + +-- !branch_after_overwrite -- +1 CN \N +2 US \N +3 \N \N +7 JP-east branch-insert +8 FR-west branch-overwrite + +-- !base_tag_after_branch_overwrite -- +1 CN +2 US +3 \N + +-- !main_after_branch_overwrite -- +1 CN \N +2 US \N +3 \N \N +4 CN-east new-spec +5 DE-west \N +6 \N null-zone diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_merge_duplicate_source_negative.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_merge_duplicate_source_negative.out new file mode 100644 index 00000000000000..3f8c510addc24d --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_merge_duplicate_source_negative.out @@ -0,0 +1,3 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !duplicate_source_atomic_state -- +1 A committed diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_merge_semantics.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_merge_semantics.out new file mode 100644 index 00000000000000..2d9051a0ac1cd0 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_merge_semantics.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !merge_conditional_clauses -- +1 A2 bucket-a2 delta new-1 updated +2 \N bucket-b beta old-2 active +4 \N bucket-d echo new-4 insert-1 +5 E bucket-e foxtrot new-5 insert-2 + +-- !merge_string_partition_metadata -- +0 6 6 + +-- !merge_null_keys -- +\N \N \N \N source-null-safe null-safe-update +\N \N \N \N source-ordinary ordinary-insert diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_merge_truncate_negative.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_merge_truncate_negative.out new file mode 100644 index 00000000000000..6a9d3165c3474d --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_merge_truncate_negative.out @@ -0,0 +1,9 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !merge_truncate_after_fix -- +1 beta after +2 gamma inserted + +-- !merge_truncate_physical_partitions -- +616C +6265 +6761 diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_nullability_atomicity.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_nullability_atomicity.out new file mode 100644 index 00000000000000..19fc09fe26cf1c --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_nullability_atomicity.out @@ -0,0 +1,7 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !required_after_retry -- +1 committed \N +2 valid-select \N +4 valid-after-invalid value +5 valid-values-retry \N + diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_nullable_truncate_negative.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_nullable_truncate_negative.out new file mode 100644 index 00000000000000..bb09eac2571535 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_nullable_truncate_negative.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !nullable_truncate_rows -- +1 CN +2 \N +3 US-west +4 \N + +-- !nullable_truncate_physical_partitions -- +0 \N \N +0 434E \N +1 \N \N +1 55532D65617374 5553 +1 55532D77657374 5553 diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_order_distribution_properties.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_order_distribution_properties.out new file mode 100644 index 00000000000000..eb0ba541b0e826 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_order_distribution_properties.out @@ -0,0 +1,14 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !ordered_evolution_changed_rows -- +1 R-updated payload-1-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 10001 updated +10001 R-new merge-insert 10001 inserted +2 \N merge-update 20002 merged +7 R-updated payload-7-xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx 10007 updated + +-- !ordered_evolution_files -- +parquet 10007 + +-- !distribution_mode_counts -- +hash 512 +none 512 +range 512 diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_overwrite_atomicity.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_overwrite_atomicity.out new file mode 100644 index 00000000000000..7cfaa975e3e384 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_overwrite_atomicity.out @@ -0,0 +1,23 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !overwrite_failure_state -- +1 A committed-a +2 B committed-b + +-- !branch_overwrite_failure_state -- +1 A committed-a +2 B committed-b + +-- !main_after_branch_overwrite_failure -- +1 A committed-a +2 B committed-b + +-- !overwrite_retry -- +10 A candidate-0 1 +11 C candidate-1 1 +12 A candidate-2 1 +13 C candidate-3 1 +14 A candidate-4 1 +15 C candidate-5 1 +16 A candidate-6 1 +17 C candidate-7 1 +2 B committed-b 1 diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_overwrite_delete_files.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_overwrite_delete_files.out new file mode 100644 index 00000000000000..8ab5949d05b624 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_overwrite_delete_files.out @@ -0,0 +1,40 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !before_overwrite_rows -- +1 A alpha keep-a +3 C gamma-new moved-to-c +5 \N null-key keep-null +6 B echo merge-insert-b + +-- !before_overwrite_delete_files -- +0 3 + +-- !after_overwrite_rows -- +10 A alpha replacement-a +11 B echo replacement-b +3 C gamma-new moved-to-c +5 \N null-key keep-null + +-- !after_overwrite_delete_files -- +0 1 + +-- !before_row_dml_tag -- +1 A alpha keep-a +2 A beta delete-a +3 B gamma move-b-to-c +4 B delta merge-delete-b +5 \N null-key keep-null + +-- !evolved_overwrite_rows -- +10 A alpha replacement-a +11 B echo replacement-b +13 \N null-new new-spec-null +14 A alpha-new new-spec-replacement-a +3 C gamma-new moved-to-c +5 \N null-key keep-null + +-- !evolved_overwrite_specs -- +0 5 5 +2 2 2 + +-- !evolved_overwrite_delete_files -- +0 1 diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_overwrite_evolution.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_overwrite_evolution.out new file mode 100644 index 00000000000000..a5095dbb0ef041 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_overwrite_evolution.out @@ -0,0 +1,45 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !overwrite_current -- +1 A alpha 2026-01-01T01:10 old-hour-1 +2 A beta 2026-01-01T02:20 old-hour-2 +3 \N null-region \N old-null +4 B delta 2026-01-02T01:30 old-other-day +6 \N null-new \N new-spec-null +7 C charlie 2026-02-01T03:00 new-spec-other-month +8 A alpha-new 2026-01-01T01:40 overwrite-current-spec + +-- !overwrite_specs -- +0 3 4 +3 3 3 + +-- !overwrite_base_tag -- +1 A alpha 2026-01-01T01:10 old-hour-1 +2 A beta 2026-01-01T02:20 old-hour-2 +3 \N null-region \N old-null +4 B delta 2026-01-02T01:30 old-other-day +-- !overwrite_audit_branch -- +1 A alpha 2026-01-01T01:10 old-hour-1 +2 A beta 2026-01-01T02:20 old-hour-2 +3 \N null-region \N old-null +4 B delta 2026-01-02T01:30 old-other-day + +-- !overwrite_after_drop_identity -- +1 A alpha 2026-01-01T01:10 old-hour-1 +2 A beta 2026-01-01T02:20 old-hour-2 +3 \N null-region \N old-null +4 B delta 2026-01-02T01:30 old-other-day +6 \N null-new \N new-spec-null +7 C charlie 2026-02-01T03:00 new-spec-other-month +8 A alpha-new 2026-01-01T01:40 overwrite-current-spec +9 \N null-new \N overwrite-null-current-spec + +-- !overwrite_after_drop_identity_specs -- +0 3 4 +3 3 3 +5 1 1 + +-- !overwrite_base_tag_after_second_evolution -- +1 A alpha 2026-01-01T01:10 old-hour-1 +2 A beta 2026-01-01T02:20 old-hour-2 +3 \N null-region \N old-null +4 B delta 2026-01-02T01:30 old-other-day diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_partition_types_null.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_partition_types_null.out new file mode 100644 index 00000000000000..552e5319a9d9b7 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_partition_types_null.out @@ -0,0 +1,64 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !string_rows -- +1 alpha bucket-a alpha a +2 alphabet bucket-b alphabet ab +3 bucket-empty empty +4 中文 bucket-unicode 中文 unicode +5 \N bucket-null-identity null-identity null-string + +-- !string_null_filter -- +5 + +-- !string_cross_spec_filter -- +1 +2 +5 +7 + +-- !string_cross_spec_bucket_filter -- +1 +6 + +-- !string_partition_specs -- +0 5 +1 2 + +-- !numeric_rows -- +1 1 101 11.11 true positive +2 -1 -101 -11.11 false negative +3 0 0 0.00 \N zero-null-bool +4 \N \N \N \N all-null + +-- !numeric_null_filter -- +3 +4 + +-- !numeric_partitions -- +0 4 + +-- !numeric_physical_partitions -- +\N \N \N \N \N \N 1 +0 4 0 7 0.00 \N 1 +0 4 100 1 11.10 true 1 +0 5 -200 4 -11.20 false 1 + +-- !temporal_rows -- +1 1969-12-31 1969-12-31 1969-12-31 1969-12-31T23:59:59 1969-12-31T23:59:59 1969-12-31T23:59:59 before-epoch +2 1970-01-01 1970-01-01 1970-01-01 1970-01-01T00:00 1970-01-01T00:00 1970-01-01T00:00 epoch +3 2024-02-29 2024-02-29 2024-02-29 2024-02-29T12:34:56 2024-02-29T12:34:56 2024-02-29T12:34:56 leap-day +4 \N \N \N \N \N \N all-null + +-- !temporal_filters -- +1 +3 +4 + +-- !temporal_partitions -- +0 4 + +-- !temporal_physical_partitions -- +\N \N \N \N \N \N 1 +0 0 0 7 1970-01-01 0 1 +4 0 0 4 1970-01-01 0 1 +6 54 649 4 2024-02-29 474780 1 + diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_source_models.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_source_models.out new file mode 100644 index 00000000000000..1583589eb295bd --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_source_models.out @@ -0,0 +1,26 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !internal_model_oracle -- +aggregate 30 E 303 +aggregate 31 F 310 +duplicate 1 A 10 +duplicate 1 A 11 +duplicate 2 \N 20 +unique_mor 20 C 201 +unique_mor 21 D 210 +unique_mow 10 A 101 +unique_mow 11 \N 110 + +-- !source_model_sink -- +aggregate 30 E 303 +aggregate 31 F 310 +duplicate 1 A 10 +duplicate 1 A 11 +duplicate 2 \N 20 +unique_mor 20 C 201 +unique_mor 21 D 210 +unique_mow 10 A 101 +unique_mow 11 \N 110 + +-- !source_model_partition_stats -- +0 9 + diff --git a/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_string_transform_metadata.out b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_string_transform_metadata.out new file mode 100644 index 00000000000000..a77b5504c2d1f8 --- /dev/null +++ b/regression-test/data/external_table_p0/iceberg/write/test_iceberg_write_string_transform_metadata.out @@ -0,0 +1,40 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !string_transform_rows -- +1 6173636969 6275636B65742D61 616C706861626574 ascii +2 E4B8ADE69687 E6A1B62DE4B8ADE69687 E4B8ADE69687E794B2 cjk +3 656D6F6A69 F09F98802D6275636B6574 F09F9880E794B2E4B999 emoji +4 65CC81 636F6D62696E696E672D6275636B6574 65CC8178 combining +5 empty +6 \N \N 6E756C6C2D6275636B6574 nullable-bucket + +-- !string_transform_physical_partitions -- +\N \N 6E75 1 + 0 1 +ascii 0 616C 1 +emoji 5 F09F9880E794B2 1 +é 0 65CC81 1 +中文 1 E4B8ADE69687 1 + +-- !string_transform_evolved_specs -- +2 6 6 +4 2 2 + +-- !string_transform_evolved_physical_partitions -- +2 \N \N 6E75 \N \N 1 +2 0 \N \N 1 +2 6173636969 0 616C \N \N 1 +2 656D6F6A69 5 F09F9880E794B2 \N \N 1 +2 65CC81 0 65CC81 \N \N 1 +2 E4B8ADE69687 1 E4B8ADE69687 \N \N 1 +4 \N \N \N \N F09F9880E794B2E4B999 1 +4 6E6577 \N \N 6 E4B8ADE69687E794B2 1 + +-- !string_transform_evolved_rows -- +1 6173636969 6275636B65742D61 616C706861626574 ascii +2 E4B8ADE69687 E6A1B62DE4B8ADE69687 E4B8ADE69687E794B2 cjk +3 656D6F6A69 F09F98802D6275636B6574 F09F9880E794B2E4B999 emoji +4 65CC81 636F6D62696E696E672D6275636B6574 65CC8178 combining +5 empty +6 \N \N 6E756C6C2D6275636B6574 nullable-bucket +7 6E6577 6275636B65742D6E6577 E4B8ADE69687E794B2E4B999 new-cjk +8 \N \N F09F9880E794B2E4B999E4B899 new-null-bucket diff --git a/regression-test/suites/external_table_p0/iceberg/write/ICEBERG_WRITE_P0_COVERAGE.md b/regression-test/suites/external_table_p0/iceberg/write/ICEBERG_WRITE_P0_COVERAGE.md new file mode 100644 index 00000000000000..7f0ba5b849c725 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/ICEBERG_WRITE_P0_COVERAGE.md @@ -0,0 +1,120 @@ + + +# Iceberg 写入 P0 覆盖矩阵 + +## 范围与判定原则 + +本文档覆盖 Doris 向 Iceberg 表写入时的正确性、兼容性和失败原子性。矩阵既检查单项能力，也检查 schema change、Partition Evolution、snapshot/tag/branch、行级 delete/update/merge、表模型、分区与 bucket、数据类型和 NULL 语义之间的交互。 + +所有正向 suite 都在最后一次写入后比较 Doris 与 Spark 的同表逻辑结果；涉及 transform metadata 的 suite 还比较物理分区值，涉及历史引用时另行校验 snapshot、tag 和 branch 的隔离性。 + +覆盖状态含义： + +- 已覆盖（验证通过）：P0 suite 对该场景有确定性结果断言，且已在多 BE 环境验证。 +- 预期拒绝：Doris 明确不支持该操作，P0 suite 验证错误信息与失败原子性。 +- 已覆盖（隔离负向）：已形成可复现产品问题的 regression；默认 P0 隔离运行，避免杀死共享 BE 或提交不可读文件。 + +## 风险点 + +| 编号 | 风险描述 | 来源 | 影响面 | 级别 | +| --- | --- | --- | --- | --- | +| R01 | schema change 后 writer 仍按旧列位置或旧 field id 写入，造成静默错列 | 白盒：Iceberg field id 与 Doris slot 映射 | 数据正确性 | P0 | +| R02 | Partition Evolution 后新文件落入旧 spec、分区值计算错误，或跨 spec 过滤漏数 | 黑盒 + 白盒：多 partition spec 并存 | 写入与查询正确性 | P0 | +| R03 | 字符串、数值、日期时间、decimal、布尔和 NULL 作为 identity/bucket/truncate/time transform 源时行为不一致 | 黑盒：类型与边界输入 | 分区路由、裁剪 | P0 | +| R04 | schema/partition 演进后 snapshot、tag、branch 绑定了错误 schema 或数据版本 | 黑盒：历史读与引用 | time travel 正确性 | P0 | +| R05 | MOR 的 delete/update/merge 跨新旧 spec 时生成错误 delete file；COW 被拒绝后仍发布快照 | 白盒：row-level DML commit | 数据丢失、失败原子性 | P0 | +| R06 | Duplicate、Unique MOW、Unique MOR、Aggregate 源表语义在 INSERT SELECT 时被改变 | 黑盒：不同 Doris 表模型 | 跨表写入正确性 | P0 | +| R07 | RANGE/LIST/无分区源表以及 HASH/RANDOM/AUTO bucket 在多 BE 执行时产生重复或丢行 | 黑盒 + 白盒：分布式 exchange 与 sink writer | 分布式写入正确性 | P0 | +| R08 | primitive、ARRAY、MAP、STRUCT 及嵌套 NULL 在 schema change 前后写入错误 | 黑盒：复杂类型与 NULL | 数据正确性、兼容性 | P0 | +| R09 | NULL 写入 Iceberg required 列未报错、部分数据或空快照被提交 | 白盒：required 校验与 commit | 约束、失败原子性 | P0 | +| R10 | INSERT OVERWRITE 在演进后的当前 spec、branch 或 NULL 分区上误删其他分区 | 黑盒：覆盖写 | 数据丢失 | P0 | +| R11 | 单 BE 可通过但多 BE 并发 sink 出现文件名、commit 或分区冲突 | 白盒：并行 writer 与统一 commit | 分布式稳定性 | P0 | +| R12 | nullable STRING 或 DML 产生的 Nullable block 经过 truncate transform 时 BE FATAL | 白盒：partition transformer 列类型约束 | 集群可用性 | P0 | +| R13 | MERGE 的多个源行匹配同一目标行时未执行基数校验，错误提交重复数据 | 黑盒 + 白盒：MERGE cardinality 与 commit | 数据正确性 | P0 | +| R14 | branch 写入污染 main，或 tag/不支持的 branch 行级 DML 失败后仍发布快照 | 黑盒：reference write 边界 | 历史引用、失败原子性 | P0 | +| R15 | 当前 spec 覆盖写未正确清理旧 spec delete file，或失败覆盖写留下部分快照 | 白盒：overwrite commit 与 delete file | 数据丢失、失败原子性 | P0 | +| R16 | CTAS 对复杂类型、NULL、分区 transform、文件格式和失败清理的行为不一致 | 黑盒：DDL + writer 一体提交 | schema、文件格式、原子性 | P0 | +| R17 | sort order、distribution mode、多次文件 flush 和并发 commit 组合导致乱序、丢行或重复提交 | 白盒：exchange、sort writer、optimistic commit | 分布式正确性、稳定性 | P0 | +| R18 | STRING identity/bucket/truncate 对空串、中文、emoji、组合字符和 NULL 的物理分区值计算错误 | 黑盒：UTF-8 transform metadata | 分区路由、裁剪 | P0 | + +## 组合覆盖 + +| 维度 | 场景 | 状态 | P0 suite | +| --- | --- | --- | --- | +| 基础写入 | Parquet/ORC、primitive/复杂类型、INSERT/OVERWRITE | 已覆盖 | `test_iceberg_write_insert`、`test_iceberg_insert_overwrite` | +| Partition transform | identity、bucket、truncate、year/month/day/hour | 已覆盖 | `test_iceberg_write_transform_partitions`、`test_iceberg_static_partition_overwrite` | +| schema + partition 演进 | add/rename/drop/type promotion 与 ADD/REPLACE/DROP partition field 后继续写入和过滤 | 已覆盖（验证通过） | `test_iceberg_write_evolution_refs` | +| 复杂类型演进 | ARRAY/MAP/STRUCT promotion、STRUCT 新增字段、旧文件与新写入并存 | 已覆盖（验证通过） | `test_iceberg_write_complex_evolution` | +| 历史版本 | 演进前后 snapshot、tag、branch；branch 独立写入和覆盖写 | 已覆盖（验证通过） | `test_iceberg_write_evolution_refs` | +| MOR | partition evolution 后 DELETE/UPDATE/MERGE，校验当前、delete files 与历史版本 | 已覆盖（验证通过） | `test_iceberg_write_dml_modes_evolution` | +| COW | partition evolution 后 DELETE/UPDATE/MERGE 拒绝，且数据和 snapshot 数不变 | 预期拒绝 | `test_iceberg_write_dml_modes_evolution` | +| Doris 源表模型 | Duplicate、Unique MOW、Unique MOR、Aggregate | 已覆盖（验证通过） | `test_iceberg_write_source_models` | +| Doris 源分区 | 无分区、RANGE、LIST | 已覆盖（验证通过） | `test_iceberg_write_source_models` | +| Doris 源 bucket | HASH 固定 bucket、RANDOM bucket、HASH AUTO bucket | 已覆盖（验证通过） | `test_iceberg_write_source_models` | +| 分区源类型 | STRING/INT/BIGINT/DATE/DATETIME/DECIMAL 的 bucket 与适用 transform；BOOLEAN identity 与非法 bucket | 已覆盖（验证通过） | `test_iceberg_write_partition_types_null` | +| NULL 分区 | identity NULL、数值/decimal bucket 与 truncate NULL、time transform NULL、多列组合 NULL | 已覆盖（验证通过） | `test_iceberg_write_partition_types_null` | +| nullable STRING truncate | nullable STRING 经过 truncate transform 的 INSERT，以及 UPDATE 产生的 Nullable projection 写入 | 已覆盖（隔离负向） | `test_iceberg_write_nullable_truncate_negative` | +| MERGE 完整语义 | 条件 MATCHED、DELETE/UPDATE、多个条件 NOT MATCHED、NULL-safe 与普通 NULL key | 已覆盖（验证通过） | `test_iceberg_write_merge_semantics` | +| MERGE 基数约束 | 多个源行匹配同一目标行必须整句失败且不发布快照 | 已覆盖（隔离负向） | `test_iceberg_write_merge_duplicate_source_negative` | +| MERGE + STRING truncate | required truncate 源列经 MERGE nullable projection 写入 | 已覆盖（隔离负向） | `test_iceberg_write_merge_truncate_negative` | +| branch/tag 写入边界 | branch INSERT/OVERWRITE 隔离；tag 写入和 branch DELETE/UPDATE/MERGE 明确拒绝 | 已覆盖（验证通过） | `test_iceberg_write_branch_dml_boundary` | +| nullable 数据 | 顶层 NULL、ARRAY NULL 元素、MAP NULL value、STRUCT NULL child | 已覆盖并增强 | `test_iceberg_write_insert`、`test_iceberg_write_complex_evolution` | +| required 列正向与 schema change | required 列合法写入、nullable 列写 NULL、增加 required 列与 nullable→required 拒绝 | 已覆盖（验证通过） | `test_iceberg_write_nullability_atomicity` | +| required 列写 NULL | VALUES 与分布式 INSERT SELECT 混合批次写 NULL | 已覆盖（隔离负向） | `test_iceberg_write_required_null_values_negative`、`test_iceberg_write_required_null_select_negative` | +| 覆盖写 | 当前 spec、静态分区、branch、空输入、连续多次 partition evolution、NULL 当前分区 | 已覆盖并增强 | `test_iceberg_static_partition_overwrite`、`test_iceberg_write_evolution_refs`、`test_iceberg_write_overwrite_evolution` | +| 覆盖写 + delete files | MOR DELETE/UPDATE/MERGE 后覆盖写，演进前后 delete files 与历史 tag 共存 | 已覆盖（验证通过） | `test_iceberg_write_overwrite_delete_files` | +| 覆盖写失败原子性 | main/branch 分布式严格类型转换失败、快照/文件/数据不变、修正后重试 | 已覆盖（验证通过） | `test_iceberg_write_overwrite_atomicity` | +| STRING 物理 transform | identity、nullable bucket、required truncate 的 UTF-8 边界值及 transform width evolution | 已覆盖（验证通过） | `test_iceberg_write_string_transform_metadata` | +| CTAS | 复杂类型、嵌套 NULL、identity+bucket、ORC 压缩、失败建表清理 | 已覆盖（验证通过） | `test_iceberg_write_ctas_format_boundary` | +| 文件格式边界 | Parquet/ORC 正向写入；Avro 表写入明确拒绝并保持快照和文件不变 | 已覆盖（正向 + 预期拒绝） | `test_iceberg_write_ctas_format_boundary` | +| 排序与分布属性 | 多列 sort order、NULL ordering、none/hash/range distribution、强制多文件 flush | 已覆盖（验证通过） | `test_iceberg_write_order_distribution_properties` | +| 并发写入 | 同行冲突 MERGE 的串行化不变量、非冲突分布式 append | 已覆盖（验证通过） | `test_iceberg_write_concurrent_merge_invariants` | +| 分布式执行 | 多 bucket 源表、多分区 Iceberg sink、多 BE writer、suite 间无共享 catalog/database | 已覆盖（验证通过） | 所有本次新增 suite | +| Spark 交叉验证 | 十五个正向 suite 在最后一次写入后逐行比较；STRING transform suite 另行比较演进前后的物理分区 metadata | 已覆盖（验证通过） | 十五个正向 suite；物理 metadata 见 `test_iceberg_write_string_transform_metadata` | + +## 本次新增用例设计 + +| 用例 | 目标 | 覆盖风险 | 测试维度 | 前置条件 | 负载描述 | 执行预期 | +| --- | --- | --- | --- | --- | --- | --- | +| W01 | 验证 schema 与 partition spec 同时演进后的写入、过滤和历史引用 | R01、R02、R04、R10 | 功能、正确性、兼容性 | Iceberg REST catalog | 演进前后多批 Doris 写入，建立 snapshot/tag/branch，并对 branch 覆盖写 | 当前、历史和 branch 各自返回确定数据；跨 spec 过滤不漏数 | +| W02 | 验证复杂类型 field id 在演进后保持正确 | R01、R08 | 功能、正确性 | Iceberg v2 | ARRAY/MAP value promotion、STRUCT child promotion/add，写入含嵌套 NULL 的新旧行 | 旧值按新 schema 可读，新值不串字段，嵌套 NULL 保留 | +| W03 | 验证 MOR/COW 与 partition evolution、NULL 分区、time travel 的交互 | R02、R04、R05 | 功能、正确性、异常 | Iceberg v2 MOR/COW | MOR 执行 delete/update/merge；COW 执行相同操作 | MOR 当前与历史版本一致；COW 明确拒绝且无新 snapshot | +| W04 | 验证不同 Doris 表模型、分区和 bucket 作为 Iceberg 写入源 | R06、R07、R11 | 正确性、兼容性 | 多 BE Doris | 四种表模型、三种分区方式、HASH/RANDOM/AUTO bucket 执行 INSERT SELECT | 写入结果保持各源表语义，无重复或丢行 | +| W05 | 验证不同类型与 NULL 的 partition/bucket transform | R02、R03、R11 | 功能、正确性、边界 | Iceberg v2 | identity/bucket/truncate/time transform 多列组合，包含 NULL | 数据与 `$partitions` 统计一致；NULL 行可过滤且可继续写入 | +| W06 | 验证 required/nullable schema change 与合法写入 | R09、R11 | 异常、正确性 | Iceberg required 列 | 拒绝增加无默认值 required 列和 nullable→required；执行 VALUES/INSERT SELECT 合法写入 | schema change 失败不产生 snapshot；合法写入与 Spark 结果一致 | +| W07 | 验证 required 列 NULL 拒绝和 statement 原子性 | R09、R11 | 隔离负向、正确性 | 隔离 Iceberg database | VALUES 写 NULL；多 bucket 源表 INSERT SELECT 混合有效与 NULL 行 | 修复前会错误提交并产生不可读文件；修复后整条语句在 snapshot 发布前拒绝 | +| W08 | 验证 STRING truncate 的 Nullable block 处理 | R03、R05、R12 | 隔离负向、稳定性 | 可重启的隔离 Doris 集群 | nullable STRING INSERT；partition evolution 后 UPDATE 产生 Nullable block | 修复前 BE FATAL；修复后写入成功并保持 NULL 分区语义 | +| W09 | 验证 MERGE 条件动作、多个 NOT MATCHED 与 NULL key 语义 | R02、R03、R05 | 功能、正确性 | Iceberg v2 MOR | identity/bucket 分区间移动、删除、插入、NULL-safe 与普通等值匹配 | 每个源行只选择一个动作，Spark 与 Doris 结果一致 | +| W10 | 验证 MERGE 多源匹配单目标的基数约束 | R13 | 隔离负向、原子性 | Iceberg v2 MOR | 两个源行同时更新一个目标行 | 修复前错误提交重复行；修复后整句拒绝且无新快照和文件 | +| W11 | 验证 branch/tag 的写入能力边界 | R04、R14 | 功能、异常、原子性 | 已建立 branch 与 tag | branch INSERT/OVERWRITE；branch 行级 DML 与 tag 写入 | branch 与 main 隔离；不支持操作明确拒绝且引用不变化 | +| W12 | 验证多次 Partition Evolution 后覆盖写和历史引用 | R02、R04、R10、R15 | 功能、正确性 | Iceberg v2 | ADD/REPLACE/DROP identity、bucket、truncate、day/hour 后动态覆盖写 | 仅替换当前 spec 命中的分区，tag/branch 和旧 spec 保持可读 | +| W13 | 验证 delete files 与覆盖写、演进的交互 | R02、R05、R15 | 正确性、兼容性 | Iceberg v2 MOR | DELETE/UPDATE/MERGE 生成 delete files，再在新旧 spec 上覆盖写 | replacement 行不被旧 delete files 隐藏，历史 tag 不受影响 | +| W14 | 验证 main/branch 覆盖写失败与重试原子性 | R09、R10、R15 | 异常、原子性 | 多 BE Doris | 分布式严格类型转换失败后检查数据、文件和快照，再执行修正重试 | 失败零提交；重试恰好产生一个快照且无重复 | +| W15 | 验证 STRING transform 的真实物理分区值 | R03、R18 | 边界、正确性 | Iceberg v2 | 空串、ASCII、中文、emoji、组合字符、NULL bucket，随后替换 bucket/truncate 宽度 | 行结果和 `$partitions` 物理值均与 Spark 一致 | +| W16 | 验证 CTAS、复杂类型、格式和失败清理 | R08、R09、R16 | 功能、异常、兼容性 | 内部多 bucket 源表 | CTAS 到 ORC 分区表；严格转换失败；向 Avro 表写入 | ORC 与 Spark 一致；失败不遗留表或快照；Avro 明确拒绝 | +| W17 | 验证 sort order、distribution mode 和多文件 flush | R03、R11、R17 | 正确性、稳定性 | 多 BE Doris | NULL sort key、多列升降序、none/hash/range、低 target file size | 计划包含声明排序，多文件总行数正确，三种分布模式结果一致 | +| W18 | 验证并发 MERGE 与 append 的提交不变量 | R11、R13、R17 | 并发、原子性 | 多 BE Doris | readiness barrier 保证两个独立会话同时具备 dispatch 条件；同行更新与互不冲突 append | 同行提交可串行化且基数为一；仅接受 Iceberg validation/commit conflict；非冲突写入无丢失或重复 | +| W19 | 验证 MERGE source projection 进入 truncate transform 的类型安全 | R12、R18 | 隔离负向、稳定性 | 可重启的隔离 Doris 集群 | required STRING truncate 列执行匹配更新与未匹配插入 | 修复前 BE FATAL；修复后 MERGE 成功且物理分区正确 | + +## P0 覆盖检查 + +R01-R18 均映射到至少一个 P0 regression。十五个正向 suite 在最后一次成功写入后均由 Spark/Doris 交叉校验同表逻辑结果；transform、rollover、CTAS property 和失败原子性另有物理 metadata 或状态 oracle。稳定性或已确认正确性缺陷使用独立 suite、完整预期输出和显式隔离开关保存复现，避免默认 P0 破坏共享集群或固化错误结果。 + +本矩阵未覆盖项为 0。COW 行级 DML、branch 行级 DML、tag 写入和 Avro 写入属于当前明确能力边界，均以预期拒绝用例固化错误语义与失败原子性；已确认的产品缺陷均有隔离负向 regression。 diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_branch_dml_boundary.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_branch_dml_boundary.groovy new file mode 100644 index 00000000000000..f5f8ef5bb9cd49 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_branch_dml_boundary.groovy @@ -0,0 +1,169 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_write_branch_dml_boundary", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_write_branch_dml_boundary" + String dbName = "iceberg_write_branch_dml_boundary_db" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1" + ) + """ + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + + sql """drop table if exists branch_dml_boundary""" + sql """ + create table branch_dml_boundary ( + id int, + region string, + payload string + ) + partition by list (region) () + properties ( + "format-version" = "2", + "write.delete.mode" = "merge-on-read", + "write.update.mode" = "merge-on-read", + "write.merge.mode" = "merge-on-read" + ) + """ + sql """insert into branch_dml_boundary values (1, 'A', 'main')""" + sql """alter table branch_dml_boundary create branch audit_branch""" + sql """alter table branch_dml_boundary create tag protected_tag""" + + // WB01-S01: Seed the exact identity partition that OVERWRITE targets so an + // append-equivalent implementation cannot satisfy the final row oracle. + sql """insert into branch_dml_boundary@branch(audit_branch) values (2, 'B', 'branch-insert')""" + sql """ + insert into branch_dml_boundary@branch(audit_branch) + values (30, 'C', 'branch-overwrite-seed') + """ + assertEquals(1L, (sql """ + select count(*) from branch_dml_boundary@branch(audit_branch) + where payload = 'branch-overwrite-seed' + """)[0][0] as long) + sql """ + insert overwrite table branch_dml_boundary@branch(audit_branch) + values (3, 'C', 'branch-overwrite') + """ + assertEquals(0L, (sql """ + select count(*) from branch_dml_boundary@branch(audit_branch) + where payload = 'branch-overwrite-seed' + """)[0][0] as long) + order_qt_branch_write """ + select id, region, payload + from branch_dml_boundary@branch(audit_branch) + order by id + """ + order_qt_main_after_branch_write """ + select id, region, payload + from branch_dml_boundary + order by id + """ + + long mainSnapshots = (sql """select count(*) from branch_dml_boundary\$snapshots""")[0][0] as long + + // WB01-S02: The current Doris SQL surface does not accept branch-qualified + // targets for row-level DML. Keep the capability boundary explicit and atomic. + test { + sql """delete from branch_dml_boundary@branch(audit_branch) where id = 3""" + exception "@" + } + test { + sql """ + update branch_dml_boundary@branch(audit_branch) + set payload = 'updated' + where id = 3 + """ + exception "@" + } + test { + sql """ + merge into branch_dml_boundary@branch(audit_branch) t + using (select 3 as id, 'merged' as payload) s + on t.id = s.id + when matched then update set payload = s.payload + """ + exception "@" + } + assertEquals(mainSnapshots, + (sql """select count(*) from branch_dml_boundary\$snapshots""")[0][0] as long) + order_qt_branch_after_rejected_dml """ + select id, region, payload + from branch_dml_boundary@branch(audit_branch) + order by id + """ + + // WB01-S03: Tags are immutable write targets. + test { + sql """insert into branch_dml_boundary@branch(protected_tag) values (9, 'T', 'tag-write')""" + // TestAction stores only one `exception` substring, so one closure must + // verify both parts of the capability-boundary diagnostic. + check { result, exception, startTime, endTime -> + assertTrue(exception != null) + String message = exception.toString() + assertTrue(message.contains("tag")) + assertTrue(message.contains("not a branch")) + } + } + + // Cross-engine reads cover both the unchanged main ref and the branch + // after its successful writes and rejected row-level DML attempts. + spark_iceberg """refresh table demo.${dbName}.branch_dml_boundary""" + def sparkMainRows = spark_iceberg """ + select id, region, payload + from demo.${dbName}.branch_dml_boundary + order by id + """ + def dorisMainRows = sql """ + select id, region, payload + from branch_dml_boundary + order by id + """ + assertSparkDorisResultEquals(sparkMainRows, dorisMainRows) + def sparkBranchRows = spark_iceberg """ + select id, region, payload + from demo.${dbName}.branch_dml_boundary version as of 'audit_branch' + order by id + """ + def dorisBranchRows = sql """ + select id, region, payload + from branch_dml_boundary@branch(audit_branch) + order by id + """ + assertSparkDorisResultEquals(sparkBranchRows, dorisBranchRows) +} diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_complex_evolution.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_complex_evolution.groovy new file mode 100644 index 00000000000000..e5ad9e7c6ed19e --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_complex_evolution.groovy @@ -0,0 +1,178 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_write_complex_evolution", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_write_complex_evolution" + String dbName = "iceberg_write_complex_evolution_db" + + def assertSparkMatchesDoris = { + sql """refresh table ${dbName}.complex_evolution""" + spark_iceberg """refresh table demo.${dbName}.complex_evolution""" + def sparkRows = spark_iceberg """ + select id, group_key, arr, mp, payload + from demo.${dbName}.complex_evolution + order by id + """ + def dorisRows = sql """ + select id, group_key, arr, mp, payload + from complex_evolution + order by id + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1", + "meta.cache.iceberg.table.ttl-second" = "0", + "meta.cache.iceberg.schema.ttl-second" = "0" + ) + """ + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + sql """set enable_fallback_to_original_planner = false""" + + sql """drop table if exists complex_evolution""" + sql """ + create table complex_evolution ( + id int not null, + group_key string not null, + arr array, + mp map, + payload struct< + metric:int, + label:string, + nested:struct + > + ) + partition by list (group_key) () + properties ( + "format-version" = "2", + "write.format.default" = "orc" + ) + """ + + // W02-S01: Write old-schema rows, including NULL collections, elements, values and children. + sql """ + insert into complex_evolution values + (1, 'A', array(1, null, 3), map('x', 10, 'null-value', null), + struct(10, 'old-a', struct(1, null))), + (2, 'N', null, map('x', null), + struct(20, null, struct(null, 'old-null'))), + (3, 'B', array(), map(), null) + """ + String baseSnapshot = (sql """ + select snapshot_id from complex_evolution\$snapshots + order by committed_at desc limit 1 + """)[0][0].toString() + sql """alter table complex_evolution create tag complex_base as of version ${baseSnapshot}""" + assertSparkMatchesDoris() + + // W02-S02: Promote every supported nested primitive and add STRUCT children. + // The following write checks that Doris uses Iceberg field ids rather than child positions. + sql """alter table complex_evolution modify column arr array""" + sql """alter table complex_evolution modify column mp map""" + sql """ + alter table complex_evolution modify column payload struct< + metric:bigint, + label:string, + nested:struct, + tags:array, + attributes:map + > + """ + sql """alter table complex_evolution add partition key bucket(8, id) as id_bucket""" + sql """alter table complex_evolution add partition key truncate(1, group_key) as group_prefix""" + + sql """ + insert into complex_evolution values + (4, 'A1', array(cast(4000000000 as bigint), null), + map('large', cast(5000000000 as bigint), 'null-value', null), + struct( + cast(6000000000 as bigint), + 'new-a', + struct(cast(7000000000 as bigint), 'nested-new', cast(7.5 as double)), + array('x', null, 'z'), + map('a', cast(8000000000 as bigint), 'b', null) + )), + (5, 'N2', array(null), null, + struct( + cast(50 as bigint), + null, + struct(cast(5 as bigint), null, null), + null, + map('null-value', null) + )) + """ + + // W02-S03: Current schema reads both old and new files without moving old child values. + order_qt_complex_current """ + select id, group_key, arr, mp, payload + from complex_evolution + order by id + """ + order_qt_complex_children """ + select id, payload.metric, payload.nested.count, payload.nested.score, + payload.tags, payload.attributes + from complex_evolution + order by id + """ + order_qt_complex_nulls """ + select id + from complex_evolution + where group_key is null + or arr is null + or mp is null + or payload is null + or payload.nested.score is null + order by id + """ + order_qt_complex_partition_specs """ + select spec_id, sum(record_count) + from complex_evolution\$partitions + group by spec_id + order by spec_id + """ + assertSparkMatchesDoris() + + // W02-S04: A pre-evolution tag binds the old files to their historical complex schema. + order_qt_complex_base_tag """ + select id, arr, mp, payload.metric, payload.label, + payload.nested.count, payload.nested.comment + from complex_evolution@tag(complex_base) + order by id + """ +} diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_concurrent_merge_invariants.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_concurrent_merge_invariants.groovy new file mode 100644 index 00000000000000..6d46cf68a3db7d --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_concurrent_merge_invariants.groovy @@ -0,0 +1,209 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +import java.util.Collections +import java.util.concurrent.CountDownLatch +import java.util.concurrent.TimeUnit + +suite("test_iceberg_write_concurrent_merge_invariants", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_write_concurrent_merge_invariants" + String dbName = "iceberg_write_concurrent_merge_invariants_db" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1", + "meta.cache.iceberg.table.ttl-second" = "0", + "meta.cache.iceberg.schema.ttl-second" = "0" + ) + """ + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + + sql """drop table if exists concurrent_merge""" + sql """ + create table concurrent_merge ( + id int not null, + region string, + payload string + ) + partition by list (region) () + properties ( + "format-version" = "2", + "write.format.default" = "parquet", + "write.delete.mode" = "merge-on-read", + "write.update.mode" = "merge-on-read", + "write.merge.mode" = "merge-on-read", + "write.merge.isolation-level" = "serializable" + ) + """ + sql """insert into concurrent_merge values (1, 'A', 'base')""" + long snapshotsBefore = (sql """select count(*) from concurrent_merge\$snapshots""")[0][0] as long + + def isExpectedIcebergCommitConflict = { Exception exception -> + String message = exception.toString() + return message.contains("org.apache.iceberg.exceptions.ValidationException") + || message.contains("org.apache.iceberg.exceptions.CommitFailedException") + || message.contains("Found conflicting files") + } + + // WC02-S01: A readiness barrier prevents a fast worker from completing + // before the second session is even eligible for concurrent dispatch. + // The exact winner is intentionally unspecified; cardinality, snapshot + // accounting and cross-engine visibility are deterministic invariants. + CountDownLatch ready = new CountDownLatch(2) + CountDownLatch start = new CountDownLatch(1) + List successes = Collections.synchronizedList(new ArrayList()) + List failures = Collections.synchronizedList(new ArrayList()) + + def first = thread("iceberg-merge-one") { + ready.countDown() + start.await() + try { + sql """ + merge into ${catalogName}.${dbName}.concurrent_merge t + using (select 1 as id, 'B' as region, 'winner-one' as payload) s + on t.id = s.id + when matched then update set region = s.region, payload = s.payload + """ + successes.add("one") + } catch (Exception e) { + // Only an Iceberg optimistic-validation conflict is an admissible + // loser; planner, RPC, catalog and BE failures must fail the suite. + if (!isExpectedIcebergCommitConflict(e)) { + throw e + } + failures.add(e.getMessage()) + } + } + def second = thread("iceberg-merge-two") { + ready.countDown() + start.await() + try { + sql """ + merge into ${catalogName}.${dbName}.concurrent_merge t + using (select 1 as id, 'C' as region, 'winner-two' as payload) s + on t.id = s.id + when matched then update set region = s.region, payload = s.payload + """ + successes.add("two") + } catch (Exception e) { + if (!isExpectedIcebergCommitConflict(e)) { + throw e + } + failures.add(e.getMessage()) + } + } + assertTrue(ready.await(30, TimeUnit.SECONDS), + "Both MERGE workers must reach the dispatch barrier") + start.countDown() + first.get() + second.get() + + assertTrue(successes.size() >= 1) + assertEquals(2, successes.size() + failures.size()) + assertEquals(1L, (sql """select count(*) from concurrent_merge where id = 1""")[0][0] as long) + assertEquals(snapshotsBefore + successes.size(), + (sql """select count(*) from concurrent_merge\$snapshots""")[0][0] as long) + def visible = sql """ + select payload + from concurrent_merge + where id = 1 + """ + assertTrue(["winner-one", "winner-two"].contains(visible[0][0].toString())) + + spark_iceberg """refresh table demo.${dbName}.concurrent_merge""" + def sparkRows = spark_iceberg """ + select id, region, payload + from demo.${dbName}.concurrent_merge + order by id + """ + def dorisRows = sql """ + select id, region, payload + from concurrent_merge + order by id + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + + // WC02-S02: Use the same readiness invariant for non-conflicting appends; + // otherwise sequential execution can falsely satisfy the row-count oracle. + CountDownLatch appendReady = new CountDownLatch(2) + CountDownLatch appendStart = new CountDownLatch(1) + def appendOne = thread("iceberg-append-one") { + appendReady.countDown() + appendStart.await() + sql """ + insert into ${catalogName}.${dbName}.concurrent_merge + select number + 10, 'append-one', concat('one-', number) + from numbers('number' = '128') + """ + } + def appendTwo = thread("iceberg-append-two") { + appendReady.countDown() + appendStart.await() + sql """ + insert into ${catalogName}.${dbName}.concurrent_merge + select number + 1000, 'append-two', concat('two-', number) + from numbers('number' = '128') + """ + } + assertTrue(appendReady.await(30, TimeUnit.SECONDS), + "Both append workers must reach the dispatch barrier") + appendStart.countDown() + appendOne.get() + appendTwo.get() + order_qt_concurrent_append_counts """ + select region, count(*), count(distinct id) + from concurrent_merge + where region in ('append-one', 'append-two') + group by region + order by region + """ + + // Refresh after both appends so the cross-engine oracle covers the + // concurrent commits rather than only the earlier MERGE result. + spark_iceberg """refresh table demo.${dbName}.concurrent_merge""" + sparkRows = spark_iceberg """ + select id, region, payload + from demo.${dbName}.concurrent_merge + order by id + """ + dorisRows = sql """ + select id, region, payload + from concurrent_merge + order by id + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) +} diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_ctas_format_boundary.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_ctas_format_boundary.groovy new file mode 100644 index 00000000000000..9be6f3ba421281 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_ctas_format_boundary.groovy @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_write_ctas_format_boundary", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_write_ctas_format_boundary" + String dbName = "iceberg_write_ctas_format_boundary_db" + String internalDbName = "iceberg_write_ctas_format_boundary_internal_db" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1", + "meta.cache.iceberg.table.ttl-second" = "0", + "meta.cache.iceberg.schema.ttl-second" = "0" + ) + """ + + sql """drop database if exists internal.${internalDbName} force""" + sql """create database internal.${internalDbName}""" + sql """drop table if exists internal.${internalDbName}.ctas_source""" + sql """ + create table internal.${internalDbName}.ctas_source ( + id int, + region varchar(20), + tags array, + attrs map, + detail struct + ) + duplicate key(id) + distributed by hash(id) buckets 4 + properties ("replication_num" = "1") + """ + sql """ + insert into internal.${internalDbName}.ctas_source values + (1, 'A', ['x', null], map('k', 'v'), struct(10, 'one')), + (2, null, [], map('null-value', null), struct(null, 'two')), + (3, '中文', ['😀'], map(), struct(30, null)) + """ + + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + + // WC01-S01: CTAS preserves complex types, NULL values, partitioning and + // writer properties when the source is a distributed Doris table. + sql """drop table if exists ctas_complex_partitioned""" + sql """ + create table ctas_complex_partitioned + partition by list (region, bucket(4, id)) () + properties ( + "format-version" = "2", + "write.format.default" = "orc", + "write.orc.compression-codec" = "lz4" + ) + as + select id, + cast(if(id in (1, 3), 'A', region) as string) as region, + tags, attrs, detail + from internal.${internalDbName}.ctas_source + """ + order_qt_ctas_complex_rows """ + select id, region, tags, attrs, detail + from ctas_complex_partitioned + order by id + """ + order_qt_ctas_complex_files """ + select lower(file_format), sum(record_count) + from ctas_complex_partitioned\$files + group by lower(file_format) + order by lower(file_format) + """ + order_qt_ctas_complex_partitions """ + select spec_id, count(*), sum(record_count) + from ctas_complex_partitioned\$partitions + group by spec_id + order by spec_id + """ + order_qt_ctas_complex_physical_partitions """ + select struct_element(`partition`, 'region'), + struct_element(`partition`, 'id_bucket'), + record_count + from ctas_complex_partitioned\$partitions + order by 1, 2 + """ + // Two rows share identity region A, so more than one physical partition + // proves bucket(4,id) contributes to CTAS routing. + assertTrue(((sql """ + select count(distinct struct_element(`partition`, 'id_bucket')) + from ctas_complex_partitioned\$partitions + where struct_element(`partition`, 'region') = 'A' + """)[0][0] as long) > 1L) + spark_iceberg """refresh table demo.${dbName}.ctas_complex_partitioned""" + def sparkRows = spark_iceberg """ + select id, region, tags, attrs, detail + from demo.${dbName}.ctas_complex_partitioned + order by id + """ + def dorisRows = sql """ + select id, region, tags, attrs, detail + from ctas_complex_partitioned + order by id + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + def compressionProperty = spark_iceberg """ + show tblproperties demo.${dbName}.ctas_complex_partitioned + ('write.orc.compression-codec') + """ + // File-format checks do not prove the requested codec survived CTAS. + assertEquals("lz4", compressionProperty[0][1].toString().toLowerCase()) + + // WC01-S02: CTAS is atomic. A source expression failure must not leave a + // visible Iceberg table or a partially committed snapshot. + sql """set enable_strict_cast = true""" + sql """drop table if exists ctas_failed_atomicity""" + test { + sql """ + create table ctas_failed_atomicity + properties ("format-version" = "2") + as + select cast(if(number = 2, 'invalid-id', cast(number as string)) as int) as id, + concat('candidate-', number) as payload + from numbers('number' = '8') + """ + exception "can't cast to INT in strict mode" + } + assertEquals(0, (sql """show tables like 'ctas_failed_atomicity'""").size()) + + // WC01-S03: Iceberg allows Avro, but the current Doris writer supports + // Parquet and ORC only. Reject Avro explicitly instead of silently falling back. + sql """drop table if exists avro_write_boundary""" + sql """ + create table avro_write_boundary ( + id int, + payload string + ) + properties ( + "format-version" = "2", + "write.format.default" = "avro" + ) + """ + long avroSnapshots = (sql """select count(*) from avro_write_boundary\$snapshots""")[0][0] as long + test { + sql """insert into avro_write_boundary values (1, 'must-not-fallback')""" + exception "Unsupported input format type: avro" + } + assertEquals(avroSnapshots, + (sql """select count(*) from avro_write_boundary\$snapshots""")[0][0] as long) + assertEquals(0, (sql """select count(*) from avro_write_boundary\$files""")[0][0] as long) +} diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_dml_modes_evolution.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_dml_modes_evolution.groovy new file mode 100644 index 00000000000000..db999e19651e5c --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_dml_modes_evolution.groovy @@ -0,0 +1,269 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_write_dml_modes_evolution", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_write_dml_modes_evolution" + String dbName = "iceberg_write_dml_modes_evolution_db" + + def assertSparkMatchesDoris = { String tableName -> + sql """refresh table ${dbName}.${tableName}""" + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + def sparkRows = spark_iceberg """ + select id, region, bucket_key, event_time, score, status + from demo.${dbName}.${tableName} + order by id + """ + def dorisRows = sql """ + select id, region, bucket_key, event_time, score, status + from ${tableName} + order by id + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1", + "meta.cache.iceberg.table.ttl-second" = "0", + "meta.cache.iceberg.schema.ttl-second" = "0" + ) + """ + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + sql """set enable_fallback_to_original_planner = false""" + + sql """drop table if exists mor_evolution""" + sql """ + create table mor_evolution ( + id int not null, + region string, + bucket_key string not null, + event_time datetime, + score int + ) + partition by list (region, bucket(4, bucket_key), day(event_time)) () + properties ( + "format-version" = "2", + "write.format.default" = "parquet", + "write.delete.mode" = "merge-on-read", + "write.update.mode" = "merge-on-read", + "write.merge.mode" = "merge-on-read" + ) + """ + + // W03-S01: The MOR baseline includes NULL in every partition transform family. + sql """ + insert into mor_evolution values + (1, 'A', 'alpha', '2026-01-01 01:00:00', 10), + (2, 'B', 'beta', '2026-01-02 02:00:00', 20), + (3, null, 'null-key', null, 30), + (4, 'A', 'delta', '2026-01-04 04:00:00', 40) + """ + String morBaseSnapshot = (sql """ + select snapshot_id from mor_evolution\$snapshots + order by committed_at desc limit 1 + """)[0][0].toString() + sql """alter table mor_evolution create tag mor_base as of version ${morBaseSnapshot}""" + + // W03-S02: Evolve schema and partition spec, then write more files before row-level DML. + sql """alter table mor_evolution add column status string""" + sql """ + alter table mor_evolution + replace partition key day(event_time) with month(event_time) as event_month + """ + sql """ + alter table mor_evolution + replace partition key bucket(4, bucket_key) with bucket(8, id) as id_bucket + """ + sql """ + insert into mor_evolution values + (5, 'B', 'echo', '2026-02-01 05:00:00', 50, 'new-spec'), + (6, null, 'foxtrot', null, 60, 'new-null'), + (7, 'C', 'golf', '2026-03-01 07:00:00', 70, null) + """ + String morBeforeDmlSnapshot = (sql """ + select snapshot_id from mor_evolution\$snapshots + order by committed_at desc limit 1 + """)[0][0].toString() + sql """alter table mor_evolution create tag mor_before_dml as of version ${morBeforeDmlSnapshot}""" + + // W03-S03: DELETE spans old/new specs and removes NULL partition rows. + sql """delete from mor_evolution where region is null""" + + // W03-S04: UPDATE changes partition source values in files from both specs. + sql """ + update mor_evolution + set region = concat(region, '-updated'), + score = score + 100, + status = 'updated' + where region = 'A' + """ + + // W03-S05: MERGE deletes, updates and inserts across different transformed partitions. + sql """ + merge into mor_evolution t + using ( + select 2 as id, 'B-merged' as region, 'beta-merged' as bucket_key, + timestamp '2026-04-02 02:00:00' as event_time, 220 as score, + 'U' as op + union all + select 5, 'B', 'echo', timestamp '2026-02-01 05:00:00', 50, 'D' + union all + select 8, 'D', 'hotel', timestamp '2026-05-01 08:00:00', 80, 'I' + ) s + on t.id = s.id + when matched and s.op = 'D' then delete + when matched then update set + region = s.region, + bucket_key = s.bucket_key, + event_time = s.event_time, + score = s.score, + status = 'merged' + when not matched then insert (id, region, bucket_key, event_time, score, status) + values (s.id, s.region, s.bucket_key, s.event_time, s.score, 'inserted') + """ + + order_qt_mor_current """ + select id, region, bucket_key, event_time, score, status + from mor_evolution + order by id + """ + order_qt_mor_base_tag """ + select id, region, bucket_key, event_time, score + from mor_evolution@tag(mor_base) + order by id + """ + order_qt_mor_before_dml_tag """ + select id, region, bucket_key, event_time, score, status + from mor_evolution@tag(mor_before_dml) + order by id + """ + order_qt_mor_delete_files """ + select spec_id, count(*), sum(record_count) + from mor_evolution\$delete_files + group by spec_id + order by spec_id + """ + assertSparkMatchesDoris("mor_evolution") + + // W03-S06: COW accepts INSERT after partition evolution but Doris explicitly rejects + // DELETE/UPDATE/MERGE. Each rejection must leave both data and snapshot count unchanged. + sql """drop table if exists cow_evolution""" + sql """ + create table cow_evolution ( + id int not null, + region string, + bucket_key string not null, + event_time datetime, + score int, + status string + ) + partition by list (region, bucket(4, bucket_key), day(event_time)) () + properties ( + "format-version" = "2", + "write.format.default" = "orc", + "write.delete.mode" = "copy-on-write", + "write.update.mode" = "copy-on-write", + "write.merge.mode" = "copy-on-write" + ) + """ + sql """ + insert into cow_evolution values + (1, 'A', 'alpha', '2026-01-01 01:00:00', 10, 'base'), + (2, null, 'null-key', null, 20, 'null-partition') + """ + sql """ + alter table cow_evolution + replace partition key bucket(4, bucket_key) with bucket(8, id) as id_bucket + """ + sql """ + alter table cow_evolution + replace partition key day(event_time) with month(event_time) as event_month + """ + sql """ + insert into cow_evolution values + (3, 'B', 'beta', '2026-02-01 03:00:00', 30, 'new-spec') + """ + + long cowSnapshots = (sql """select count(*) from cow_evolution\$snapshots""")[0][0] as long + test { + sql """delete from cow_evolution where region is null""" + // TestAction has one exception slot; a check closure preserves both + // the rejected operation and its actionable property guidance. + check { result, exception, startTime, endTime -> + assertTrue(exception != null) + String message = exception.toString() + assertTrue(message.contains( + "Doris does not support DELETE on Iceberg copy-on-write tables")) + assertTrue(message.contains( + "Set table property 'write.delete.mode' to 'merge-on-read'")) + } + } + test { + sql """update cow_evolution set score = score + 1 where id = 1""" + check { result, exception, startTime, endTime -> + assertTrue(exception != null) + String message = exception.toString() + assertTrue(message.contains( + "Doris does not support UPDATE on Iceberg copy-on-write tables")) + assertTrue(message.contains( + "Set table property 'write.update.mode' to 'merge-on-read'")) + } + } + test { + sql """ + merge into cow_evolution t + using (select 1 as id, 100 as score) s + on t.id = s.id + when matched then update set score = s.score + """ + check { result, exception, startTime, endTime -> + assertTrue(exception != null) + String message = exception.toString() + assertTrue(message.contains( + "Doris does not support MERGE INTO on Iceberg copy-on-write tables")) + assertTrue(message.contains( + "Set table property 'write.merge.mode' to 'merge-on-read'")) + } + } + assertEquals(cowSnapshots, (sql """select count(*) from cow_evolution\$snapshots""")[0][0] as long) + order_qt_cow_after_rejections """ + select id, region, bucket_key, event_time, score, status + from cow_evolution + order by id + """ + assertSparkMatchesDoris("cow_evolution") +} diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_evolution_refs.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_evolution_refs.groovy new file mode 100644 index 00000000000000..3d6ec5a2616335 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_evolution_refs.groovy @@ -0,0 +1,253 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_write_evolution_refs", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_write_evolution_refs" + String dbName = "iceberg_write_evolution_refs_db" + + def latestSnapshotId = { + return (sql """ + select snapshot_id + from evolution_refs\$snapshots + order by committed_at desc + limit 1 + """)[0][0].toString() + } + + def assertSparkMatchesDoris = { String relation, String projection -> + sql """refresh table ${dbName}.evolution_refs""" + spark_iceberg """refresh table demo.${dbName}.evolution_refs""" + def sparkRows = spark_iceberg """ + select ${projection} + from demo.${dbName}.evolution_refs${relation} + order by id + """ + def dorisRows = sql """ + select ${projection} + from evolution_refs${relation} + order by id + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + def assertSparkBranchMatchesDoris = { String branch, String projection -> + sql """refresh table ${dbName}.evolution_refs""" + spark_iceberg """refresh table demo.${dbName}.evolution_refs""" + def sparkRows = spark_iceberg """ + select ${projection} + from demo.${dbName}.evolution_refs version as of '${branch}' + order by id + """ + def dorisRows = sql """ + select ${projection} + from evolution_refs@branch(${branch}) + order by id + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1", + "meta.cache.iceberg.table.ttl-second" = "0", + "meta.cache.iceberg.schema.ttl-second" = "0" + ) + """ + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + sql """set enable_fallback_to_original_planner = false""" + + sql """drop table if exists evolution_refs""" + sql """ + create table evolution_refs ( + id int not null, + region string, + bucket_key string not null, + event_time datetime, + amount decimal(12, 2), + payload struct + ) + partition by list (region, bucket(4, bucket_key), day(event_time)) () + properties ( + "format-version" = "2", + "write.format.default" = "parquet", + "write.delete.mode" = "merge-on-read", + "write.update.mode" = "merge-on-read", + "write.merge.mode" = "merge-on-read" + ) + """ + + // W01-S01: Doris writes the first snapshot using identity, string bucket and day transforms. + sql """ + insert into evolution_refs values + (1, 'CN', 'alpha', '2026-01-01 08:00:00', 10.10, struct(10, 'base-cn')), + (2, 'US', 'beta', '2026-01-02 09:00:00', 20.20, struct(20, 'base-us')), + (3, null, 'null-key', null, 30.30, struct(30, null)) + """ + String baseSnapshot = latestSnapshotId() + sql """alter table evolution_refs create tag base_tag as of version ${baseSnapshot}""" + sql """alter table evolution_refs create branch base_branch as of version ${baseSnapshot}""" + assertSparkMatchesDoris("", "id, region, bucket_key, event_time, amount") + + // W01-S02: Schema and partition spec evolve together before the next Doris write. + // Renaming the partition source column must preserve its Iceberg field id. + sql """alter table evolution_refs add column note string""" + sql """alter table evolution_refs rename column region zone""" + sql """ + alter table evolution_refs modify column payload struct< + metric:bigint, + label:string, + extra:string + > + """ + sql """ + alter table evolution_refs + replace partition key day(event_time) with month(event_time) as event_month + """ + sql """ + alter table evolution_refs + replace partition key bucket(4, bucket_key) with bucket(8, id) as id_bucket + """ + sql """alter table evolution_refs drop partition key region""" + sql """alter table evolution_refs add partition key truncate(2, bucket_key) as bucket_prefix""" + + sql """ + insert into evolution_refs values + (4, 'CN-east', 'gamma', '2026-02-01 10:00:00', 40.40, + struct(4000000000, 'new-cn', 'after-evolution'), 'new-spec'), + (5, 'DE-west', 'delta', '2026-03-02 11:00:00', 50.50, + struct(50, 'new-de', null), null), + (6, null, 'epsilon', null, 60.60, + struct(60, null, 'null-partition'), 'null-zone') + """ + String evolvedSnapshot = latestSnapshotId() + sql """alter table evolution_refs create tag evolved_tag as of version ${evolvedSnapshot}""" + + // W01-S03: Source-column filters must cover files written with both partition specs. + order_qt_current_rows """ + select id, zone, bucket_key, event_time, amount, payload.metric, payload.extra, note + from evolution_refs + order by id + """ + order_qt_cross_spec_zone_filter """ + select id from evolution_refs + where zone = 'CN' or zone like 'CN-%' + order by id + """ + order_qt_cross_spec_time_filter """ + select id from evolution_refs + where event_time is null or event_time >= timestamp '2026-02-01 00:00:00' + order by id + """ + order_qt_partition_specs """ + select spec_id, sum(record_count) + from evolution_refs\$partitions + group by spec_id + order by spec_id + """ + assertSparkMatchesDoris("", "id, zone, bucket_key, event_time, amount") + + // W01-S04: Numeric snapshot and tag retain both base data and the historical schema. + order_qt_base_snapshot """ + select id, region + from evolution_refs for version as of ${baseSnapshot} + order by id + """ + order_qt_base_tag """ + select id, region + from evolution_refs@tag(base_tag) + order by id + """ + order_qt_evolved_tag """ + select id, zone, note + from evolution_refs@tag(evolved_tag) + order by id + """ + + // W01-S05: Seed the current-spec branch partition before overwriting it; + // this proves replacement semantics while main and the base tag stay isolated. + sql """ + insert into evolution_refs@branch(base_branch) + (id, zone, bucket_key, event_time, amount, payload, note) + values + (7, 'JP-east', 'branch-a', '2026-04-01 12:00:00', 70.70, + struct(70, 'branch', 'current-schema'), 'branch-insert'), + (8, 'FR-west', 'branch-b', '2026-05-01 13:00:00', 80.80, + struct(80, 'branch-seed', 'current-schema'), 'branch-overwrite-seed') + """ + order_qt_branch_after_insert """ + select id, zone, note + from evolution_refs@branch(base_branch) + order by id + """ + order_qt_main_unchanged_after_branch_insert """ + select id from evolution_refs order by id + """ + assertSparkBranchMatchesDoris( + "base_branch", + "id, zone, bucket_key, event_time, amount, note") + + sql """ + insert overwrite table evolution_refs@branch(base_branch) + select 8, 'FR-west', 'branch-b', timestamp '2026-05-01 13:00:00', + cast(80.80 as decimal(12, 2)), + struct(cast(80 as bigint), 'branch-overwrite', 'current-schema'), + 'branch-overwrite' + """ + assertEquals(0L, (sql """ + select count(*) from evolution_refs@branch(base_branch) + where note = 'branch-overwrite-seed' + """)[0][0] as long) + order_qt_branch_after_overwrite """ + select id, zone, note + from evolution_refs@branch(base_branch) + order by id + """ + order_qt_base_tag_after_branch_overwrite """ + select id, region + from evolution_refs@tag(base_tag) + order by id + """ + order_qt_main_after_branch_overwrite """ + select id, zone, note + from evolution_refs + order by id + """ + assertSparkBranchMatchesDoris( + "base_branch", + "id, zone, bucket_key, event_time, amount, note") + assertSparkMatchesDoris("", "id, zone, bucket_key, event_time, amount") +} diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_duplicate_source_negative.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_duplicate_source_negative.groovy new file mode 100644 index 00000000000000..fbb3be22c44834 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_duplicate_source_negative.groovy @@ -0,0 +1,102 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_write_merge_duplicate_source_negative", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + String knownBugEnabled = context.config.otherConfigs.get("enableIcebergKnownBugTest") + if (knownBugEnabled == null || !knownBugEnabled.equalsIgnoreCase("true")) { + logger.info("skip isolated Iceberg known-bug test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_write_merge_duplicate_source_negative" + String dbName = "iceberg_write_merge_duplicate_source_negative_db" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1" + ) + """ + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + + sql """drop table if exists duplicate_source_target""" + sql """ + create table duplicate_source_target ( + id int, + region string, + payload string + ) + partition by list (region) () + properties ( + "format-version" = "2", + "write.delete.mode" = "merge-on-read", + "write.update.mode" = "merge-on-read", + "write.merge.mode" = "merge-on-read" + ) + """ + sql """insert into duplicate_source_target values (1, 'A', 'committed')""" + + long snapshotsBefore = + (sql """select count(*) from duplicate_source_target\$snapshots""")[0][0] as long + long filesBefore = + (sql """select count(*) from duplicate_source_target\$files""")[0][0] as long + + // Negative scenario: Iceberg MERGE cardinality permits only one source row + // to update a target row. The entire statement must fail before publishing. + test { + sql """ + merge into duplicate_source_target t + using ( + select 1 as id, 'B' as region, 'first-update' as payload + union all + select 1, 'C', 'second-update' + ) s + on t.id = s.id + when matched then update set + region = s.region, + payload = s.payload + """ + exception "more than one" + } + assertEquals(snapshotsBefore, + (sql """select count(*) from duplicate_source_target\$snapshots""")[0][0] as long) + assertEquals(filesBefore, + (sql """select count(*) from duplicate_source_target\$files""")[0][0] as long) + order_qt_duplicate_source_atomic_state """ + select id, region, payload + from duplicate_source_target + order by id + """ +} diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_semantics.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_semantics.groovy new file mode 100644 index 00000000000000..0bdac94f8959ea --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_semantics.groovy @@ -0,0 +1,207 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_write_merge_semantics", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_write_merge_semantics" + String dbName = "iceberg_write_merge_semantics_db" + + def assertSparkMatchesDoris = { String tableName -> + sql """refresh table ${dbName}.${tableName}""" + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + def sparkRows = spark_iceberg """ + select id, p_identity, p_bucket, p_truncate, payload, status + from demo.${dbName}.${tableName} + order by id, payload + """ + def dorisRows = sql """ + select id, p_identity, p_bucket, p_truncate, payload, status + from ${tableName} + order by id, payload + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1", + "meta.cache.iceberg.table.ttl-second" = "0", + "meta.cache.iceberg.schema.ttl-second" = "0" + ) + """ + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + + sql """drop table if exists merge_semantics""" + sql """ + create table merge_semantics ( + id int, + p_identity string, + p_bucket string, + p_truncate string not null, + payload string, + status string + ) + partition by list ( + p_identity, + bucket(8, p_bucket) + ) () + properties ( + "format-version" = "2", + "write.format.default" = "parquet", + "write.delete.mode" = "merge-on-read", + "write.update.mode" = "merge-on-read", + "write.merge.mode" = "merge-on-read" + ) + """ + sql """ + insert into merge_semantics values + (1, 'A', 'bucket-a', 'alpha', 'old-1', 'active'), + (2, null, 'bucket-b', 'beta', 'old-2', 'active'), + (3, 'C', 'bucket-c', 'charlie', 'old-3', 'active') + """ + + // WM01-S01: Conditions on MATCHED and NOT MATCHED clauses select exactly one + // action, including an update that moves a row across STRING identity and bucket transforms. + sql """ + merge into merge_semantics t + using ( + select 1 as id, 'A2' as p_identity, 'bucket-a2' as p_bucket, + 'delta' as p_truncate, 'new-1' as payload, 'U' as op, true as accepted + union all + select 3, 'C', 'bucket-c', 'charlie', 'old-3', 'D', true + union all + select 4, null, 'bucket-d', 'echo', 'new-4', 'I1', true + union all + select 5, 'E', 'bucket-e', 'foxtrot', 'new-5', 'I2', true + union all + select 6, 'F', 'bucket-f', 'golf', 'filtered-6', 'I1', false + ) s + on t.id = s.id + when matched and s.op = 'D' then delete + when matched and s.op = 'U' then update set + p_identity = s.p_identity, + p_bucket = s.p_bucket, + p_truncate = s.p_truncate, + payload = s.payload, + status = 'updated' + when not matched and s.op = 'I1' and s.accepted then + insert (id, p_identity, p_bucket, p_truncate, payload, status) + values (s.id, s.p_identity, s.p_bucket, s.p_truncate, s.payload, 'insert-1') + when not matched and s.op = 'I2' and s.accepted then + insert (id, p_identity, p_bucket, p_truncate, payload, status) + values (s.id, s.p_identity, s.p_bucket, s.p_truncate, s.payload, 'insert-2') + """ + order_qt_merge_conditional_clauses """ + select id, p_identity, p_bucket, p_truncate, payload, status + from merge_semantics + order by id + """ + order_qt_merge_string_partition_metadata """ + select spec_id, count(*), sum(record_count) + from merge_semantics\$partitions + group by spec_id + order by spec_id + """ + assertSparkMatchesDoris("merge_semantics") + + // WM01-S02: NULL-safe equality updates one nullable key while ordinary + // equality leaves NULL unmatched and executes the NOT MATCHED action. + sql """drop table if exists merge_null_keys""" + sql """ + create table merge_null_keys ( + id int, + p_identity string, + p_bucket string, + p_truncate string, + payload string, + status string + ) + properties ( + "format-version" = "2", + "write.format.default" = "orc", + "write.delete.mode" = "merge-on-read", + "write.update.mode" = "merge-on-read", + "write.merge.mode" = "merge-on-read" + ) + """ + sql """insert into merge_null_keys values (null, null, null, null, 'target-null', 'old')""" + sql """ + merge into merge_null_keys t + using ( + select cast(null as int) as id, cast(null as string) as p_identity, + cast(null as string) as p_bucket, cast(null as string) as p_truncate, + 'source-null-safe' as payload + ) s + on t.id <=> s.id + when matched then update set payload = s.payload, status = 'null-safe-update' + """ + sql """ + merge into merge_null_keys t + using ( + select cast(null as int) as id, cast(null as string) as p_identity, + cast(null as string) as p_bucket, cast(null as string) as p_truncate, + 'source-ordinary' as payload + ) s + on t.id = s.id + when matched then update set payload = 'must-not-update' + when not matched then + insert (id, p_identity, p_bucket, p_truncate, payload, status) + values (s.id, s.p_identity, s.p_bucket, s.p_truncate, s.payload, 'ordinary-insert') + """ + order_qt_merge_null_keys """ + select id, p_identity, p_bucket, p_truncate, payload, status + from merge_null_keys + order by payload + """ + assertSparkMatchesDoris("merge_null_keys") + + // WM01-S03: An unconditional clause must be last within its clause family; + // otherwise a later conditional clause is unreachable. + long snapshotsBeforeInvalidClause = + (sql """select count(*) from merge_semantics\$snapshots""")[0][0] as long + test { + sql """ + merge into merge_semantics t + using (select 2 as id, 'X' as payload) s + on t.id = s.id + when matched then update set payload = s.payload + when matched and s.payload = 'X' then delete + """ + exception "Only the last matched clause could without case predicate" + } + assertEquals(snapshotsBeforeInvalidClause, + (sql """select count(*) from merge_semantics\$snapshots""")[0][0] as long) +} diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_truncate_negative.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_truncate_negative.groovy new file mode 100644 index 00000000000000..31bfeb251ab609 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_merge_truncate_negative.groovy @@ -0,0 +1,100 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_write_merge_truncate_negative", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + String crashTestEnabled = context.config.otherConfigs.get("enableIcebergCrashTest") + if (enabled == null || !enabled.equalsIgnoreCase("true") + || crashTestEnabled == null || !crashTestEnabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg crash test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_write_merge_truncate_negative" + String dbName = "iceberg_write_merge_truncate_negative_db" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1", + "meta.cache.iceberg.table.ttl-second" = "0", + "meta.cache.iceberg.schema.ttl-second" = "0" + ) + """ + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + + sql """drop table if exists merge_truncate_negative""" + sql """ + create table merge_truncate_negative ( + id int not null, + partition_value string not null, + payload string + ) + partition by list (truncate(2, partition_value)) () + properties ( + "format-version" = "2", + "write.format.default" = "parquet", + "write.merge.mode" = "merge-on-read", + "write.update.mode" = "merge-on-read" + ) + """ + sql """insert into merge_truncate_negative values (1, 'alpha', 'before')""" + + // WM03-S01: A MERGE source projection is nullable even when every source + // value and the Iceberg target column are NOT NULL. The writer must reject + // an invalid input as a query error and must never terminate a BE. + sql """ + merge into merge_truncate_negative t + using ( + select 1 as id, 'beta' as partition_value, 'after' as payload + union all + select 2, 'gamma', 'inserted' + ) s + on t.id = s.id + when matched then update set + partition_value = s.partition_value, + payload = s.payload + when not matched then + insert (id, partition_value, payload) + values (s.id, s.partition_value, s.payload) + """ + order_qt_merge_truncate_after_fix """ + select id, partition_value, payload + from merge_truncate_negative + order by id + """ + // Logical rows cannot prove the MERGE writer used truncate(2) when routing + // its updated and inserted records. + order_qt_merge_truncate_physical_partitions """ + select distinct hex(struct_element(`partition`, 'partition_value_trunc_2')) + from merge_truncate_negative\$partitions + order by 1 + """ +} diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_nullability_atomicity.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_nullability_atomicity.groovy new file mode 100644 index 00000000000000..263134dac3c235 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_nullability_atomicity.groovy @@ -0,0 +1,133 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_write_nullability_atomicity", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_write_nullability_atomicity" + String dbName = "iceberg_write_nullability_atomicity_db" + String internalDb = "iceberg_write_nullability_atomicity_internal_db" + + sql """drop database if exists internal.${internalDb} force""" + sql """create database internal.${internalDb}""" + sql """drop table if exists internal.${internalDb}.nullable_source""" + sql """ + create table internal.${internalDb}.nullable_source ( + id int, + required_text string, + optional_text string + ) + duplicate key(id) + distributed by hash(id) buckets 3 + properties ("replication_num" = "1") + """ + sql """ + insert into internal.${internalDb}.nullable_source values + (2, 'valid-select', null), + (4, 'valid-after-invalid', 'value') + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1", + "meta.cache.iceberg.table.ttl-second" = "0", + "meta.cache.iceberg.schema.ttl-second" = "0" + ) + """ + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + sql """set enable_fallback_to_original_planner = false""" + + sql """drop table if exists required_sink""" + sql """ + create table required_sink ( + id int not null, + required_text string not null, + optional_text string + ) + partition by list (bucket(8, id)) () + properties ( + "format-version" = "2", + "write.format.default" = "parquet" + ) + """ + sql """insert into required_sink values (1, 'committed', null)""" + + long snapshotsBeforeEvolution = (sql """ + select count(*) from required_sink\$snapshots + """)[0][0] as long + + // W06-S01: Adding a required field or tightening a nullable field is rejected. + test { + sql """alter table required_sink add column new_required int not null""" + exception "doesn't have a default value" + } + test { + sql """alter table required_sink modify column optional_text string not null""" + exception "Can not change nullable column optional_text to not null" + } + assertEquals(snapshotsBeforeEvolution, (sql """ + select count(*) from required_sink\$snapshots + """)[0][0] as long) + + // W06-S02: Distributed and VALUES writes preserve nullable fields while required fields are valid. + sql """ + insert into required_sink + select id, required_text, optional_text + from internal.${internalDb}.nullable_source + """ + sql """insert into required_sink values (5, 'valid-values-retry', null)""" + assertEquals(snapshotsBeforeEvolution + 2, (sql """ + select count(*) from required_sink\$snapshots + """)[0][0] as long) + order_qt_required_after_retry """ + select id, required_text, optional_text + from required_sink + order by id + """ + + sql """refresh table ${dbName}.required_sink""" + spark_iceberg """refresh table demo.${dbName}.required_sink""" + def sparkRows = spark_iceberg """ + select id, required_text, optional_text + from demo.${dbName}.required_sink + order by id + """ + def dorisRows = sql """ + select id, required_text, optional_text + from required_sink + order by id + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) +} diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_nullable_truncate_negative.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_nullable_truncate_negative.groovy new file mode 100644 index 00000000000000..25074a0af78d63 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_nullable_truncate_negative.groovy @@ -0,0 +1,97 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_write_nullable_truncate_negative", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + // This opt-in switch isolates a BE-fatal negative scenario from the shared P0 cluster. + // Enable it only in a cluster whose BE processes can be restarted after the suite. + String crashTestEnabled = context.config.otherConfigs.get("enableIcebergCrashTest") + if (crashTestEnabled == null || !crashTestEnabled.equalsIgnoreCase("true")) { + logger.info("skip isolated Iceberg crash regression") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_write_nullable_truncate_negative" + String dbName = "iceberg_write_nullable_truncate_negative_db" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1" + ) + """ + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + + sql """ + create table nullable_truncate ( + id int not null, + zone string + ) + partition by list (zone) () + properties ( + "format-version" = "2", + "write.delete.mode" = "merge-on-read", + "write.update.mode" = "merge-on-read", + "write.merge.mode" = "merge-on-read" + ) + """ + sql """insert into nullable_truncate values (1, 'CN'), (2, null)""" + + // Negative scenario: evolve to a truncate transform whose source remains nullable, + // then write both non-NULL and NULL partition values through Doris. + sql """ + alter table nullable_truncate + add partition key truncate(2, zone) as zone_prefix + """ + sql """insert into nullable_truncate values (3, 'US-east'), (4, null)""" + // UPDATE creates a Nullable projection even though the non-NULL branch is + // selected for id 3; the partition transformer must preserve that wrapper. + sql """ + update nullable_truncate + set zone = if(id = 3, 'US-west', cast(null as string)) + where id in (3, 4) + """ + + order_qt_nullable_truncate_rows """ + select id, zone from nullable_truncate order by id + """ + order_qt_nullable_truncate_physical_partitions """ + select distinct spec_id, + hex(struct_element(`partition`, 'zone')), + hex(struct_element(`partition`, 'zone_prefix')) + from nullable_truncate\$partitions + order by spec_id, 2, 3 + """ +} diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_order_distribution_properties.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_order_distribution_properties.groovy new file mode 100644 index 00000000000000..41e8b5a199ef4e --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_order_distribution_properties.groovy @@ -0,0 +1,215 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_write_order_distribution_properties", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_write_order_distribution_properties" + String dbName = "iceberg_write_order_distribution_properties_db" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1", + "meta.cache.iceberg.table.ttl-second" = "0", + "meta.cache.iceberg.schema.ttl-second" = "0" + ) + """ + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + + sql """drop table if exists ordered_evolution""" + sql """ + create table ordered_evolution ( + id int, + region string, + payload string, + score int + ) + order by (region asc nulls last, id desc nulls first) + partition by list (region) () + properties ( + "format-version" = "2", + "write.format.default" = "parquet", + "write.delete.mode" = "merge-on-read", + "write.update.mode" = "merge-on-read", + "write.merge.mode" = "merge-on-read", + "write.distribution-mode" = "range" + ) + """ + + // WP01-S01: The planned Iceberg write contains the declared global order, + // including direction and NULL ordering. + explain { + sql """ + insert into ordered_evolution + select number, + if(number % 4 = 0, null, concat('R', number % 3)), + concat('payload-', number), + number + from numbers('number' = '32') + """ + contains "ORDER BY (`region` ASC NULLS LAST, `id` DESC NULLS FIRST)" + } + + // WP01-S02: Force several sorted writer flushes on a distributed source. + // This exercises global order, NULL partitions and file rollover together. + sql """set iceberg_write_target_file_size_bytes = 51200""" + sql """ + insert into ordered_evolution + select number, + if(number % 7 = 0, null, concat('R', number % 5)), + concat('payload-', number, '-', repeat('x', 64)), + number + from numbers('number' = '10000') + """ + def filesAfterInsert = sql """ + select count(*), sum(record_count) + from ordered_evolution\$files + """ + assertEquals(10000L, filesAfterInsert[0][1] as long) + def maxFilesPerPartition = sql """ + select max(file_count) + from ( + select `partition`, count(*) as file_count + from ordered_evolution\$files + group by `partition` + ) partition_file_counts + """ + // Partition fan-out already creates multiple files globally, so rollover + // is proven only when one physical partition owns more than one file. + assertTrue((maxFilesPerPartition[0][0] as long) > 1L) + + // WP01-S03: Schema evolution and row-level DML continue to use the current + // sort order and preserve Spark/Doris visible results. + sql """alter table ordered_evolution add column status string""" + sql """ + update ordered_evolution + set region = 'R-updated', score = score + 10000, status = 'updated' + where id in (1, 7) + """ + sql """ + merge into ordered_evolution t + using ( + select 2 as id, cast(null as string) as region, 'merge-update' as payload, + 20002 as score, 'U' as op + union all + select 10001, 'R-new', 'merge-insert', 10001, 'I' + ) s + on t.id = s.id + when matched then update set + region = s.region, + payload = s.payload, + score = s.score, + status = 'merged' + when not matched then + insert (id, region, payload, score, status) + values (s.id, s.region, s.payload, s.score, 'inserted') + """ + order_qt_ordered_evolution_changed_rows """ + select id, region, payload, score, status + from ordered_evolution + where id in (1, 2, 7, 10001) + order by id + """ + order_qt_ordered_evolution_files """ + select lower(file_format), sum(record_count) + from ordered_evolution\$files + group by lower(file_format) + order by lower(file_format) + """ + spark_iceberg """refresh table demo.${dbName}.ordered_evolution""" + def sparkRows = spark_iceberg """ + select id, region, payload, score, status + from demo.${dbName}.ordered_evolution + where id in (1, 2, 7, 10001) + order by id + """ + def dorisRows = sql """ + select id, region, payload, score, status + from ordered_evolution + where id in (1, 2, 7, 10001) + order by id + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + sql """set iceberg_write_target_file_size_bytes = 0""" + + // WP01-S04: All official distribution-mode property values remain + // correctness-compatible with Doris distributed writes. + for (String mode : ["none", "hash", "range"]) { + String tableName = "distribution_${mode}" + sql """drop table if exists ${tableName}""" + sql """ + create table ${tableName} ( + id int, + region string, + payload string + ) + partition by list (region, bucket(8, id)) () + properties ( + "format-version" = "2", + "write.format.default" = "orc", + "write.distribution-mode" = "${mode}" + ) + """ + sql """ + insert into ${tableName} + select number, + if(number % 11 = 0, null, concat('R', number % 9)), + concat('${mode}-', number) + from numbers('number' = '512') + """ + def distributionRows = sql """select count(*), count(distinct id) from ${tableName}""" + assertEquals(512L, distributionRows[0][0] as long) + assertEquals(512L, distributionRows[0][1] as long) + def sparkDistributionRows = spark_iceberg """ + select id, region, payload + from demo.${dbName}.${tableName} + order by id + """ + def dorisDistributionRows = sql """ + select id, region, payload + from ${tableName} + order by id + """ + assertSparkDorisResultEquals(sparkDistributionRows, dorisDistributionRows) + } + order_qt_distribution_mode_counts """ + select 'hash', count(*) from distribution_hash + union all + select 'none', count(*) from distribution_none + union all + select 'range', count(*) from distribution_range + order by 1 + """ +} diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_overwrite_atomicity.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_overwrite_atomicity.groovy new file mode 100644 index 00000000000000..fe6d8f2cc3799a --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_overwrite_atomicity.groovy @@ -0,0 +1,163 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_write_overwrite_atomicity", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_write_overwrite_atomicity" + String dbName = "iceberg_write_overwrite_atomicity_db" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1" + ) + """ + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + + sql """drop table if exists overwrite_atomicity""" + sql """ + create table overwrite_atomicity ( + id int not null, + region string, + payload string + ) + partition by list (region) () + properties ( + "format-version" = "2", + "write.format.default" = "parquet" + ) + """ + sql """ + insert into overwrite_atomicity values + (1, 'A', 'committed-a'), + (2, 'B', 'committed-b') + """ + sql """alter table overwrite_atomicity create branch retry_branch""" + sql """set enable_strict_cast = true""" + + // WO03-S01: A distributed expression failure must not publish a partial + // overwrite, remove an existing partition, or create a new snapshot. + long snapshotsBeforeFailure = + (sql """select count(*) from overwrite_atomicity\$snapshots""")[0][0] as long + long filesBeforeFailure = + (sql """select count(*) from overwrite_atomicity\$files""")[0][0] as long + test { + sql """ + insert overwrite table overwrite_atomicity + select cast(if(number = 2, 'invalid-id', cast(number + 10 as string)) as int), + if(number % 2 = 0, 'A', 'C'), + concat('candidate-', number) + from numbers('number' = '8') + """ + exception "can't cast to INT in strict mode" + } + assertEquals(snapshotsBeforeFailure, + (sql """select count(*) from overwrite_atomicity\$snapshots""")[0][0] as long) + assertEquals(filesBeforeFailure, + (sql """select count(*) from overwrite_atomicity\$files""")[0][0] as long) + order_qt_overwrite_failure_state """ + select id, region, payload + from overwrite_atomicity + order by id + """ + + // WO03-S02: The same invariant applies to a branch-qualified overwrite. + test { + sql """ + insert overwrite table overwrite_atomicity@branch(retry_branch) + select cast(if(number = 3, 'invalid-id', cast(number + 20 as string)) as int), + 'A', + concat('branch-candidate-', number) + from numbers('number' = '8') + """ + exception "can't cast to INT in strict mode" + } + order_qt_branch_overwrite_failure_state """ + select id, region, payload + from overwrite_atomicity@branch(retry_branch) + order by id + """ + order_qt_main_after_branch_overwrite_failure """ + select id, region, payload + from overwrite_atomicity + order by id + """ + // A branch failure is only atomic if both engines still observe the same + // pre-write branch state; a Doris-only query could hide unreadable files. + spark_iceberg """refresh table demo.${dbName}.overwrite_atomicity""" + def sparkBranchRows = spark_iceberg """ + select id, region, payload + from demo.${dbName}.overwrite_atomicity version as of 'retry_branch' + order by id + """ + def dorisBranchRows = sql """ + select id, region, payload + from overwrite_atomicity@branch(retry_branch) + order by id + """ + assertSparkDorisResultEquals(sparkBranchRows, dorisBranchRows) + + // WO03-S03: Retry the corrected logical operation. Each replacement row + // becomes visible exactly once and only one new main snapshot is committed. + sql """ + insert overwrite table overwrite_atomicity + select number + 10, + if(number % 2 = 0, 'A', 'C'), + concat('candidate-', number) + from numbers('number' = '8') + """ + assertEquals(snapshotsBeforeFailure + 1, + (sql """select count(*) from overwrite_atomicity\$snapshots""")[0][0] as long) + order_qt_overwrite_retry """ + select id, region, payload, count(*) + from overwrite_atomicity + group by id, region, payload + order by id + """ + // Refresh after the retry so cross-engine validation covers the corrected + // overwrite commit, not merely the failure baselines. + spark_iceberg """refresh table demo.${dbName}.overwrite_atomicity""" + def sparkRows = spark_iceberg """ + select id, region, payload + from demo.${dbName}.overwrite_atomicity + order by id + """ + def dorisRows = sql """ + select id, region, payload + from overwrite_atomicity + order by id + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) +} diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_overwrite_delete_files.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_overwrite_delete_files.groovy new file mode 100644 index 00000000000000..d787962094a72a --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_overwrite_delete_files.groovy @@ -0,0 +1,197 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_write_overwrite_delete_files", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_write_overwrite_delete_files" + String dbName = "iceberg_write_overwrite_delete_files_db" + + def assertSparkMatchesDoris = { + sql """refresh table ${dbName}.overwrite_delete_files""" + spark_iceberg """refresh table demo.${dbName}.overwrite_delete_files""" + def sparkRows = spark_iceberg """ + select id, region, bucket_key, payload + from demo.${dbName}.overwrite_delete_files + order by id + """ + def dorisRows = sql """ + select id, region, bucket_key, payload + from overwrite_delete_files + order by id + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1", + "meta.cache.iceberg.table.ttl-second" = "0", + "meta.cache.iceberg.schema.ttl-second" = "0" + ) + """ + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + + sql """drop table if exists overwrite_delete_files""" + sql """ + create table overwrite_delete_files ( + id int not null, + region string, + bucket_key string not null, + payload string + ) + partition by list (region, bucket(4, bucket_key)) () + properties ( + "format-version" = "2", + "write.format.default" = "parquet", + "write.delete.mode" = "merge-on-read", + "write.update.mode" = "merge-on-read", + "write.merge.mode" = "merge-on-read" + ) + """ + sql """ + insert into overwrite_delete_files values + (1, 'A', 'alpha', 'keep-a'), + (2, 'A', 'beta', 'delete-a'), + (3, 'B', 'gamma', 'move-b-to-c'), + (4, 'B', 'delta', 'merge-delete-b'), + (5, null, 'null-key', 'keep-null') + """ + String baseSnapshot = (sql """ + select snapshot_id from overwrite_delete_files\$snapshots + order by committed_at desc limit 1 + """)[0][0].toString() + sql """alter table overwrite_delete_files create tag before_row_dml as of version ${baseSnapshot}""" + + // WO02-S01: Generate position deletes in several partitions and move one + // updated row to a new partition before overwrite. + sql """delete from overwrite_delete_files where id = 2""" + sql """ + update overwrite_delete_files + set region = 'C', bucket_key = 'gamma-new', payload = 'moved-to-c' + where id = 3 + """ + sql """ + merge into overwrite_delete_files t + using ( + select 4 as id, 'D' as region, 'delta-new' as bucket_key, + 'delete' as payload, 'D' as op + union all + select 6, 'B', 'echo', 'merge-insert-b', 'I' + ) s + on t.id = s.id + when matched and s.op = 'D' then delete + when not matched then + insert (id, region, bucket_key, payload) + values (s.id, s.region, s.bucket_key, s.payload) + """ + order_qt_before_overwrite_rows """ + select id, region, bucket_key, payload + from overwrite_delete_files + order by id + """ + order_qt_before_overwrite_delete_files """ + select spec_id, sum(record_count) + from overwrite_delete_files\$delete_files + group by spec_id + order by spec_id + """ + + // WO02-S02: Overwrite only current partitions produced by the input. Delete + // files that refer to replaced data must not hide the replacement rows. + sql """ + insert overwrite table overwrite_delete_files + values + (10, 'A', 'alpha', 'replacement-a'), + (11, 'B', 'echo', 'replacement-b') + """ + order_qt_after_overwrite_rows """ + select id, region, bucket_key, payload + from overwrite_delete_files + order by id + """ + order_qt_after_overwrite_delete_files """ + select spec_id, sum(record_count) + from overwrite_delete_files\$delete_files + group by spec_id + order by spec_id + """ + order_qt_before_row_dml_tag """ + select id, region, bucket_key, payload + from overwrite_delete_files@tag(before_row_dml) + order by id + """ + assertSparkMatchesDoris() + + // WO02-S03: Repeat after partition evolution so old-spec delete files and + // current-spec replacements coexist without leaking across specs. + sql """ + alter table overwrite_delete_files + replace partition key bucket(4, bucket_key) + with bucket(8, bucket_key) as bucket_key_8 + """ + sql """ + alter table overwrite_delete_files + add partition key truncate(1, bucket_key) as bucket_key_prefix + """ + sql """ + insert into overwrite_delete_files values + (12, 'A', 'alpha-new', 'new-spec-a'), + (13, null, 'null-new', 'new-spec-null') + """ + sql """delete from overwrite_delete_files where id = 12""" + sql """ + insert overwrite table overwrite_delete_files + values (14, 'A', 'alpha-new', 'new-spec-replacement-a') + """ + order_qt_evolved_overwrite_rows """ + select id, region, bucket_key, payload + from overwrite_delete_files + order by id + """ + order_qt_evolved_overwrite_specs """ + select spec_id, count(*), sum(record_count) + from overwrite_delete_files\$partitions + group by spec_id + order by spec_id + """ + order_qt_evolved_overwrite_delete_files """ + select spec_id, sum(record_count) + from overwrite_delete_files\$delete_files + group by spec_id + order by spec_id + """ + assertSparkMatchesDoris() +} diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_overwrite_evolution.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_overwrite_evolution.groovy new file mode 100644 index 00000000000000..07fa302ba52193 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_overwrite_evolution.groovy @@ -0,0 +1,180 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_write_overwrite_evolution", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_write_overwrite_evolution" + String dbName = "iceberg_write_overwrite_evolution_db" + + def assertSparkMatchesDoris = { + sql """refresh table ${dbName}.overwrite_evolution""" + spark_iceberg """refresh table demo.${dbName}.overwrite_evolution""" + def sparkRows = spark_iceberg """ + select id, region, code, event_time, payload + from demo.${dbName}.overwrite_evolution + order by id + """ + def dorisRows = sql """ + select id, region, code, event_time, payload + from overwrite_evolution + order by id + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1", + "meta.cache.iceberg.table.ttl-second" = "0", + "meta.cache.iceberg.schema.ttl-second" = "0" + ) + """ + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + + sql """drop table if exists overwrite_evolution""" + sql """ + create table overwrite_evolution ( + id int not null, + region string, + code string not null, + event_time datetime, + payload string + ) + partition by list (region, bucket(4, code), day(event_time)) () + properties ( + "format-version" = "2", + "write.format.default" = "parquet" + ) + """ + + // WO01-S01: Build old-spec files and protect the baseline with both a tag + // and a branch before changing the partition granularity. + sql """ + insert into overwrite_evolution values + (1, 'A', 'alpha', '2026-01-01 01:10:00', 'old-hour-1'), + (2, 'A', 'beta', '2026-01-01 02:20:00', 'old-hour-2'), + (3, null, 'null-region', null, 'old-null'), + (4, 'B', 'delta', '2026-01-02 01:30:00', 'old-other-day') + """ + String baseSnapshot = (sql """ + select snapshot_id from overwrite_evolution\$snapshots + order by committed_at desc limit 1 + """)[0][0].toString() + sql """alter table overwrite_evolution create tag overwrite_base as of version ${baseSnapshot}""" + sql """alter table overwrite_evolution create branch overwrite_audit as of version ${baseSnapshot}""" + + // WO01-S02: Keep day(event_time), add hour(event_time), replace the STRING + // bucket and add STRING truncate. Old and new specs must remain independently visible. + sql """alter table overwrite_evolution add partition key hour(event_time) as event_hour""" + sql """ + alter table overwrite_evolution + replace partition key bucket(4, code) with bucket(8, code) as code_bucket_8 + """ + sql """alter table overwrite_evolution add partition key truncate(2, code) as code_prefix""" + sql """ + insert into overwrite_evolution values + (5, 'A', 'alpha-new', '2026-01-01 01:40:00', 'new-spec-before-overwrite'), + (6, null, 'null-new', null, 'new-spec-null'), + (7, 'C', 'charlie', '2026-02-01 03:00:00', 'new-spec-other-month') + """ + + // WO01-S03: Dynamic overwrite operates on current-spec partitions. It must + // not silently remove old day-level files that cannot be equal to a new spec. + sql """ + insert overwrite table overwrite_evolution + values (8, 'A', 'alpha-new', '2026-01-01 01:40:00', 'overwrite-current-spec') + """ + order_qt_overwrite_current """ + select id, region, code, event_time, payload + from overwrite_evolution + order by id + """ + order_qt_overwrite_specs """ + select spec_id, count(*), sum(record_count) + from overwrite_evolution\$partitions + group by spec_id + order by spec_id + """ + order_qt_overwrite_base_tag """ + select id, region, code, event_time, payload + from overwrite_evolution@tag(overwrite_base) + order by id + """ + order_qt_overwrite_audit_branch """ + select id, region, code, event_time, payload + from overwrite_evolution@branch(overwrite_audit) + order by id + """ + assertSparkMatchesDoris() + + // WO01-S04: Seed the complete post-evolution partition tuple before its + // overwrite; reusing id 9 also fixes the new bucket value across both writes. + sql """alter table overwrite_evolution drop partition key region""" + sql """alter table overwrite_evolution add partition key bucket(4, id) as id_bucket""" + sql """ + insert into overwrite_evolution + values (9, null, 'null-new', null, 'overwrite-seed-null-current-spec') + """ + assertEquals(1L, (sql """ + select count(*) from overwrite_evolution + where payload = 'overwrite-seed-null-current-spec' + """)[0][0] as long) + sql """ + insert overwrite table overwrite_evolution + values (9, null, 'null-new', null, 'overwrite-null-current-spec') + """ + assertEquals(0L, (sql """ + select count(*) from overwrite_evolution + where payload = 'overwrite-seed-null-current-spec' + """)[0][0] as long) + order_qt_overwrite_after_drop_identity """ + select id, region, code, event_time, payload + from overwrite_evolution + order by id + """ + order_qt_overwrite_after_drop_identity_specs """ + select spec_id, count(*), sum(record_count) + from overwrite_evolution\$partitions + group by spec_id + order by spec_id + """ + order_qt_overwrite_base_tag_after_second_evolution """ + select id, region, code, event_time, payload + from overwrite_evolution@tag(overwrite_base) + order by id + """ + assertSparkMatchesDoris() +} diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_partition_types_null.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_partition_types_null.groovy new file mode 100644 index 00000000000000..aeb8614d5bebfa --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_partition_types_null.groovy @@ -0,0 +1,287 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_write_partition_types_null", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_write_partition_types_null" + String dbName = "iceberg_write_partition_types_null_db" + + def assertSparkMatchesDoris = { String tableName, String projection -> + sql """refresh table ${dbName}.${tableName}""" + spark_iceberg """refresh table demo.${dbName}.${tableName}""" + def sparkRows = spark_iceberg """ + select ${projection} + from demo.${dbName}.${tableName} + order by id + """ + def dorisRows = sql """ + select ${projection} + from ${tableName} + order by id + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + } + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1", + "meta.cache.iceberg.table.ttl-second" = "0", + "meta.cache.iceberg.schema.ttl-second" = "0" + ) + """ + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + sql """set enable_fallback_to_original_planner = false""" + + // W05-S00: BOOLEAN is valid for identity but not for Iceberg's bucket transform. + // The invalid table must be rejected instead of creating a table that fails on its first write. + test { + sql """ + create table invalid_boolean_bucket ( + id int, + p_bool boolean + ) + partition by list (bucket(4, p_bool)) () + """ + exception "Invalid source type boolean for transform: bucket[4]" + } + + // W05-S01: STRING supports identity, bucket and truncate together. + // NULL is routed by the nullable identity source while transform-specific sources stay required. + sql """drop table if exists string_partitions""" + sql """ + create table string_partitions ( + id int not null, + p_string string, + p_bucket string not null, + p_truncate string not null, + payload string + ) + partition by list (p_string, bucket(8, p_bucket), truncate(2, p_truncate)) () + properties ( + "format-version" = "2", + "write.format.default" = "parquet" + ) + """ + sql """ + insert into string_partitions values + (1, 'alpha', 'bucket-a', 'alpha', 'a'), + (2, 'alphabet', 'bucket-b', 'alphabet', 'ab'), + (3, '', 'bucket-empty', '', 'empty'), + (4, '中文', 'bucket-unicode', '中文', 'unicode'), + (5, null, 'bucket-null-identity', 'null-identity', 'null-string') + """ + order_qt_string_rows """ + select id, p_string, p_bucket, p_truncate, payload + from string_partitions + order by id + """ + order_qt_string_null_filter """ + select id from string_partitions where p_string is null order by id + """ + + // W05-S02: Replace a STRING bucket transform and keep old/new specs filterable. + sql """ + alter table string_partitions + replace partition key bucket(8, p_bucket) + with bucket(16, p_bucket) as p_string_bucket_16 + """ + sql """ + insert into string_partitions values + (6, 'beta', 'bucket-new', 'beta', 'new-spec'), + (7, null, 'bucket-new-null-identity', 'null-identity', 'new-null-string') + """ + order_qt_string_cross_spec_filter """ + select id from string_partitions + where p_string is null or p_string like 'alp%' + order by id + """ + // Filtering the replaced bucket source exercises both bucket widths; + // predicates on the unchanged identity source cannot detect a bad bucket. + order_qt_string_cross_spec_bucket_filter """ + select id from string_partitions + where p_bucket in ('bucket-a', 'bucket-new') + order by id + """ + order_qt_string_partition_specs """ + select spec_id, sum(record_count) + from string_partitions\$partitions + group by spec_id + order by spec_id + """ + assertSparkMatchesDoris( + "string_partitions", + "id, p_string, p_bucket, p_truncate, payload") + + // W05-S03: Integer/BIGINT/DECIMAL bucket or truncate transforms and BOOLEAN identity + // must all route NULL to valid Iceberg partitions. + sql """drop table if exists numeric_partitions""" + sql """ + create table numeric_partitions ( + id int not null, + p_int int, + p_bigint bigint, + p_decimal decimal(12, 2), + p_bool boolean, + payload string + ) + partition by list ( + bucket(4, p_int), + bucket(8, p_bigint), + truncate(100, p_bigint), + bucket(8, p_decimal), + truncate(10, p_decimal), + p_bool + ) () + properties ( + "format-version" = "2", + "write.format.default" = "orc" + ) + """ + sql """ + insert into numeric_partitions values + (1, 1, 101, 11.11, true, 'positive'), + (2, -1, -101, -11.11, false, 'negative'), + (3, 0, 0, 0.00, null, 'zero-null-bool'), + (4, null, null, null, null, 'all-null') + """ + order_qt_numeric_rows """ + select id, p_int, p_bigint, p_decimal, p_bool, payload + from numeric_partitions + order by id + """ + order_qt_numeric_null_filter """ + select id from numeric_partitions + where p_int is null or p_bigint is null or p_decimal is null or p_bool is null + order by id + """ + order_qt_numeric_partitions """ + select spec_id, sum(record_count) + from numeric_partitions\$partitions + group by spec_id + order by spec_id + """ + // Logical rows and spec totals do not expose incorrect transform values. + order_qt_numeric_physical_partitions """ + select struct_element(`partition`, 'p_int_bucket'), + struct_element(`partition`, 'p_bigint_bucket'), + struct_element(`partition`, 'p_bigint_trunc'), + struct_element(`partition`, 'p_decimal_bucket'), + struct_element(`partition`, 'p_decimal_trunc'), + struct_element(`partition`, 'p_bool'), + record_count + from numeric_partitions\$partitions + order by 1, 2, 3, 4, 5, 6 + """ + assertSparkMatchesDoris( + "numeric_partitions", + "id, p_int, p_bigint, p_decimal, p_bool, payload") + + // W05-S04: DATE/DATETIME time transforms accept boundary values and NULL. + sql """drop table if exists temporal_partitions""" + sql """ + create table temporal_partitions ( + id int not null, + p_date_bucket date, + p_date_year date, + p_date_month date, + p_ts_bucket datetime, + p_ts_day datetime, + p_ts_hour datetime, + payload string + ) + partition by list ( + bucket(8, p_date_bucket), + year(p_date_year), + month(p_date_month), + bucket(8, p_ts_bucket), + day(p_ts_day), + hour(p_ts_hour) + ) () + properties ( + "format-version" = "2", + "write.format.default" = "parquet" + ) + """ + sql """ + insert into temporal_partitions values + (1, '1969-12-31', '1969-12-31', '1969-12-31', + '1969-12-31 23:59:59', '1969-12-31 23:59:59', '1969-12-31 23:59:59', + 'before-epoch'), + (2, '1970-01-01', '1970-01-01', '1970-01-01', + '1970-01-01 00:00:00', '1970-01-01 00:00:00', '1970-01-01 00:00:00', + 'epoch'), + (3, '2024-02-29', '2024-02-29', '2024-02-29', + '2024-02-29 12:34:56', '2024-02-29 12:34:56', '2024-02-29 12:34:56', + 'leap-day'), + (4, null, null, null, null, null, null, 'all-null') + """ + order_qt_temporal_rows """ + select id, p_date_bucket, p_date_year, p_date_month, + p_ts_bucket, p_ts_day, p_ts_hour, payload + from temporal_partitions + order by id + """ + order_qt_temporal_filters """ + select id from temporal_partitions + where p_date_bucket is null + or p_ts_hour < timestamp '1970-01-01 00:00:00' + or p_date_month = date '2024-02-29' + order by id + """ + order_qt_temporal_partitions """ + select spec_id, sum(record_count) + from temporal_partitions\$partitions + group by spec_id + order by spec_id + """ + // Read each time-transform field from Iceberg metadata so epoch boundaries + // and NULL routing are observable independently of source-row equality. + order_qt_temporal_physical_partitions """ + select struct_element(`partition`, 'p_date_bucket_bucket'), + struct_element(`partition`, 'p_date_year_year'), + struct_element(`partition`, 'p_date_month_month'), + struct_element(`partition`, 'p_ts_bucket_bucket'), + struct_element(`partition`, 'p_ts_day_day'), + struct_element(`partition`, 'p_ts_hour_hour'), + record_count + from temporal_partitions\$partitions + order by 1, 2, 3, 4, 5, 6 + """ + assertSparkMatchesDoris( + "temporal_partitions", + "id, p_date_bucket, p_date_year, p_date_month, " + + "p_ts_bucket, p_ts_day, p_ts_hour, payload") +} diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_required_null_select_negative.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_required_null_select_negative.groovy new file mode 100644 index 00000000000000..765e5c58345dae --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_required_null_select_negative.groovy @@ -0,0 +1,103 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_write_required_null_select_negative", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + // This opt-in switch isolates a write that can publish an unreadable Iceberg data file. + String knownBugTestEnabled = context.config.otherConfigs.get("enableIcebergKnownBugTest") + if (knownBugTestEnabled == null || !knownBugTestEnabled.equalsIgnoreCase("true")) { + logger.info("skip isolated Iceberg known-bug regression") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_write_required_null_select_negative" + String dbName = "iceberg_write_required_null_select_negative_db" + String internalDb = "iceberg_write_required_null_select_negative_internal_db" + + sql """drop database if exists internal.${internalDb} force""" + sql """create database internal.${internalDb}""" + sql """ + create table internal.${internalDb}.nullable_source ( + id int, + required_text string + ) + duplicate key(id) + distributed by hash(id) buckets 3 + properties ("replication_num" = "1") + """ + sql """ + insert into internal.${internalDb}.nullable_source values + (1, 'valid'), + (2, null) + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1" + ) + """ + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + + sql """ + create table required_select ( + id int not null, + required_text string not null + ) + partition by list (bucket(8, id)) () + properties ("format-version" = "2") + """ + + long snapshotsBefore = (sql """select count(*) from required_select\$snapshots""")[0][0] as long + long filesBefore = (sql """select count(*) from required_select\$files""")[0][0] as long + long rowsBefore = (sql """select count(*) from required_select""")[0][0] as long + + // W07-S02: The metadata and row baselines distinguish an early rejection + // from a late failure that has already published a snapshot or data file. + test { + sql """ + insert into required_select + select id, required_text + from internal.${internalDb}.nullable_source + """ + exception "null" + } + assertEquals(snapshotsBefore, + (sql """select count(*) from required_select\$snapshots""")[0][0] as long) + assertEquals(filesBefore, + (sql """select count(*) from required_select\$files""")[0][0] as long) + assertEquals(rowsBefore, + (sql """select count(*) from required_select""")[0][0] as long) +} diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_required_null_values_negative.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_required_null_values_negative.groovy new file mode 100644 index 00000000000000..86c433dc634554 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_required_null_values_negative.groovy @@ -0,0 +1,81 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_write_required_null_values_negative", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + // This opt-in switch isolates a write that can publish an unreadable Iceberg data file. + String knownBugTestEnabled = context.config.otherConfigs.get("enableIcebergKnownBugTest") + if (knownBugTestEnabled == null || !knownBugTestEnabled.equalsIgnoreCase("true")) { + logger.info("skip isolated Iceberg known-bug regression") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_write_required_null_values_negative" + String dbName = "iceberg_write_required_null_values_negative_db" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1" + ) + """ + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + + sql """ + create table required_values ( + id int not null, + required_text string not null + ) + partition by list (bucket(8, id)) () + properties ("format-version" = "2") + """ + + long snapshotsBefore = (sql """select count(*) from required_values\$snapshots""")[0][0] as long + long filesBefore = (sql """select count(*) from required_values\$files""")[0][0] as long + long rowsBefore = (sql """select count(*) from required_values""")[0][0] as long + + // W07-S01: A thrown NULL error is insufficient proof of atomicity; all + // visible and metadata state must remain at the pre-statement baseline. + test { + sql """insert into required_values values (1, null)""" + exception "null" + } + assertEquals(snapshotsBefore, + (sql """select count(*) from required_values\$snapshots""")[0][0] as long) + assertEquals(filesBefore, + (sql """select count(*) from required_values\$files""")[0][0] as long) + assertEquals(rowsBefore, + (sql """select count(*) from required_values""")[0][0] as long) +} diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_source_models.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_source_models.groovy new file mode 100644 index 00000000000000..c98e270ddd3913 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_source_models.groovy @@ -0,0 +1,220 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_write_source_models", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_write_source_models" + String dbName = "iceberg_write_source_models_db" + String internalDb = "iceberg_write_source_models_internal_db" + + sql """drop database if exists internal.${internalDb} force""" + sql """create database internal.${internalDb}""" + + // W04-S01: Duplicate model, no source partition, RANDOM distribution with three buckets. + sql """drop table if exists internal.${internalDb}.source_duplicate""" + sql """ + create table internal.${internalDb}.source_duplicate ( + id int, + category varchar(20), + amount bigint + ) + duplicate key(id) + distributed by random buckets 3 + properties ("replication_num" = "1") + """ + sql """ + insert into internal.${internalDb}.source_duplicate values + (1, 'A', 10), + (1, 'A', 11), + (2, null, 20) + """ + + // W04-S02: Unique MOW model, LIST source partition and HASH AUTO buckets. + sql """drop table if exists internal.${internalDb}.source_unique_mow""" + sql """ + create table internal.${internalDb}.source_unique_mow ( + id int, + category varchar(20), + amount bigint + ) + unique key(id, category) + partition by list(category) ( + partition p_ab values in ('A', 'B'), + partition p_null values in (null) + ) + distributed by hash(id) buckets auto + properties ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "true" + ) + """ + sql """insert into internal.${internalDb}.source_unique_mow values (10, 'A', 100), (11, null, 110)""" + sql """insert into internal.${internalDb}.source_unique_mow values (10, 'A', 101)""" + + // W04-S03: Unique MOR model, RANGE source partition and fixed HASH buckets. + sql """drop table if exists internal.${internalDb}.source_unique_mor""" + sql """ + create table internal.${internalDb}.source_unique_mor ( + id int, + category varchar(20), + amount bigint + ) + unique key(id) + partition by range(id) ( + partition p_lt_20 values less than (20), + partition p_max values less than maxvalue + ) + distributed by hash(id) buckets 2 + properties ( + "replication_num" = "1", + "enable_unique_key_merge_on_write" = "false" + ) + """ + sql """insert into internal.${internalDb}.source_unique_mor values (20, 'C', 200), (21, 'D', 210)""" + sql """insert into internal.${internalDb}.source_unique_mor values (20, 'C', 201)""" + + // W04-S04: Aggregate model, RANGE source partition and four fixed HASH buckets. + sql """drop table if exists internal.${internalDb}.source_aggregate""" + sql """ + create table internal.${internalDb}.source_aggregate ( + id int, + category varchar(20), + amount bigint sum + ) + aggregate key(id, category) + partition by range(id) ( + partition p_lt_40 values less than (40), + partition p_max values less than maxvalue + ) + distributed by hash(id, category) buckets 4 + properties ("replication_num" = "1") + """ + sql """ + insert into internal.${internalDb}.source_aggregate values + (30, 'E', 300), + (30, 'E', 3), + (31, 'F', 310) + """ + + order_qt_internal_model_oracle """ + select 'duplicate', id, category, amount + from internal.${internalDb}.source_duplicate + union all + select 'unique_mow', id, category, amount + from internal.${internalDb}.source_unique_mow + union all + select 'unique_mor', id, category, amount + from internal.${internalDb}.source_unique_mor + union all + select 'aggregate', id, category, amount + from internal.${internalDb}.source_aggregate + order by 1, 2, 3, 4 + """ + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1", + "meta.cache.iceberg.table.ttl-second" = "0", + "meta.cache.iceberg.schema.ttl-second" = "0" + ) + """ + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + sql """set enable_fallback_to_original_planner = false""" + + sql """drop table if exists source_model_sink""" + sql """ + create table source_model_sink ( + source_model string not null, + id int, + category string, + amount bigint + ) + partition by list (source_model, bucket(4, category)) () + properties ( + "format-version" = "2", + "write.format.default" = "parquet" + ) + """ + + // W04-S05: Independent INSERT SELECT statements keep each source model's read semantics. + // Multiple source buckets exercise distributed sink writers on more than one BE. + sql """ + insert into source_model_sink + select 'duplicate', id, category, amount + from internal.${internalDb}.source_duplicate + """ + sql """ + insert into source_model_sink + select 'unique_mow', id, category, amount + from internal.${internalDb}.source_unique_mow + """ + sql """ + insert into source_model_sink + select 'unique_mor', id, category, amount + from internal.${internalDb}.source_unique_mor + """ + sql """ + insert into source_model_sink + select 'aggregate', id, category, amount + from internal.${internalDb}.source_aggregate + """ + + order_qt_source_model_sink """ + select source_model, id, category, amount + from source_model_sink + order by source_model, id, category, amount + """ + order_qt_source_model_partition_stats """ + select spec_id, sum(record_count) + from source_model_sink\$partitions + group by spec_id + order by spec_id + """ + + sql """refresh table ${dbName}.source_model_sink""" + spark_iceberg """refresh table demo.${dbName}.source_model_sink""" + def sparkRows = spark_iceberg """ + select source_model, id, category, amount + from demo.${dbName}.source_model_sink + order by source_model, id, category, amount + """ + def dorisRows = sql """ + select source_model, id, category, amount + from source_model_sink + order by source_model, id, category, amount + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) +} diff --git a/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_string_transform_metadata.groovy b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_string_transform_metadata.groovy new file mode 100644 index 00000000000000..ea3b8995ec0017 --- /dev/null +++ b/regression-test/suites/external_table_p0/iceberg/write/test_iceberg_write_string_transform_metadata.groovy @@ -0,0 +1,212 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_iceberg_write_string_transform_metadata", + "p0,external,iceberg,external_docker,external_docker_iceberg") { + String enabled = context.config.otherConfigs.get("enableIcebergTest") + if (enabled == null || !enabled.equalsIgnoreCase("true")) { + logger.info("disable iceberg test") + return + } + + String restPort = context.config.otherConfigs.get("iceberg_rest_uri_port") + String minioPort = context.config.otherConfigs.get("iceberg_minio_port") + String externalEnvIp = context.config.otherConfigs.get("externalEnvIp") + String catalogName = "test_iceberg_write_string_transform_metadata" + String dbName = "iceberg_write_string_transform_metadata_db" + + sql """drop catalog if exists ${catalogName}""" + sql """ + create catalog ${catalogName} properties ( + "type" = "iceberg", + "iceberg.catalog.type" = "rest", + "uri" = "http://${externalEnvIp}:${restPort}", + "s3.access_key" = "admin", + "s3.secret_key" = "password", + "s3.endpoint" = "http://${externalEnvIp}:${minioPort}", + "s3.region" = "us-east-1", + "meta.cache.iceberg.table.ttl-second" = "0", + "meta.cache.iceberg.schema.ttl-second" = "0" + ) + """ + sql """switch ${catalogName}""" + sql """drop database if exists ${dbName} force""" + sql """create database ${dbName}""" + sql """use ${dbName}""" + + sql """drop table if exists string_transform_metadata""" + sql """ + create table string_transform_metadata ( + id int not null, + p_identity string, + p_bucket string, + p_truncate string not null, + payload string + ) + partition by list (p_identity) () + properties ( + "format-version" = "2", + "write.format.default" = "parquet" + ) + """ + sql """ + alter table string_transform_metadata + add partition key bucket(8, p_bucket) as p_bucket_8 + """ + sql """ + alter table string_transform_metadata + add partition key truncate(2, p_truncate) as p_truncate_2 + """ + + // WS01-S01: Validate the physical partition values, not only logical row + // equality. STRING bucket accepts NULL and truncate preserves valid UTF-8. + sql """ + insert into string_transform_metadata values + (1, 'ascii', 'bucket-a', 'alphabet', 'ascii'), + (2, '中文', '桶-中文', '中文甲', 'cjk'), + (3, 'emoji', '😀-bucket', '😀甲乙', 'emoji'), + (4, concat('e', unhex('CC81')), 'combining-bucket', + concat('e', unhex('CC81'), 'x'), 'combining'), + (5, '', '', '', 'empty'), + (6, null, null, 'null-bucket', 'nullable-bucket') + """ + order_qt_string_transform_rows """ + select id, hex(p_identity), hex(p_bucket), hex(p_truncate), payload + from string_transform_metadata + order by id + """ + order_qt_string_transform_physical_partitions """ + select struct_element(`partition`, 'p_identity') as p_identity_partition, + struct_element(`partition`, 'p_bucket_8') as p_bucket_partition, + hex(struct_element(`partition`, 'p_truncate_2')) as p_truncate_partition, + record_count + from string_transform_metadata\$partitions + order by p_identity_partition, p_bucket_partition, p_truncate_partition + """ + + spark_iceberg """refresh table demo.${dbName}.string_transform_metadata""" + def sparkRows = spark_iceberg """ + select id, p_identity, p_bucket, p_truncate, payload + from demo.${dbName}.string_transform_metadata + order by id + """ + def dorisRows = sql """ + select id, p_identity, p_bucket, p_truncate, payload + from string_transform_metadata + order by id + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + + def sparkPartitions = spark_iceberg """ + select partition.p_identity, + partition.p_bucket_8, + hex(partition.p_truncate_2), + record_count + from demo.${dbName}.string_transform_metadata.partitions + order by partition.p_identity, partition.p_bucket_8, hex(partition.p_truncate_2) + """ + def dorisPartitions = sql """ + select struct_element(`partition`, 'p_identity'), + struct_element(`partition`, 'p_bucket_8'), + hex(struct_element(`partition`, 'p_truncate_2')), + record_count + from string_transform_metadata\$partitions + order by struct_element(`partition`, 'p_identity'), + struct_element(`partition`, 'p_bucket_8'), + hex(struct_element(`partition`, 'p_truncate_2')) + """ + assertSparkDorisResultEquals(sparkPartitions, dorisPartitions) + + // WS01-S02: Evolve the bucket and truncate widths and verify that both + // physical specs remain readable by Doris and Spark. + sql """ + alter table string_transform_metadata + replace partition key p_bucket_8 with bucket(16, p_bucket) as p_bucket_16 + """ + sql """ + alter table string_transform_metadata + replace partition key p_truncate_2 with truncate(3, p_truncate) as p_truncate_3 + """ + sql """ + insert into string_transform_metadata values + (7, 'new', 'bucket-new', '中文甲乙', 'new-cjk'), + (8, null, null, '😀甲乙丙', 'new-null-bucket') + """ + order_qt_string_transform_evolved_specs """ + select spec_id, count(*), sum(record_count) + from string_transform_metadata\$partitions + group by spec_id + order by spec_id + """ + // Old and new transform fields must coexist in the union partition struct; + // spec counts alone cannot distinguish bucket(8) from bucket(16), or + // truncate(2) from truncate(3). + order_qt_string_transform_evolved_physical_partitions """ + select spec_id, + hex(struct_element(`partition`, 'p_identity')), + struct_element(`partition`, 'p_bucket_8'), + hex(struct_element(`partition`, 'p_truncate_2')), + struct_element(`partition`, 'p_bucket_16'), + hex(struct_element(`partition`, 'p_truncate_3')), + record_count + from string_transform_metadata\$partitions + order by spec_id, 2, 3, 4, 5, 6 + """ + order_qt_string_transform_evolved_rows """ + select id, hex(p_identity), hex(p_bucket), hex(p_truncate), payload + from string_transform_metadata + order by id + """ + spark_iceberg """refresh table demo.${dbName}.string_transform_metadata""" + sparkRows = spark_iceberg """ + select id, p_identity, p_bucket, p_truncate, payload + from demo.${dbName}.string_transform_metadata + order by id + """ + dorisRows = sql """ + select id, p_identity, p_bucket, p_truncate, payload + from string_transform_metadata + order by id + """ + assertSparkDorisResultEquals(sparkRows, dorisRows) + + // Compare the evolved physical metadata as well as logical rows, including + // both superseded and current transform fields. + sparkPartitions = spark_iceberg """ + select spec_id, + hex(partition.p_identity), + partition.p_bucket_8, + hex(partition.p_truncate_2), + partition.p_bucket_16, + hex(partition.p_truncate_3), + record_count + from demo.${dbName}.string_transform_metadata.partitions + order by spec_id, 2, 3, 4, 5, 6 + """ + dorisPartitions = sql """ + select spec_id, + hex(struct_element(`partition`, 'p_identity')), + struct_element(`partition`, 'p_bucket_8'), + hex(struct_element(`partition`, 'p_truncate_2')), + struct_element(`partition`, 'p_bucket_16'), + hex(struct_element(`partition`, 'p_truncate_3')), + record_count + from string_transform_metadata\$partitions + order by spec_id, 2, 3, 4, 5, 6 + """ + assertSparkDorisResultEquals(sparkPartitions, dorisPartitions) +}