From 931f2bcff846d54fc7aa062af84e2d698375d80d Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 8 Apr 2026 17:37:36 -0400 Subject: [PATCH 01/12] Start testing what will be DataFusion 54.0. --- native/Cargo.lock | 221 +++++++++--------- native/Cargo.toml | 8 +- native/core/Cargo.toml | 2 +- native/core/src/execution/operators/expand.rs | 18 +- .../src/execution/operators/iceberg_scan.rs | 9 +- .../src/execution/operators/parquet_writer.rs | 17 +- native/core/src/execution/operators/scan.rs | 9 +- .../src/execution/operators/shuffle_scan.rs | 9 +- native/core/src/execution/planner.rs | 2 +- native/shuffle/src/shuffle_writer.rs | 16 +- native/spark-expr/src/agg_funcs/avg.rs | 9 +- .../spark-expr/src/agg_funcs/avg_decimal.rs | 7 +- .../spark-expr/src/agg_funcs/correlation.rs | 7 +- native/spark-expr/src/agg_funcs/covariance.rs | 6 - native/spark-expr/src/agg_funcs/stddev.rs | 7 +- .../spark-expr/src/agg_funcs/sum_decimal.rs | 6 +- native/spark-expr/src/agg_funcs/sum_int.rs | 6 +- native/spark-expr/src/agg_funcs/variance.rs | 6 - .../src/array_funcs/array_compact.rs | 5 - native/spark-expr/src/array_funcs/size.rs | 5 - .../src/bloom_filter/bloom_filter_agg.rs | 6 +- .../bloom_filter_might_contain.rs | 5 - native/spark-expr/src/comet_scalar_funcs.rs | 5 - .../src/datetime_funcs/date_diff.rs | 5 - .../src/datetime_funcs/date_trunc.rs | 5 - .../src/datetime_funcs/extract_date_part.rs | 6 +- .../src/datetime_funcs/make_date.rs | 5 - .../src/datetime_funcs/unix_timestamp.rs | 6 +- .../spark-expr/src/string_funcs/contains.rs | 5 - 29 files changed, 184 insertions(+), 239 deletions(-) diff --git a/native/Cargo.lock b/native/Cargo.lock index 0c0c9e97b7..3dbaed308b 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -23,7 +23,7 @@ version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d122413f284cf2d62fb1b7db97e02edb8cda96d769b16e443a4f6195e35662b0" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "generic-array", ] @@ -184,7 +184,7 @@ checksum = "36fa98bc79671c7981272d91a8753a928ff6a1cd8e4f20a44c45bd5d313840bf" dependencies = [ "bigdecimal", "bon", - "digest", + "digest 0.10.7", "log", "miniz_oxide", "num-bigint", @@ -646,7 +646,7 @@ dependencies = [ "fastrand", "hex", "http 1.4.0", - "sha1", + "sha1 0.10.6", "time", "tokio", "tracing", @@ -1108,7 +1108,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46502ad458c9a52b69d4d4d32775c788b7a1b85e8bc9d482d92250fc0e3f8efe" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -1134,6 +1134,15 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-buffer" +version = "0.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cdd35008169921d80bc60d3d0ab416eecb028c4cd653352907921d95084790be" +dependencies = [ + "hybrid-array", +] + [[package]] name = "block-padding" version = "0.3.3" @@ -1377,7 +1386,7 @@ version = "0.4.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "inout", ] @@ -1503,6 +1512,12 @@ version = "0.9.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" +[[package]] +name = "const-oid" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6ef517f0926dd24a1582492c791b6a4818a4d94e789a334894aa15b0d12f55c" + [[package]] name = "const-random" version = "0.1.18" @@ -1676,6 +1691,15 @@ dependencies = [ "typenum", ] +[[package]] +name = "crypto-common" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77727bb15fa921304124b128af125e7e3b968275d1b108b379190264f4423710" +dependencies = [ + "hybrid-array", +] + [[package]] name = "csv" version = "1.4.0" @@ -1808,13 +1832,11 @@ dependencies = [ [[package]] name = "datafusion" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de9f8117889ba9503440f1dd79ebab32ba52ccf1720bb83cd718a29d4edc0d16" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "arrow", "arrow-schema", "async-trait", - "bytes", "chrono", "datafusion-catalog", "datafusion-catalog-listing", @@ -1847,8 +1869,6 @@ dependencies = [ "object_store", "parking_lot", "parquet", - "rand 0.9.2", - "regex", "sqlparser", "tempfile", "tokio", @@ -1859,8 +1879,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "be893b73a13671f310ffcc8da2c546b81efcc54c22e0382c0a28aa3537017137" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "arrow", "async-trait", @@ -1884,8 +1903,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "830487b51ed83807d6b32d6325f349c3144ae0c9bf772cf2a712db180c31d5e6" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "arrow", "async-trait", @@ -2078,13 +2096,13 @@ dependencies = [ [[package]] name = "datafusion-common" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0d7663f3af955292f8004e74bcaf8f7ea3d66cc38438749615bb84815b61a293" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ - "ahash", "arrow", "arrow-ipc", + "arrow-schema", "chrono", + "foldhash 0.2.0", "half", "hashbrown 0.16.1", "hex", @@ -2094,17 +2112,16 @@ dependencies = [ "log", "object_store", "parquet", - "paste", "sqlparser", "tokio", + "uuid", "web-time", ] [[package]] name = "datafusion-common-runtime" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f590205c7e32fe1fea48dd53ffb406e56ae0e7a062213a3ac848db8771641bd" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "futures", "log", @@ -2114,8 +2131,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fde1e030a9dc87b743c806fbd631f5ecfa2ccaa4ffb61fa19144a07fea406b79" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "arrow", "async-compression", @@ -2149,8 +2165,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "331ebae7055dc108f9b54994b93dff91f3a17445539efe5b74e89264f7b36e15" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "arrow", "arrow-ipc", @@ -2173,8 +2188,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9e0d475088325e2986876aa27bb30d0574f72a22955a527d202f454681d55c5c" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "arrow", "async-trait", @@ -2196,8 +2210,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ea1520d81f31770f3ad6ee98b391e75e87a68a5bb90de70064ace5e0a7182fe8" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "arrow", "async-trait", @@ -2212,7 +2225,6 @@ dependencies = [ "datafusion-session", "futures", "object_store", - "serde_json", "tokio", "tokio-stream", ] @@ -2220,8 +2232,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "95be805d0742ab129720f4c51ad9242cd872599cdb076098b03f061fcdc7f946" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "arrow", "async-trait", @@ -2231,6 +2242,7 @@ dependencies = [ "datafusion-datasource", "datafusion-execution", "datafusion-expr", + "datafusion-functions", "datafusion-functions-aggregate-common", "datafusion-physical-expr", "datafusion-physical-expr-adapter", @@ -2250,19 +2262,16 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c93ad9e37730d2c7196e68616f3f2dd3b04c892e03acd3a8eeca6e177f3c06a" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" [[package]] name = "datafusion-execution" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9437d3cd5d363f9319f8122182d4d233427de79c7eb748f23054c9aaa0fdd8df" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "arrow", "arrow-buffer", "async-trait", - "chrono", "dashmap", "datafusion-common", "datafusion-expr", @@ -2280,10 +2289,10 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "67164333342b86521d6d93fa54081ee39839894fb10f7a700c099af96d7552cf" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "arrow", + "arrow-schema", "async-trait", "chrono", "datafusion-common", @@ -2294,7 +2303,6 @@ dependencies = [ "datafusion-physical-expr-common", "indexmap 2.13.1", "itertools 0.14.0", - "paste", "serde_json", "sqlparser", ] @@ -2302,21 +2310,18 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab05fdd00e05d5a6ee362882546d29d6d3df43a6c55355164a7fbee12d163bc9" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "arrow", "datafusion-common", "indexmap 2.13.1", "itertools 0.14.0", - "paste", ] [[package]] name = "datafusion-functions" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "04fb863482d987cf938db2079e07ab0d3bb64595f28907a6c2f8671ad71cca7e" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "arrow", "arrow-buffer", @@ -2347,10 +2352,8 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "829856f4e14275fb376c104f27cbf3c3b57a9cfe24885d98677525f5e43ce8d6" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-doc", @@ -2360,19 +2363,17 @@ dependencies = [ "datafusion-macros", "datafusion-physical-expr", "datafusion-physical-expr-common", + "foldhash 0.2.0", "half", "log", "num-traits", - "paste", ] [[package]] name = "datafusion-functions-aggregate-common" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "08af79cc3d2aa874a362fb97decfcbd73d687190cb096f16a6c85a7780cce311" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr-common", @@ -2382,8 +2383,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "465ae3368146d49c2eda3e2c0ef114424c87e8a6b509ab34c1026ace6497e790" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "arrow", "arrow-ord", @@ -2401,14 +2401,13 @@ dependencies = [ "itertools 0.14.0", "itoa", "log", - "paste", + "memchr", ] [[package]] name = "datafusion-functions-table" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6156e6b22fcf1784112fc0173f3ae6e78c8fdb4d3ed0eace9543873b437e2af6" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "arrow", "async-trait", @@ -2417,14 +2416,12 @@ dependencies = [ "datafusion-expr", "datafusion-physical-plan", "parking_lot", - "paste", ] [[package]] name = "datafusion-functions-window" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca7baec14f866729012efb89011a6973f3a346dc8090c567bfcd328deff551c1" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "arrow", "datafusion-common", @@ -2435,14 +2432,12 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "log", - "paste", ] [[package]] name = "datafusion-functions-window-common" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "159228c3280d342658466bb556dc24de30047fe1d7e559dc5d16ccc5324166f9" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2451,8 +2446,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5427e5da5edca4d21ea1c7f50e1c9421775fe33d7d5726e5641a833566e7578" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "datafusion-doc", "quote", @@ -2462,8 +2456,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "89099eefcd5b223ec685c36a41d35c69239236310d71d339f2af0fa4383f3f46" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "arrow", "chrono", @@ -2481,10 +2474,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0f222df5195d605d79098ef37bdd5323bff0131c9d877a24da6ec98dfca9fe36" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ - "ahash", "arrow", "datafusion-common", "datafusion-expr", @@ -2496,7 +2487,6 @@ dependencies = [ "indexmap 2.13.1", "itertools 0.14.0", "parking_lot", - "paste", "petgraph", "tokio", ] @@ -2504,8 +2494,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40838625d63d9c12549d81979db3dd675d159055eb9135009ba272ab0e8d0f64" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "arrow", "datafusion-common", @@ -2519,10 +2508,8 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eacbcc4cfd502558184ed58fa3c72e775ec65bf077eef5fd2b3453db676f893c" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ - "ahash", "arrow", "chrono", "datafusion-common", @@ -2536,8 +2523,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d501d0e1d0910f015677121601ac177ec59272ef5c9324d1147b394988f40941" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "arrow", "datafusion-common", @@ -2554,10 +2540,8 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "463c88ad6f1ecab1810f4c9f046898bee035b370137eb79b2b2db925e270631d" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ - "ahash", "arrow", "arrow-ord", "arrow-schema", @@ -2586,8 +2570,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2857618a0ecbd8cd0cf29826889edd3a25774ec26b2995fc3862095c95d88fc6" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "arrow", "datafusion-common", @@ -2596,15 +2579,13 @@ dependencies = [ "datafusion-physical-expr", "datafusion-physical-expr-common", "datafusion-physical-plan", - "itertools 0.14.0", "log", ] [[package]] name = "datafusion-session" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ef8637e35022c5c775003b3ab1debc6b4a8f0eb41b069bdd5475dd3aa93f6eba" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "async-trait", "datafusion-common", @@ -2617,8 +2598,7 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "923a8b871962a9d860f036f743a20af50ff04729f1da2468ed220dab4f61c97d" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "arrow", "bigdecimal", @@ -2633,10 +2613,11 @@ dependencies = [ "datafusion-functions-aggregate", "datafusion-functions-nested", "log", + "num-traits", "percent-encoding", "rand 0.9.2", "serde_json", - "sha1", + "sha1 0.11.0", "sha2", "url", ] @@ -2644,8 +2625,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "53.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "12d9e9f16a1692a11c94bcc418191fa15fd2b4d72a0c1a0c607db93c0b84dd81" +source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" dependencies = [ "arrow", "bigdecimal", @@ -2674,7 +2654,7 @@ version = "0.7.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" dependencies = [ - "const-oid", + "const-oid 0.9.6", "pem-rfc7468", "zeroize", ] @@ -2754,12 +2734,23 @@ version = "0.10.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ed9a281f7bc9b7576e61468ba615a66a5c8cfdff42420a70aa82701a3b1e292" dependencies = [ - "block-buffer", - "const-oid", - "crypto-common", + "block-buffer 0.10.4", + "const-oid 0.9.6", + "crypto-common 0.1.7", "subtle", ] +[[package]] +name = "digest" +version = "0.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4850db49bf08e663084f7fb5c87d202ef91a3907271aff24a94eb97ff039153c" +dependencies = [ + "block-buffer 0.12.0", + "const-oid 0.10.2", + "crypto-common 0.2.1", +] + [[package]] name = "displaydoc" version = "0.2.5" @@ -3310,7 +3301,7 @@ version = "0.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" dependencies = [ - "digest", + "digest 0.10.7", ] [[package]] @@ -3389,6 +3380,15 @@ version = "2.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "135b12329e5e3ce057a9f972339ea52bc954fe1e9358ef27f95e89716fbc5424" +[[package]] +name = "hybrid-array" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3944cf8cf766b40e2a1a333ee5e9b563f854d5fa49d6a8ca2764e97c6eddb214" +dependencies = [ + "typenum", +] + [[package]] name = "hyper" version = "1.9.0" @@ -4191,7 +4191,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ "cfg-if", - "digest", + "digest 0.10.7", ] [[package]] @@ -4798,7 +4798,7 @@ version = "0.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ - "digest", + "digest 0.10.7", "hmac", ] @@ -5467,7 +5467,7 @@ dependencies = [ "rust-ini", "serde", "serde_json", - "sha1", + "sha1 0.10.6", "sha2", "tokio", ] @@ -5489,7 +5489,7 @@ dependencies = [ "jiff", "log", "percent-encoding", - "sha1", + "sha1 0.10.6", "sha2", "windows-sys 0.61.2", ] @@ -5576,8 +5576,8 @@ version = "0.9.10" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b8573f03f5883dcaebdfcf4725caa1ecb9c15b2ef50c43a07b816e06799bb12d" dependencies = [ - "const-oid", - "digest", + "const-oid 0.9.6", + "digest 0.10.7", "num-bigint-dig", "num-integer", "num-traits", @@ -5965,7 +5965,18 @@ checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", +] + +[[package]] +name = "sha1" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "aacc4cc499359472b4abe1bf11d0b12e688af9a805fa5e3016f9a386dc2d0214" +dependencies = [ + "cfg-if", + "cpufeatures 0.3.0", + "digest 0.11.2", ] [[package]] @@ -5976,7 +5987,7 @@ checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ "cfg-if", "cpufeatures 0.2.17", - "digest", + "digest 0.10.7", ] [[package]] @@ -6001,7 +6012,7 @@ version = "2.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" dependencies = [ - "digest", + "digest 0.10.7", "rand_core 0.6.4", ] @@ -6664,7 +6675,7 @@ version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fc1de2c688dc15305988b563c3854064043356019f97a4b46276fe734c4f07ea" dependencies = [ - "crypto-common", + "crypto-common 0.1.7", "subtle", ] diff --git a/native/Cargo.toml b/native/Cargo.toml index b71bc0c73c..6e66056b55 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -38,10 +38,10 @@ arrow = { version = "58.1.0", features = ["prettyprint", "ffi", "chrono-tz"] } async-trait = { version = "0.1" } bytes = { version = "1.11.1" } parquet = { version = "58.1.0", default-features = false, features = ["experimental"] } -datafusion = { version = "53.0.0", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } -datafusion-datasource = { version = "53.0.0" } -datafusion-physical-expr-adapter = { version = "53.0.0" } -datafusion-spark = { version = "53.0.0", features = ["core"] } +datafusion = { git = "https://github.com/mbutrovich/datafusion.git", branch = "dyncomparator", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } +datafusion-datasource = { git = "https://github.com/mbutrovich/datafusion.git", branch = "dyncomparator" } +datafusion-physical-expr-adapter = { git = "https://github.com/mbutrovich/datafusion.git", branch = "dyncomparator" } +datafusion-spark = { git = "https://github.com/mbutrovich/datafusion.git", branch = "dyncomparator", features = ["core"] } datafusion-comet-spark-expr = { path = "spark-expr" } datafusion-comet-common = { path = "common" } datafusion-comet-jni-bridge = { path = "jni-bridge" } diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml index 872260c02f..c922493d20 100644 --- a/native/core/Cargo.toml +++ b/native/core/Cargo.toml @@ -91,7 +91,7 @@ jni = { version = "0.22.4", features = ["invocation"] } lazy_static = "1.4" assertables = "9" hex = "0.4.3" -datafusion-functions-nested = { version = "53.0.0" } +datafusion-functions-nested = { git = "https://github.com/mbutrovich/datafusion.git", branch = "dyncomparator" } [features] backtrace = ["datafusion/backtrace"] diff --git a/native/core/src/execution/operators/expand.rs b/native/core/src/execution/operators/expand.rs index e06fab23ec..c2c9c671d5 100644 --- a/native/core/src/execution/operators/expand.rs +++ b/native/core/src/execution/operators/expand.rs @@ -17,6 +17,7 @@ use arrow::array::{RecordBatch, RecordBatchOptions}; use arrow::datatypes::SchemaRef; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::DataFusionError; use datafusion::physical_expr::{EquivalenceProperties, PhysicalExpr}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; @@ -29,7 +30,6 @@ use datafusion::{ }; use futures::{Stream, StreamExt}; use std::{ - any::Any, pin::Pin, sync::Arc, task::{Context, Poll}, @@ -91,8 +91,20 @@ impl DisplayAs for ExpandExec { } impl ExecutionPlan for ExpandExec { - fn as_any(&self) -> &dyn Any { - self + fn apply_expressions( + &self, + f: &mut dyn FnMut(&dyn PhysicalExpr) -> datafusion::common::Result, + ) -> datafusion::common::Result { + for projection in &self.projections { + for expr in projection { + match f(expr.as_ref())? { + TreeNodeRecursion::Continue => {} + TreeNodeRecursion::Jump => {} + TreeNodeRecursion::Stop => return Ok(TreeNodeRecursion::Stop), + } + } + } + Ok(TreeNodeRecursion::Continue) } fn schema(&self) -> SchemaRef { diff --git a/native/core/src/execution/operators/iceberg_scan.rs b/native/core/src/execution/operators/iceberg_scan.rs index d217ebc34b..a2dae49742 100644 --- a/native/core/src/execution/operators/iceberg_scan.rs +++ b/native/core/src/execution/operators/iceberg_scan.rs @@ -17,7 +17,6 @@ //! Native Iceberg table scan operator using iceberg-rust -use std::any::Any; use std::collections::HashMap; use std::fmt; use std::pin::Pin; @@ -26,6 +25,7 @@ use std::task::{Context, Poll}; use arrow::array::{ArrayRef, RecordBatch, RecordBatchOptions}; use arrow::datatypes::SchemaRef; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{DataFusionError, Result as DFResult}; use datafusion::execution::{RecordBatchStream, SendableRecordBatchStream, TaskContext}; use datafusion::physical_expr::expressions::Column; @@ -108,8 +108,11 @@ impl ExecutionPlan for IcebergScanExec { "IcebergScanExec" } - fn as_any(&self) -> &dyn Any { - self + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&dyn PhysicalExpr) -> DFResult, + ) -> DFResult { + Ok(TreeNodeRecursion::Continue) } fn schema(&self) -> SchemaRef { diff --git a/native/core/src/execution/operators/parquet_writer.rs b/native/core/src/execution/operators/parquet_writer.rs index 8ba79098d4..f66bc00186 100644 --- a/native/core/src/execution/operators/parquet_writer.rs +++ b/native/core/src/execution/operators/parquet_writer.rs @@ -18,7 +18,6 @@ //! Parquet writer operator for writing RecordBatches to Parquet files use std::{ - any::Any, collections::HashMap, fmt, fmt::{Debug, Formatter}, @@ -38,6 +37,7 @@ use crate::parquet::parquet_support::{create_hdfs_operator, prepare_object_store use arrow::datatypes::{Schema, SchemaRef}; use arrow::record_batch::RecordBatch; use async_trait::async_trait; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::{ error::{DataFusionError, Result}, execution::context::TaskContext, @@ -46,8 +46,8 @@ use datafusion::{ execution_plan::{Boundedness, EmissionType}, metrics::{ExecutionPlanMetricsSet, MetricsSet}, stream::RecordBatchStreamAdapter, - DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PlanProperties, - SendableRecordBatchStream, + DisplayAs, DisplayFormatType, ExecutionPlan, ExecutionPlanProperties, PhysicalExpr, + PlanProperties, SendableRecordBatchStream, }, }; use futures::TryStreamExt; @@ -404,14 +404,17 @@ impl DisplayAs for ParquetWriterExec { #[async_trait] impl ExecutionPlan for ParquetWriterExec { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { "ParquetWriterExec" } + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result, + ) -> Result { + Ok(TreeNodeRecursion::Continue) + } + fn metrics(&self) -> Option { Some(self.metrics.clone_inner()) } diff --git a/native/core/src/execution/operators/scan.rs b/native/core/src/execution/operators/scan.rs index 90bb741b5e..9ac9e4586a 100644 --- a/native/core/src/execution/operators/scan.rs +++ b/native/core/src/execution/operators/scan.rs @@ -28,6 +28,7 @@ use arrow::compute::{cast_with_options, take, CastOptions}; use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; use arrow::ffi::FFI_ArrowArray; use arrow::ffi::FFI_ArrowSchema; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{arrow_datafusion_err, DataFusionError, Result as DataFusionResult}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::metrics::{ @@ -43,7 +44,6 @@ use itertools::Itertools; use jni::objects::{Global, JObject, JValue}; use std::rc::Rc; use std::{ - any::Any, pin::Pin, sync::{Arc, Mutex}, task::{Context, Poll}, @@ -383,8 +383,11 @@ fn schema_from_data_types(data_types: &[DataType]) -> SchemaRef { } impl ExecutionPlan for ScanExec { - fn as_any(&self) -> &dyn Any { - self + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&dyn PhysicalExpr) -> DataFusionResult, + ) -> DataFusionResult { + Ok(TreeNodeRecursion::Continue) } fn schema(&self) -> SchemaRef { diff --git a/native/core/src/execution/operators/shuffle_scan.rs b/native/core/src/execution/operators/shuffle_scan.rs index 92c4dc8780..18ae134484 100644 --- a/native/core/src/execution/operators/shuffle_scan.rs +++ b/native/core/src/execution/operators/shuffle_scan.rs @@ -24,6 +24,7 @@ use crate::{ }; use arrow::array::ArrayRef; use arrow::datatypes::{DataType, Field, Schema, SchemaRef}; +use datafusion::common::tree_node::TreeNodeRecursion; use datafusion::common::{arrow_datafusion_err, Result as DataFusionResult}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::metrics::{ @@ -37,7 +38,6 @@ use datafusion::{ use futures::Stream; use jni::objects::{Global, JByteBuffer, JObject}; use std::{ - any::Any, pin::Pin, sync::{Arc, Mutex}, task::{Context, Poll}, @@ -221,8 +221,11 @@ fn schema_from_data_types(data_types: &[DataType]) -> SchemaRef { } impl ExecutionPlan for ShuffleScanExec { - fn as_any(&self) -> &dyn Any { - self + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&dyn PhysicalExpr) -> DataFusionResult, + ) -> DataFusionResult { + Ok(TreeNodeRecursion::Continue) } fn schema(&self) -> SchemaRef { diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index ac35925ace..7275470db1 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -1704,7 +1704,7 @@ impl PhysicalPlanner { hash_join.as_ref().swap_inputs(PartitionMode::Partitioned)?; let mut additional_native_plans = vec![]; - if swapped_hash_join.as_any().is::() { + if swapped_hash_join.is::() { // a projection was added to the hash join additional_native_plans.push(Arc::clone(swapped_hash_join.children()[0])); } diff --git a/native/shuffle/src/shuffle_writer.rs b/native/shuffle/src/shuffle_writer.rs index 4ac4fc287b..3cc46704be 100644 --- a/native/shuffle/src/shuffle_writer.rs +++ b/native/shuffle/src/shuffle_writer.rs @@ -25,7 +25,8 @@ use crate::partitioners::{ use crate::{CometPartitioning, CompressionCodec}; use async_trait::async_trait; use datafusion::common::exec_datafusion_err; -use datafusion::physical_expr::{EquivalenceProperties, Partitioning}; +use datafusion::common::tree_node::TreeNodeRecursion; +use datafusion::physical_expr::{EquivalenceProperties, Partitioning, PhysicalExpr}; use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType}; use datafusion::physical_plan::EmptyRecordBatchStream; use datafusion::{ @@ -41,7 +42,6 @@ use datafusion::{ use datafusion_comet_common::tracing::with_trace_async; use futures::{StreamExt, TryFutureExt, TryStreamExt}; use std::{ - any::Any, fmt, fmt::{Debug, Formatter}, sync::Arc, @@ -120,15 +120,17 @@ impl DisplayAs for ShuffleWriterExec { #[async_trait] impl ExecutionPlan for ShuffleWriterExec { - /// Return a reference to Any that can be used for downcasting - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { "ShuffleWriterExec" } + fn apply_expressions( + &self, + _f: &mut dyn FnMut(&dyn PhysicalExpr) -> Result, + ) -> Result { + Ok(TreeNodeRecursion::Continue) + } + fn metrics(&self) -> Option { Some(self.metrics.clone_inner()) } diff --git a/native/spark-expr/src/agg_funcs/avg.rs b/native/spark-expr/src/agg_funcs/avg.rs index 3760b42504..24a9a30991 100644 --- a/native/spark-expr/src/agg_funcs/avg.rs +++ b/native/spark-expr/src/agg_funcs/avg.rs @@ -28,7 +28,7 @@ use datafusion::logical_expr::{ Accumulator, AggregateUDFImpl, EmitTo, GroupsAccumulator, ReversedUDAF, Signature, }; use datafusion::physical_expr::expressions::format_state_name; -use std::{any::Any, sync::Arc}; +use std::sync::Arc; use arrow::array::ArrowNativeTypeOp; use datafusion::logical_expr::function::{AccumulatorArgs, StateFieldsArgs}; @@ -67,11 +67,6 @@ impl Avg { } impl AggregateUDFImpl for Avg { - /// Return a reference to Any that can be used for downcasting - fn as_any(&self) -> &dyn Any { - self - } - fn accumulator(&self, _acc_args: AccumulatorArgs) -> Result> { // All numeric types use Float64 accumulation after casting match (&self.input_data_type, &self.result_data_type) { @@ -239,7 +234,7 @@ where impl GroupsAccumulator for AvgGroupsAccumulator where T: ArrowNumericType + Send, - F: Fn(T::Native, i64) -> Result + Send, + F: Fn(T::Native, i64) -> Result + Send + 'static, { fn update_batch( &mut self, diff --git a/native/spark-expr/src/agg_funcs/avg_decimal.rs b/native/spark-expr/src/agg_funcs/avg_decimal.rs index 9e8a31afa5..2722add556 100644 --- a/native/spark-expr/src/agg_funcs/avg_decimal.rs +++ b/native/spark-expr/src/agg_funcs/avg_decimal.rs @@ -28,7 +28,7 @@ use datafusion::logical_expr::{ Accumulator, AggregateUDFImpl, EmitTo, GroupsAccumulator, ReversedUDAF, Signature, }; use datafusion::physical_expr::expressions::format_state_name; -use std::{any::Any, sync::Arc}; +use std::sync::Arc; use crate::utils::{build_bool_state, is_valid_decimal_precision, unlikely}; use crate::{decimal_sum_overflow_error, EvalMode, SparkErrorWithContext}; @@ -108,11 +108,6 @@ impl AvgDecimal { } impl AggregateUDFImpl for AvgDecimal { - /// Return a reference to Any that can be used for downcasting - fn as_any(&self) -> &dyn Any { - self - } - fn accumulator(&self, _acc_args: AccumulatorArgs) -> Result> { match (&self.sum_data_type, &self.result_data_type) { (Decimal128(sum_precision, sum_scale), Decimal128(target_precision, target_scale)) => { diff --git a/native/spark-expr/src/agg_funcs/correlation.rs b/native/spark-expr/src/agg_funcs/correlation.rs index 9803855e35..aebf322620 100644 --- a/native/spark-expr/src/agg_funcs/correlation.rs +++ b/native/spark-expr/src/agg_funcs/correlation.rs @@ -17,7 +17,7 @@ use arrow::compute::{and, filter, is_not_null}; -use std::{any::Any, sync::Arc}; +use std::sync::Arc; use crate::agg_funcs::covariance::CovarianceAccumulator; use crate::agg_funcs::stddev::StddevAccumulator; @@ -58,11 +58,6 @@ impl Correlation { } impl AggregateUDFImpl for Correlation { - /// Return a reference to Any that can be used for downcasting - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { &self.name } diff --git a/native/spark-expr/src/agg_funcs/covariance.rs b/native/spark-expr/src/agg_funcs/covariance.rs index 15759eb155..a2256da525 100644 --- a/native/spark-expr/src/agg_funcs/covariance.rs +++ b/native/spark-expr/src/agg_funcs/covariance.rs @@ -29,7 +29,6 @@ use datafusion::logical_expr::type_coercion::aggregates::NUMERICS; use datafusion::logical_expr::{Accumulator, AggregateUDFImpl, Signature, Volatility}; use datafusion::physical_expr::expressions::format_state_name; use datafusion::physical_expr::expressions::StatsType; -use std::any::Any; use std::sync::Arc; /// COVAR_SAMP and COVAR_POP aggregate expression @@ -73,11 +72,6 @@ impl Covariance { } impl AggregateUDFImpl for Covariance { - /// Return a reference to Any that can be used for downcasting - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { &self.name } diff --git a/native/spark-expr/src/agg_funcs/stddev.rs b/native/spark-expr/src/agg_funcs/stddev.rs index b231b8afa7..2b09339dc1 100644 --- a/native/spark-expr/src/agg_funcs/stddev.rs +++ b/native/spark-expr/src/agg_funcs/stddev.rs @@ -15,7 +15,7 @@ // specific language governing permissions and limitations // under the License. -use std::{any::Any, sync::Arc}; +use std::sync::Arc; use crate::agg_funcs::variance::VarianceAccumulator; use arrow::datatypes::FieldRef; @@ -78,11 +78,6 @@ impl Stddev { } impl AggregateUDFImpl for Stddev { - /// Return a reference to Any that can be used for downcasting - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { &self.name } diff --git a/native/spark-expr/src/agg_funcs/sum_decimal.rs b/native/spark-expr/src/agg_funcs/sum_decimal.rs index 46db7f36b3..faa51252f8 100644 --- a/native/spark-expr/src/agg_funcs/sum_decimal.rs +++ b/native/spark-expr/src/agg_funcs/sum_decimal.rs @@ -27,7 +27,7 @@ use datafusion::logical_expr::Volatility::Immutable; use datafusion::logical_expr::{ Accumulator, AggregateUDFImpl, EmitTo, GroupsAccumulator, ReversedUDAF, Signature, }; -use std::{any::Any, sync::Arc}; +use std::sync::Arc; #[derive(Debug)] pub struct SumDecimal { @@ -99,10 +99,6 @@ impl SumDecimal { } impl AggregateUDFImpl for SumDecimal { - fn as_any(&self) -> &dyn Any { - self - } - fn accumulator(&self, _args: AccumulatorArgs) -> DFResult> { Ok(Box::new(SumDecimalAccumulator::new( self.precision, diff --git a/native/spark-expr/src/agg_funcs/sum_int.rs b/native/spark-expr/src/agg_funcs/sum_int.rs index 781528521b..7d1df1568b 100644 --- a/native/spark-expr/src/agg_funcs/sum_int.rs +++ b/native/spark-expr/src/agg_funcs/sum_int.rs @@ -29,7 +29,7 @@ use datafusion::logical_expr::Volatility::Immutable; use datafusion::logical_expr::{ Accumulator, AggregateUDFImpl, EmitTo, GroupsAccumulator, ReversedUDAF, Signature, }; -use std::{any::Any, sync::Arc}; +use std::sync::Arc; #[derive(Debug, PartialEq, Eq, Hash)] pub struct SumInteger { @@ -52,10 +52,6 @@ impl SumInteger { } impl AggregateUDFImpl for SumInteger { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { "sum" } diff --git a/native/spark-expr/src/agg_funcs/variance.rs b/native/spark-expr/src/agg_funcs/variance.rs index c97e664dd6..5f2c249da7 100644 --- a/native/spark-expr/src/agg_funcs/variance.rs +++ b/native/spark-expr/src/agg_funcs/variance.rs @@ -26,7 +26,6 @@ use datafusion::logical_expr::Volatility::Immutable; use datafusion::logical_expr::{Accumulator, AggregateUDFImpl, Signature}; use datafusion::physical_expr::expressions::format_state_name; use datafusion::physical_expr::expressions::StatsType; -use std::any::Any; use std::sync::Arc; /// VAR_SAMP and VAR_POP aggregate expression @@ -71,11 +70,6 @@ impl Variance { } impl AggregateUDFImpl for Variance { - /// Return a reference to Any that can be used for downcasting - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { &self.name } diff --git a/native/spark-expr/src/array_funcs/array_compact.rs b/native/spark-expr/src/array_funcs/array_compact.rs index 4653f966a5..d481242705 100644 --- a/native/spark-expr/src/array_funcs/array_compact.rs +++ b/native/spark-expr/src/array_funcs/array_compact.rs @@ -33,7 +33,6 @@ use datafusion::common::{exec_err, utils::take_function_args, Result}; use datafusion::logical_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignature, Volatility, }; -use std::any::Any; use std::sync::Arc; #[derive(Debug, PartialEq, Eq, Hash)] @@ -56,10 +55,6 @@ impl SparkArrayCompact { } impl ScalarUDFImpl for SparkArrayCompact { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { "spark_array_compact" } diff --git a/native/spark-expr/src/array_funcs/size.rs b/native/spark-expr/src/array_funcs/size.rs index 9777553341..f206b299d6 100644 --- a/native/spark-expr/src/array_funcs/size.rs +++ b/native/spark-expr/src/array_funcs/size.rs @@ -21,7 +21,6 @@ use datafusion::common::{exec_err, DataFusionError, Result as DataFusionResult, use datafusion::logical_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; -use std::any::Any; use std::sync::Arc; /// Spark size() function that returns the size of arrays or maps. @@ -73,10 +72,6 @@ impl SparkSizeFunc { } impl ScalarUDFImpl for SparkSizeFunc { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { "size" } diff --git a/native/spark-expr/src/bloom_filter/bloom_filter_agg.rs b/native/spark-expr/src/bloom_filter/bloom_filter_agg.rs index 3436b29201..3e9e5f00ad 100644 --- a/native/spark-expr/src/bloom_filter/bloom_filter_agg.rs +++ b/native/spark-expr/src/bloom_filter/bloom_filter_agg.rs @@ -17,7 +17,7 @@ use arrow::datatypes::{Field, FieldRef}; use datafusion::{arrow::datatypes::DataType, logical_expr::Volatility}; -use std::{any::Any, sync::Arc}; +use std::sync::Arc; use crate::bloom_filter::spark_bloom_filter; use crate::bloom_filter::spark_bloom_filter::SparkBloomFilter; @@ -75,10 +75,6 @@ impl BloomFilterAgg { } impl AggregateUDFImpl for BloomFilterAgg { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { "bloom_filter_agg" } diff --git a/native/spark-expr/src/bloom_filter/bloom_filter_might_contain.rs b/native/spark-expr/src/bloom_filter/bloom_filter_might_contain.rs index ea246dfb25..66168444d9 100644 --- a/native/spark-expr/src/bloom_filter/bloom_filter_might_contain.rs +++ b/native/spark-expr/src/bloom_filter/bloom_filter_might_contain.rs @@ -22,7 +22,6 @@ use datafusion::error::DataFusionError; use datafusion::logical_expr::{ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility}; use datafusion::physical_expr::PhysicalExpr; use datafusion::physical_plan::ColumnarValue; -use std::any::Any; use std::sync::Arc; use crate::bloom_filter::spark_bloom_filter::SparkBloomFilter; @@ -63,10 +62,6 @@ fn evaluate_bloom_filter( } impl ScalarUDFImpl for BloomFilterMightContain { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { "might_contain" } diff --git a/native/spark-expr/src/comet_scalar_funcs.rs b/native/spark-expr/src/comet_scalar_funcs.rs index 9c91bb69c9..ac59dafa76 100644 --- a/native/spark-expr/src/comet_scalar_funcs.rs +++ b/native/spark-expr/src/comet_scalar_funcs.rs @@ -34,7 +34,6 @@ use datafusion::logical_expr::{ Volatility, }; use datafusion::physical_plan::ColumnarValue; -use std::any::Any; use std::fmt::Debug; use std::sync::Arc; @@ -269,10 +268,6 @@ impl CometScalarFunction { } impl ScalarUDFImpl for CometScalarFunction { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { self.name.as_str() } diff --git a/native/spark-expr/src/datetime_funcs/date_diff.rs b/native/spark-expr/src/datetime_funcs/date_diff.rs index ca148c103a..be3c1d3552 100644 --- a/native/spark-expr/src/datetime_funcs/date_diff.rs +++ b/native/spark-expr/src/datetime_funcs/date_diff.rs @@ -22,7 +22,6 @@ use datafusion::common::{utils::take_function_args, DataFusionError, Result}; use datafusion::logical_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; -use std::any::Any; use std::sync::Arc; /// Spark-compatible date_diff function. @@ -52,10 +51,6 @@ impl Default for SparkDateDiff { } impl ScalarUDFImpl for SparkDateDiff { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { "date_diff" } diff --git a/native/spark-expr/src/datetime_funcs/date_trunc.rs b/native/spark-expr/src/datetime_funcs/date_trunc.rs index aeae18e36f..7ceb5234e1 100644 --- a/native/spark-expr/src/datetime_funcs/date_trunc.rs +++ b/native/spark-expr/src/datetime_funcs/date_trunc.rs @@ -22,7 +22,6 @@ use datafusion::common::{ use datafusion::logical_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; -use std::any::Any; use crate::kernels::temporal::{date_trunc_array_fmt_dyn, date_trunc_dyn}; @@ -51,10 +50,6 @@ impl Default for SparkDateTrunc { } impl ScalarUDFImpl for SparkDateTrunc { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { "date_trunc" } diff --git a/native/spark-expr/src/datetime_funcs/extract_date_part.rs b/native/spark-expr/src/datetime_funcs/extract_date_part.rs index acb7d2266e..7344a3953a 100644 --- a/native/spark-expr/src/datetime_funcs/extract_date_part.rs +++ b/native/spark-expr/src/datetime_funcs/extract_date_part.rs @@ -22,7 +22,7 @@ use datafusion::common::{internal_datafusion_err, DataFusionError}; use datafusion::logical_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; -use std::{any::Any, fmt::Debug}; +use std::fmt::Debug; macro_rules! extract_date_part { ($struct_name:ident, $fn_name:expr, $date_part_variant:ident) => { @@ -44,10 +44,6 @@ macro_rules! extract_date_part { } impl ScalarUDFImpl for $struct_name { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { $fn_name } diff --git a/native/spark-expr/src/datetime_funcs/make_date.rs b/native/spark-expr/src/datetime_funcs/make_date.rs index 58e4108580..ef29431703 100644 --- a/native/spark-expr/src/datetime_funcs/make_date.rs +++ b/native/spark-expr/src/datetime_funcs/make_date.rs @@ -23,7 +23,6 @@ use datafusion::common::{utils::take_function_args, DataFusionError, Result}; use datafusion::logical_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; -use std::any::Any; use std::sync::Arc; /// Spark-compatible make_date function. @@ -75,10 +74,6 @@ fn make_date(year: i32, month: i32, day: i32) -> Option { } impl ScalarUDFImpl for SparkMakeDate { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { "make_date" } diff --git a/native/spark-expr/src/datetime_funcs/unix_timestamp.rs b/native/spark-expr/src/datetime_funcs/unix_timestamp.rs index c4f1576293..4b7df90559 100644 --- a/native/spark-expr/src/datetime_funcs/unix_timestamp.rs +++ b/native/spark-expr/src/datetime_funcs/unix_timestamp.rs @@ -24,7 +24,7 @@ use datafusion::logical_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; use num::integer::div_floor; -use std::{any::Any, fmt::Debug, sync::Arc}; +use std::{fmt::Debug, sync::Arc}; const MICROS_PER_SECOND: i64 = 1_000_000; @@ -46,10 +46,6 @@ impl SparkUnixTimestamp { } impl ScalarUDFImpl for SparkUnixTimestamp { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { "unix_timestamp" } diff --git a/native/spark-expr/src/string_funcs/contains.rs b/native/spark-expr/src/string_funcs/contains.rs index bc34ce9cba..537227efdf 100644 --- a/native/spark-expr/src/string_funcs/contains.rs +++ b/native/spark-expr/src/string_funcs/contains.rs @@ -27,7 +27,6 @@ use datafusion::common::{exec_err, Result, ScalarValue}; use datafusion::logical_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; -use std::any::Any; use std::sync::Arc; /// Spark-optimized contains function. @@ -53,10 +52,6 @@ impl SparkContains { } impl ScalarUDFImpl for SparkContains { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { "contains" } From 8d922b5df0fa063fa17ed8c7c3697e8b6bb44164 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 8 Apr 2026 17:41:01 -0400 Subject: [PATCH 02/12] fix expandexec --- native/core/src/execution/operators/expand.rs | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/native/core/src/execution/operators/expand.rs b/native/core/src/execution/operators/expand.rs index c2c9c671d5..e2677d226d 100644 --- a/native/core/src/execution/operators/expand.rs +++ b/native/core/src/execution/operators/expand.rs @@ -95,16 +95,13 @@ impl ExecutionPlan for ExpandExec { &self, f: &mut dyn FnMut(&dyn PhysicalExpr) -> datafusion::common::Result, ) -> datafusion::common::Result { + let mut tnr = TreeNodeRecursion::Continue; for projection in &self.projections { for expr in projection { - match f(expr.as_ref())? { - TreeNodeRecursion::Continue => {} - TreeNodeRecursion::Jump => {} - TreeNodeRecursion::Stop => return Ok(TreeNodeRecursion::Stop), - } + tnr = tnr.visit_sibling(|| f(expr.as_ref()))?; } } - Ok(TreeNodeRecursion::Continue) + Ok(tnr) } fn schema(&self) -> SchemaRef { From b75c3211745da279dad9c96dd8a91040c4a52e5f Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 15 Apr 2026 15:10:09 -0400 Subject: [PATCH 03/12] Bump commits. --- native/Cargo.lock | 227 ++++++++++++++--------------------------- native/Cargo.toml | 8 +- native/core/Cargo.toml | 2 +- 3 files changed, 84 insertions(+), 153 deletions(-) diff --git a/native/Cargo.lock b/native/Cargo.lock index 3dbaed308b..40ce9aebb3 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -372,7 +372,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.13.1", + "indexmap 2.14.0", "itoa", "lexical-core", "memchr", @@ -1832,7 +1832,7 @@ dependencies = [ [[package]] name = "datafusion" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "arrow-schema", @@ -1879,7 +1879,7 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "async-trait", @@ -1903,7 +1903,7 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "async-trait", @@ -2096,7 +2096,7 @@ dependencies = [ [[package]] name = "datafusion-common" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "arrow-ipc", @@ -2104,9 +2104,9 @@ dependencies = [ "chrono", "foldhash 0.2.0", "half", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "hex", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "libc", "log", @@ -2121,7 +2121,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "futures", "log", @@ -2131,7 +2131,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "async-compression", @@ -2165,7 +2165,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "arrow-ipc", @@ -2188,7 +2188,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "async-trait", @@ -2210,7 +2210,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "async-trait", @@ -2232,7 +2232,7 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "async-trait", @@ -2262,12 +2262,12 @@ dependencies = [ [[package]] name = "datafusion-doc" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" [[package]] name = "datafusion-execution" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "arrow-buffer", @@ -2289,7 +2289,7 @@ dependencies = [ [[package]] name = "datafusion-expr" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "arrow-schema", @@ -2301,7 +2301,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-functions-window-common", "datafusion-physical-expr-common", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "serde_json", "sqlparser", @@ -2310,18 +2310,18 @@ dependencies = [ [[package]] name = "datafusion-expr-common" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "datafusion-common", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", ] [[package]] name = "datafusion-functions" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "arrow-buffer", @@ -2345,14 +2345,13 @@ dependencies = [ "rand 0.9.2", "regex", "sha2", - "unicode-segmentation", "uuid", ] [[package]] name = "datafusion-functions-aggregate" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "datafusion-common", @@ -2372,7 +2371,7 @@ dependencies = [ [[package]] name = "datafusion-functions-aggregate-common" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "datafusion-common", @@ -2383,7 +2382,7 @@ dependencies = [ [[package]] name = "datafusion-functions-nested" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "arrow-ord", @@ -2397,7 +2396,7 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-macros", "datafusion-physical-expr-common", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "itertools 0.14.0", "itoa", "log", @@ -2407,7 +2406,7 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "async-trait", @@ -2421,7 +2420,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "datafusion-common", @@ -2437,7 +2436,7 @@ dependencies = [ [[package]] name = "datafusion-functions-window-common" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "datafusion-common", "datafusion-physical-expr-common", @@ -2446,7 +2445,7 @@ dependencies = [ [[package]] name = "datafusion-macros" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "datafusion-doc", "quote", @@ -2456,7 +2455,7 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "chrono", @@ -2464,7 +2463,7 @@ dependencies = [ "datafusion-expr", "datafusion-expr-common", "datafusion-physical-expr", - "indexmap 2.13.1", + "indexmap 2.14.0", "itertools 0.14.0", "log", "regex", @@ -2474,7 +2473,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "datafusion-common", @@ -2483,8 +2482,8 @@ dependencies = [ "datafusion-functions-aggregate-common", "datafusion-physical-expr-common", "half", - "hashbrown 0.16.1", - "indexmap 2.13.1", + "hashbrown 0.17.0", + "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", "petgraph", @@ -2494,7 +2493,7 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "datafusion-common", @@ -2508,14 +2507,14 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-common" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "chrono", "datafusion-common", "datafusion-expr-common", - "hashbrown 0.16.1", - "indexmap 2.13.1", + "hashbrown 0.17.0", + "indexmap 2.14.0", "itertools 0.14.0", "parking_lot", ] @@ -2523,7 +2522,7 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "datafusion-common", @@ -2540,7 +2539,7 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "arrow-ord", @@ -2557,8 +2556,8 @@ dependencies = [ "datafusion-physical-expr-common", "futures", "half", - "hashbrown 0.16.1", - "indexmap 2.13.1", + "hashbrown 0.17.0", + "indexmap 2.14.0", "itertools 0.14.0", "log", "num-traits", @@ -2570,7 +2569,7 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "datafusion-common", @@ -2585,7 +2584,7 @@ dependencies = [ [[package]] name = "datafusion-session" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "async-trait", "datafusion-common", @@ -2598,7 +2597,7 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "bigdecimal", @@ -2625,7 +2624,7 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?branch=dyncomparator#e0d06eea91e4649d853305c0224350dd2503d5f1" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", "bigdecimal", @@ -2633,7 +2632,7 @@ dependencies = [ "datafusion-common", "datafusion-expr", "datafusion-functions-nested", - "indexmap 2.13.1", + "indexmap 2.14.0", "log", "regex", "sqlparser", @@ -2854,7 +2853,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3202,7 +3201,7 @@ dependencies = [ "futures-core", "futures-sink", "http 1.4.0", - "indexmap 2.13.1", + "indexmap 2.14.0", "slab", "tokio", "tokio-util", @@ -3247,6 +3246,12 @@ name = "hashbrown" version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" + +[[package]] +name = "hashbrown" +version = "0.17.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f467dd6dccf739c208452f8014c75c18bb8301b050ad1cfb27153803edb0f51" dependencies = [ "allocator-api2", "equivalent", @@ -3677,12 +3682,12 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.13.1" +version = "2.14.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "45a8a2b9cb3e0b0c1803dbb0758ffac5de2f425b23c28f518faabd9d805342ff" +checksum = "d466e9454f08e4a911e14806c24e16fba1b4c121d1ea474396f396069cf949d9" dependencies = [ "equivalent", - "hashbrown 0.16.1", + "hashbrown 0.17.0", "serde", "serde_core", ] @@ -3694,7 +3699,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "232929e1d75fe899576a3d5c7416ad0d88dbfbb3c3d6aa00873a7408a50ddb88" dependencies = [ "ahash", - "indexmap 2.13.1", + "indexmap 2.14.0", "is-terminal", "itoa", "log", @@ -3754,7 +3759,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -3810,7 +3815,7 @@ dependencies = [ "portable-atomic-util", "serde_core", "wasm-bindgen", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -4047,9 +4052,9 @@ checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.184" +version = "0.2.185" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "48f5d2a454e16a5ea0f4ced81bd44e4cfc7bd3a507b61887c99fd3538b28e4af" +checksum = "52ff2c0fe9bc6cb6b14a0592c2ff4fa9ceb83eea9db979b0487cd054946a2b8f" [[package]] name = "libloading" @@ -4750,7 +4755,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.13.1", + "indexmap 2.14.0", "simdutf8", "uuid", ] @@ -4765,7 +4770,7 @@ dependencies = [ "arrow-schema", "chrono", "half", - "indexmap 2.13.1", + "indexmap 2.14.0", "parquet-variant", "parquet-variant-json", "serde_json", @@ -4841,7 +4846,7 @@ checksum = "8701b58ea97060d5e5b155d383a69952a60943f0e6dfe30b04c287beb0b27455" dependencies = [ "fixedbitset", "hashbrown 0.15.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "serde", ] @@ -5125,7 +5130,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", - "itertools 0.14.0", + "itertools 0.13.0", "log", "multimap", "petgraph", @@ -5144,7 +5149,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.14.0", + "itertools 0.13.0", "proc-macro2", "quote", "syn 2.0.117", @@ -5256,7 +5261,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.60.2", + "windows-sys 0.59.0", ] [[package]] @@ -5651,7 +5656,7 @@ dependencies = [ "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -5923,7 +5928,7 @@ dependencies = [ "chrono", "hex", "indexmap 1.9.3", - "indexmap 2.13.1", + "indexmap 2.14.0", "schemars 0.9.0", "schemars 1.2.1", "serde_core", @@ -5950,7 +5955,7 @@ version = "0.9.34+deprecated" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6a8b1a1a2ebf674015cc02edccce75287f1a0130d394307b36743c2f5d504b47" dependencies = [ - "indexmap 2.13.1", + "indexmap 2.14.0", "itoa", "ryu", "serde", @@ -6256,7 +6261,7 @@ dependencies = [ "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -6875,7 +6880,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" dependencies = [ "anyhow", - "indexmap 2.13.1", + "indexmap 2.14.0", "wasm-encoder", "wasmparser", ] @@ -6901,7 +6906,7 @@ checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ "bitflags 2.11.0", "hashbrown 0.15.5", - "indexmap 2.13.1", + "indexmap 2.14.0", "semver", ] @@ -6968,7 +6973,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.61.2", + "windows-sys 0.59.0", ] [[package]] @@ -7063,15 +7068,6 @@ dependencies = [ "windows-targets 0.52.6", ] -[[package]] -name = "windows-sys" -version = "0.60.2" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" -dependencies = [ - "windows-targets 0.53.5", -] - [[package]] name = "windows-sys" version = "0.61.2" @@ -7105,30 +7101,13 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm 0.52.6", + "windows_i686_gnullvm", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] -[[package]] -name = "windows-targets" -version = "0.53.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" -dependencies = [ - "windows-link", - "windows_aarch64_gnullvm 0.53.1", - "windows_aarch64_msvc 0.53.1", - "windows_i686_gnu 0.53.1", - "windows_i686_gnullvm 0.53.1", - "windows_i686_msvc 0.53.1", - "windows_x86_64_gnu 0.53.1", - "windows_x86_64_gnullvm 0.53.1", - "windows_x86_64_msvc 0.53.1", -] - [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -7141,12 +7120,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" -[[package]] -name = "windows_aarch64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" - [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -7159,12 +7132,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" -[[package]] -name = "windows_aarch64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" - [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -7177,24 +7144,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" -[[package]] -name = "windows_i686_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" - [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" -[[package]] -name = "windows_i686_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" - [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -7207,12 +7162,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" -[[package]] -name = "windows_i686_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" - [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -7225,12 +7174,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" -[[package]] -name = "windows_x86_64_gnu" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" - [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -7243,12 +7186,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" -[[package]] -name = "windows_x86_64_gnullvm" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" - [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -7261,12 +7198,6 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" -[[package]] -name = "windows_x86_64_msvc" -version = "0.53.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" - [[package]] name = "wit-bindgen" version = "0.51.0" @@ -7295,7 +7226,7 @@ checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" dependencies = [ "anyhow", "heck", - "indexmap 2.13.1", + "indexmap 2.14.0", "prettyplease", "syn 2.0.117", "wasm-metadata", @@ -7326,7 +7257,7 @@ checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", "bitflags 2.11.0", - "indexmap 2.13.1", + "indexmap 2.14.0", "log", "serde", "serde_derive", @@ -7345,7 +7276,7 @@ checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" dependencies = [ "anyhow", "id-arena", - "indexmap 2.13.1", + "indexmap 2.14.0", "log", "semver", "serde", diff --git a/native/Cargo.toml b/native/Cargo.toml index 6e66056b55..1e1055bfd0 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -38,10 +38,10 @@ arrow = { version = "58.1.0", features = ["prettyprint", "ffi", "chrono-tz"] } async-trait = { version = "0.1" } bytes = { version = "1.11.1" } parquet = { version = "58.1.0", default-features = false, features = ["experimental"] } -datafusion = { git = "https://github.com/mbutrovich/datafusion.git", branch = "dyncomparator", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } -datafusion-datasource = { git = "https://github.com/mbutrovich/datafusion.git", branch = "dyncomparator" } -datafusion-physical-expr-adapter = { git = "https://github.com/mbutrovich/datafusion.git", branch = "dyncomparator" } -datafusion-spark = { git = "https://github.com/mbutrovich/datafusion.git", branch = "dyncomparator", features = ["core"] } +datafusion = { git = "https://github.com/mbutrovich/datafusion.git", rev = "0198302544", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } +datafusion-datasource = { git = "https://github.com/mbutrovich/datafusion.git", rev = "0198302544" } +datafusion-physical-expr-adapter = { git = "https://github.com/mbutrovich/datafusion.git", rev = "0198302544" } +datafusion-spark = { git = "https://github.com/mbutrovich/datafusion.git", rev = "0198302544", features = ["core"] } datafusion-comet-spark-expr = { path = "spark-expr" } datafusion-comet-common = { path = "common" } datafusion-comet-jni-bridge = { path = "jni-bridge" } diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml index c922493d20..2bfb720379 100644 --- a/native/core/Cargo.toml +++ b/native/core/Cargo.toml @@ -91,7 +91,7 @@ jni = { version = "0.22.4", features = ["invocation"] } lazy_static = "1.4" assertables = "9" hex = "0.4.3" -datafusion-functions-nested = { git = "https://github.com/mbutrovich/datafusion.git", branch = "dyncomparator" } +datafusion-functions-nested = { git = "https://github.com/mbutrovich/datafusion.git", rev = "0198302544" } [features] backtrace = ["datafusion/backtrace"] From af84b738e5f929690d341f5d10ca2e9bd2941918 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 15 Apr 2026 15:26:09 -0400 Subject: [PATCH 04/12] Fix. --- native/Cargo.lock | 2 +- native/core/src/parquet/schema_adapter.rs | 20 +++++++++++++------ .../src/datetime_funcs/date_from_unix_date.rs | 5 ----- native/spark-expr/src/datetime_funcs/hours.rs | 6 +----- 4 files changed, 16 insertions(+), 17 deletions(-) diff --git a/native/Cargo.lock b/native/Cargo.lock index d7a742df72..d92ee9261b 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -5888,7 +5888,7 @@ version = "1.0.149" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "83fc039473c5595ace860d8c4fafa220ff474b3fc6bfdb4293327f1a37e94d86" dependencies = [ - "indexmap 2.13.1", + "indexmap 2.14.0", "itoa", "memchr", "serde", diff --git a/native/core/src/parquet/schema_adapter.rs b/native/core/src/parquet/schema_adapter.rs index af79d9082d..593a0c063a 100644 --- a/native/core/src/parquet/schema_adapter.rs +++ b/native/core/src/parquet/schema_adapter.rs @@ -370,21 +370,29 @@ impl SparkPhysicalExprAdapter { .data() } - /// Replace CastColumnExpr (DataFusion's cast) with Spark's Cast expression. + /// Replace CastExpr (DataFusion's cast) with Spark's Cast expression. fn replace_with_spark_cast( &self, expr: Arc, ) -> DataFusionResult>> { - // Check for CastColumnExpr and replace with spark_expr::Cast - // CastColumnExpr is in datafusion_physical_expr::expressions + // Check for CastExpr and replace with spark_expr::Cast if let Some(cast) = expr .as_any() - .downcast_ref::() + .downcast_ref::() { let child = Arc::clone(cast.expr()); - let physical_type = cast.input_field().data_type(); let target_type = cast.target_field().data_type(); + // Derive input field from the child Column expression and the physical schema + let input_field = if let Some(col) = child.as_any().downcast_ref::() { + Arc::new(self.physical_file_schema.field(col.index()).clone()) + } else { + // Fallback: synthesize a field from the target field name and child data type + let child_type = cast.expr().data_type(&self.physical_file_schema)?; + Arc::new(Field::new(cast.target_field().name(), child_type, true)) + }; + let physical_type = input_field.data_type(); + // For complex nested types (Struct, List, Map), Timestamp timezone // mismatches, and Timestamp→Int64 (nanosAsLong), use CometCastColumnExpr // with spark_parquet_convert which handles field-name-based selection, @@ -413,7 +421,7 @@ impl SparkPhysicalExprAdapter { let comet_cast: Arc = Arc::new( CometCastColumnExpr::new( child, - Arc::clone(cast.input_field()), + input_field, Arc::clone(cast.target_field()), None, ) diff --git a/native/spark-expr/src/datetime_funcs/date_from_unix_date.rs b/native/spark-expr/src/datetime_funcs/date_from_unix_date.rs index 1c88fc47ab..0e624e6472 100644 --- a/native/spark-expr/src/datetime_funcs/date_from_unix_date.rs +++ b/native/spark-expr/src/datetime_funcs/date_from_unix_date.rs @@ -21,7 +21,6 @@ use datafusion::common::{utils::take_function_args, DataFusionError, Result, Sca use datafusion::logical_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; -use std::any::Any; use std::sync::Arc; /// Spark-compatible date_from_unix_date function. @@ -48,10 +47,6 @@ impl Default for SparkDateFromUnixDate { } impl ScalarUDFImpl for SparkDateFromUnixDate { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { "date_from_unix_date" } diff --git a/native/spark-expr/src/datetime_funcs/hours.rs b/native/spark-expr/src/datetime_funcs/hours.rs index ea3ef742a4..26ec35ef25 100644 --- a/native/spark-expr/src/datetime_funcs/hours.rs +++ b/native/spark-expr/src/datetime_funcs/hours.rs @@ -32,7 +32,7 @@ use datafusion::logical_expr::{ ColumnarValue, ScalarFunctionArgs, ScalarUDFImpl, Signature, Volatility, }; use num::integer::div_floor; -use std::{any::Any, fmt::Debug, sync::Arc}; +use std::{fmt::Debug, sync::Arc}; const MICROS_PER_HOUR: i64 = 3_600_000_000; @@ -56,10 +56,6 @@ impl Default for SparkHoursTransform { } impl ScalarUDFImpl for SparkHoursTransform { - fn as_any(&self) -> &dyn Any { - self - } - fn name(&self) -> &str { "hours_transform" } From e19aa50a7e9a52df18933e57f4b6bc2efd5fd24e Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 15 Apr 2026 17:05:28 -0400 Subject: [PATCH 05/12] Fix metrics aggregation for native scan. --- native/core/src/execution/metrics/utils.rs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/native/core/src/execution/metrics/utils.rs b/native/core/src/execution/metrics/utils.rs index eb7e10bfc9..8547c0c3ca 100644 --- a/native/core/src/execution/metrics/utils.rs +++ b/native/core/src/execution/metrics/utils.rs @@ -68,9 +68,13 @@ pub(crate) fn to_native_metric_node( Some(metrics.aggregate_by_name()) }; - // add metrics + // Aggregate metrics by name using DataFusion's aggregate_by_name(), which + // correctly handles duplicate metric names (e.g. BaselineMetrics registered + // by both FileStream and ParquetMorselizer on the same ExecutionPlanMetricsSet). + // The additional_native_plans branch below already does this. node_metrics .unwrap_or_default() + .aggregate_by_name() .iter() .map(|m| m.value()) .map(|m| (m.name(), m.as_usize() as i64)) From e798f9a451097aecbc2507002d771f71d4bcaf08 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 15 Apr 2026 17:25:11 -0400 Subject: [PATCH 06/12] enable SMJ with filter by default --- common/src/main/scala/org/apache/comet/CometConf.scala | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/common/src/main/scala/org/apache/comet/CometConf.scala b/common/src/main/scala/org/apache/comet/CometConf.scala index 046ccf0b1c..eede60eb76 100644 --- a/common/src/main/scala/org/apache/comet/CometConf.scala +++ b/common/src/main/scala/org/apache/comet/CometConf.scala @@ -313,9 +313,10 @@ object CometConf extends ShimCometConf { val COMET_EXEC_SORT_MERGE_JOIN_WITH_JOIN_FILTER_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.exec.sortMergeJoinWithJoinFilter.enabled") .category(CATEGORY_ENABLE_EXEC) - .doc("Experimental support for Sort Merge Join with filter") + .doc("Support for Sort Merge Join with filter. " + + "Deprecated: this config will be removed in a future release.") .booleanConf - .createWithDefault(false) + .createWithDefault(true) val COMET_TRACING_ENABLED: ConfigEntry[Boolean] = conf("spark.comet.tracing.enabled") .category(CATEGORY_TUNING) From 4236c2709f092b8999f7a176beb9beb86ebf8405 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 15 Apr 2026 20:03:11 -0400 Subject: [PATCH 07/12] add test for SPARK-43113 --- .../apache/comet/exec/CometJoinSuite.scala | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala index 49fbe10c30..c96332b66b 100644 --- a/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala +++ b/spark/src/test/scala/org/apache/comet/exec/CometJoinSuite.scala @@ -443,4 +443,40 @@ class CometJoinSuite extends CometTestBase { } } } + + // Reproducer for SPARK-43113: full outer SMJ with a join filter that references + // a nullable column should not match when the filter evaluates to NULL. + test("SPARK-43113: Full outer SMJ with NULL in join filter") { + withTempView("l", "r") { + // testData2: (a, b) — all non-null + Seq((1, 1), (1, 2), (2, 1), (2, 2), (3, 1), (3, 2)) + .toDF("a", "b") + .createOrReplaceTempView("l") + + // testData3: (a, b) — b is nullable + Seq((1, None), (2, Some(2))) + .toDF("a", "b") + .createOrReplaceTempView("r") + + val query = + """select /*+ MERGE(r) */ * + |from l + |full outer join r + |on l.a = r.a + |and l.b < (r.b + 1) + |and l.b < (r.a + 1)""".stripMargin + + val expected = Seq( + (Some(1), Some(1), None, None), + (Some(1), Some(2), None, None), + (None, None, Some(1), None), + (Some(2), Some(1), Some(2), Some(2)), + (Some(2), Some(2), Some(2), Some(2)), + (Some(3), Some(1), None, None), + (Some(3), Some(2), None, None)).toDF("a", "b", "a", "b") + + val df = sql(query) + checkAnswer(df, expected) + } + } } From 3bff80bce805628dfad39ffc3301fb82883647b9 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 15 Apr 2026 20:56:15 -0400 Subject: [PATCH 08/12] test https://github.com/apache/datafusion/pull/21660 --- native/Cargo.lock | 767 +++++++++++++++++++++++++++++++--------------- native/Cargo.toml | 8 +- 2 files changed, 531 insertions(+), 244 deletions(-) diff --git a/native/Cargo.lock b/native/Cargo.lock index d92ee9261b..08e4493ca2 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -189,7 +189,7 @@ dependencies = [ "miniz_oxide", "num-bigint", "quad-rand", - "rand 0.9.2", + "rand 0.9.4", "regex-lite", "serde", "serde_bytes", @@ -415,7 +415,7 @@ version = "58.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c30a1365d7a7dc50cc847e54154e6af49e4c4b0fddc9f607b687f29212082743" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "serde_core", "serde_json", ] @@ -459,9 +459,9 @@ checksum = "b0f477b951e452a0b6b4a10b53ccd569042d1d01729b519e02074a9c0958a063" [[package]] name = "assertables" -version = "9.8.6" +version = "9.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "098fbfe90e8af520c4a968eaad7a4031908e473394f16c99cd9cce6369328a68" +checksum = "627b0b35c347252505ec6c078827dfcecc4e9dcfa931c54c54066f9245ac752b" dependencies = [ "regex", "walkdir", @@ -668,9 +668,9 @@ dependencies = [ [[package]] name = "aws-lc-rs" -version = "1.16.2" +version = "1.16.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a054912289d18629dc78375ba2c3726a3afe3ff71b4edba9dedfca0e3446d1fc" +checksum = "0ec6fb3fe69024a75fa7e1bfb48aa6cf59706a101658ea01bfd33b2b248a038f" dependencies = [ "aws-lc-sys", "zeroize", @@ -678,9 +678,9 @@ dependencies = [ [[package]] name = "aws-lc-sys" -version = "0.39.1" +version = "0.40.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83a25cf98105baa966497416dbd42565ce3a8cf8dbfd59803ec9ad46f3126399" +checksum = "f50037ee5e1e41e7b8f9d161680a725bd1626cb6f8c7e901f91f942850852fe7" dependencies = [ "cc", "cmake", @@ -1076,7 +1076,7 @@ version = "0.72.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "cexpr", "clang-sys", "itertools 0.13.0", @@ -1098,9 +1098,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.11.0" +version = "2.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "843867be96c8daad0d758b57df9392b6d8d271134fce549de6ce169ff98a92af" +checksum = "c4512299f36f043ab09a583e57bceb5a5aab7a73db1805848e8fef3c9e8c78b3" [[package]] name = "blake2" @@ -1281,9 +1281,9 @@ dependencies = [ [[package]] name = "cc" -version = "1.2.59" +version = "1.2.60" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b7a4d3ec6524d28a329fc53654bbadc9bdd7b0431f5d65f1a56ffb28a1ee5283" +checksum = "43c5703da9466b66a946814e1adf53ea2c90f10063b86290cc9eb67ce3478a20" dependencies = [ "find-msvc-tools", "jobserver", @@ -1326,7 +1326,7 @@ checksum = "6f8d983286843e49675a4b7a2d174efe136dc93a18d69130dd18198a6c167601" dependencies = [ "cfg-if", "cpufeatures 0.3.0", - "rand_core 0.10.0", + "rand_core 0.10.1", ] [[package]] @@ -1403,9 +1403,9 @@ dependencies = [ [[package]] name = "clap" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b193af5b67834b676abd72466a96c1024e6a6ad978a1f484bd90b85c94041351" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" dependencies = [ "clap_builder", "clap_derive", @@ -1425,9 +1425,9 @@ dependencies = [ [[package]] name = "clap_derive" -version = "4.6.0" +version = "4.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1110bd8a634a1ab8cb04345d8d878267d57c3cf1b38d91b71af6686408bbca6a" +checksum = "f2ce8604710f6733aa641a2b3731eaa1e8b3d9973d5e3565da11800813f997a9" dependencies = [ "heck", "proc-macro2", @@ -1832,7 +1832,7 @@ dependencies = [ [[package]] name = "datafusion" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", "arrow-schema", @@ -1840,25 +1840,25 @@ dependencies = [ "chrono", "datafusion-catalog", "datafusion-catalog-listing", - "datafusion-common", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-common-runtime", "datafusion-datasource", "datafusion-datasource-arrow", "datafusion-datasource-csv", "datafusion-datasource-json", "datafusion-datasource-parquet", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", + "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-functions 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-functions-aggregate 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-functions-nested 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-functions-table", "datafusion-functions-window", "datafusion-optimizer", - "datafusion-physical-expr", + "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-physical-optimizer", "datafusion-physical-plan", "datafusion-session", @@ -1879,17 +1879,17 @@ dependencies = [ [[package]] name = "datafusion-catalog" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", + "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-physical-plan", "datafusion-session", "futures", @@ -1903,18 +1903,18 @@ dependencies = [ [[package]] name = "datafusion-catalog-listing" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", "async-trait", "datafusion-catalog", - "datafusion-common", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", + "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-physical-plan", "futures", "itertools 0.14.0", @@ -1941,7 +1941,7 @@ dependencies = [ "datafusion-comet-shuffle", "datafusion-comet-spark-expr", "datafusion-datasource", - "datafusion-functions-nested", + "datafusion-functions-nested 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", "datafusion-physical-expr-adapter", "datafusion-spark", "futures", @@ -2097,6 +2097,27 @@ dependencies = [ name = "datafusion-common" version = "53.0.0" source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" +dependencies = [ + "arrow", + "arrow-ipc", + "arrow-schema", + "chrono", + "foldhash 0.2.0", + "half", + "hashbrown 0.17.0", + "indexmap 2.14.0", + "itertools 0.14.0", + "libc", + "log", + "tokio", + "uuid", + "web-time", +] + +[[package]] +name = "datafusion-common" +version = "53.0.0" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", "arrow-ipc", @@ -2121,7 +2142,7 @@ dependencies = [ [[package]] name = "datafusion-common-runtime" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "futures", "log", @@ -2131,7 +2152,7 @@ dependencies = [ [[package]] name = "datafusion-datasource" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", "async-compression", @@ -2139,13 +2160,13 @@ dependencies = [ "bytes", "bzip2", "chrono", - "datafusion-common", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr", + "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-physical-plan", "datafusion-session", "flate2", @@ -2155,7 +2176,7 @@ dependencies = [ "liblzma", "log", "object_store", - "rand 0.9.2", + "rand 0.9.4", "tokio", "tokio-util", "url", @@ -2165,18 +2186,18 @@ dependencies = [ [[package]] name = "datafusion-datasource-arrow" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", "arrow-ipc", "async-trait", "bytes", - "datafusion-common", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", + "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-physical-plan", "datafusion-session", "futures", @@ -2188,17 +2209,17 @@ dependencies = [ [[package]] name = "datafusion-datasource-csv" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", + "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-physical-plan", "datafusion-session", "futures", @@ -2210,17 +2231,17 @@ dependencies = [ [[package]] name = "datafusion-datasource-json" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-physical-expr-common", + "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-physical-plan", "datafusion-session", "futures", @@ -2232,21 +2253,21 @@ dependencies = [ [[package]] name = "datafusion-datasource-parquet" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr", + "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-functions 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-physical-plan", "datafusion-pruning", "datafusion-session", @@ -2264,6 +2285,11 @@ name = "datafusion-doc" version = "53.0.0" source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" +[[package]] +name = "datafusion-doc" +version = "53.0.0" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" + [[package]] name = "datafusion-execution" version = "53.0.0" @@ -2273,15 +2299,36 @@ dependencies = [ "arrow-buffer", "async-trait", "dashmap", - "datafusion-common", - "datafusion-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "futures", + "log", + "object_store", + "parking_lot", + "rand 0.9.4", + "tempfile", + "url", +] + +[[package]] +name = "datafusion-execution" +version = "53.0.0" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" +dependencies = [ + "arrow", + "arrow-buffer", + "async-trait", + "dashmap", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "futures", "log", "object_store", "parking_lot", "parquet", - "rand 0.9.2", + "rand 0.9.4", "tempfile", "url", ] @@ -2295,12 +2342,33 @@ dependencies = [ "arrow-schema", "async-trait", "chrono", - "datafusion-common", - "datafusion-doc", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr-common", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-doc 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-functions-window-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "indexmap 2.14.0", + "itertools 0.14.0", + "serde_json", + "sqlparser", +] + +[[package]] +name = "datafusion-expr" +version = "53.0.0" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" +dependencies = [ + "arrow", + "arrow-schema", + "async-trait", + "chrono", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-doc 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-functions-window-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "indexmap 2.14.0", "itertools 0.14.0", "serde_json", @@ -2313,7 +2381,18 @@ version = "53.0.0" source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", - "datafusion-common", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "indexmap 2.14.0", + "itertools 0.14.0", +] + +[[package]] +name = "datafusion-expr-common" +version = "53.0.0" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" +dependencies = [ + "arrow", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "indexmap 2.14.0", "itertools 0.14.0", ] @@ -2322,6 +2401,32 @@ dependencies = [ name = "datafusion-functions" version = "53.0.0" source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" +dependencies = [ + "arrow", + "arrow-buffer", + "base64", + "chrono", + "chrono-tz", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-doc 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-macros 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "hex", + "itertools 0.14.0", + "log", + "memchr", + "num-traits", + "rand 0.9.4", + "regex", + "uuid", +] + +[[package]] +name = "datafusion-functions" +version = "53.0.0" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", "arrow-buffer", @@ -2330,19 +2435,19 @@ dependencies = [ "blake3", "chrono", "chrono-tz", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-macros", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-doc 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-macros 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "hex", "itertools 0.14.0", "log", "md-5", "memchr", "num-traits", - "rand 0.9.2", + "rand 0.9.4", "regex", "sha2", "uuid", @@ -2354,14 +2459,34 @@ version = "53.0.0" source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-doc 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-macros 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "foldhash 0.2.0", + "half", + "log", + "num-traits", +] + +[[package]] +name = "datafusion-functions-aggregate" +version = "53.0.0" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" +dependencies = [ + "arrow", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-doc 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-macros 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "foldhash 0.2.0", "half", "log", @@ -2374,9 +2499,20 @@ version = "53.0.0" source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", - "datafusion-common", - "datafusion-expr-common", - "datafusion-physical-expr-common", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", +] + +[[package]] +name = "datafusion-functions-aggregate-common" +version = "53.0.0" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" +dependencies = [ + "arrow", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", ] [[package]] @@ -2386,16 +2522,40 @@ source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#019830 dependencies = [ "arrow", "arrow-ord", - "datafusion-common", - "datafusion-doc", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-aggregate-common", - "datafusion-macros", - "datafusion-physical-expr-common", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-doc 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-functions 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-functions-aggregate 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-macros 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "hashbrown 0.17.0", + "itertools 0.14.0", + "itoa", + "log", + "memchr", +] + +[[package]] +name = "datafusion-functions-nested" +version = "53.0.0" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" +dependencies = [ + "arrow", + "arrow-ord", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-doc 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-functions 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-functions-aggregate 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-macros 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "hashbrown 0.17.0", "itertools 0.14.0", "itoa", @@ -2406,13 +2566,13 @@ dependencies = [ [[package]] name = "datafusion-functions-table" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", "async-trait", "datafusion-catalog", - "datafusion-common", - "datafusion-expr", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-physical-plan", "parking_lot", ] @@ -2420,16 +2580,16 @@ dependencies = [ [[package]] name = "datafusion-functions-window" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", - "datafusion-common", - "datafusion-doc", - "datafusion-expr", - "datafusion-functions-window-common", - "datafusion-macros", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-doc 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-functions-window-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-macros 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "log", ] @@ -2438,8 +2598,17 @@ name = "datafusion-functions-window-common" version = "53.0.0" source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ - "datafusion-common", - "datafusion-physical-expr-common", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", +] + +[[package]] +name = "datafusion-functions-window-common" +version = "53.0.0" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" +dependencies = [ + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", ] [[package]] @@ -2447,7 +2616,17 @@ name = "datafusion-macros" version = "53.0.0" source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ - "datafusion-doc", + "datafusion-doc 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "quote", + "syn 2.0.117", +] + +[[package]] +name = "datafusion-macros" +version = "53.0.0" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" +dependencies = [ + "datafusion-doc 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "quote", "syn 2.0.117", ] @@ -2455,14 +2634,14 @@ dependencies = [ [[package]] name = "datafusion-optimizer" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "indexmap 2.14.0", "itertools 0.14.0", "log", @@ -2476,11 +2655,31 @@ version = "53.0.0" source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" dependencies = [ "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-functions-aggregate-common", - "datafusion-physical-expr-common", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "half", + "hashbrown 0.17.0", + "indexmap 2.14.0", + "itertools 0.14.0", + "parking_lot", + "petgraph", + "tokio", +] + +[[package]] +name = "datafusion-physical-expr" +version = "53.0.0" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" +dependencies = [ + "arrow", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "half", "hashbrown 0.17.0", "indexmap 2.14.0", @@ -2493,14 +2692,14 @@ dependencies = [ [[package]] name = "datafusion-physical-expr-adapter" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", - "datafusion-common", - "datafusion-expr", - "datafusion-functions", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-functions 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "itertools 0.14.0", ] @@ -2511,8 +2710,23 @@ source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#019830 dependencies = [ "arrow", "chrono", - "datafusion-common", - "datafusion-expr-common", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "hashbrown 0.17.0", + "indexmap 2.14.0", + "itertools 0.14.0", + "parking_lot", +] + +[[package]] +name = "datafusion-physical-expr-common" +version = "53.0.0" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" +dependencies = [ + "arrow", + "chrono", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "hashbrown 0.17.0", "indexmap 2.14.0", "itertools 0.14.0", @@ -2522,15 +2736,15 @@ dependencies = [ [[package]] name = "datafusion-physical-optimizer" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-physical-plan", "datafusion-pruning", "itertools 0.14.0", @@ -2539,21 +2753,21 @@ dependencies = [ [[package]] name = "datafusion-physical-plan" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", "arrow-ord", "arrow-schema", "async-trait", - "datafusion-common", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-common-runtime", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate-common", - "datafusion-functions-window-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-functions 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-functions-window-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "futures", "half", "hashbrown 0.17.0", @@ -2569,14 +2783,14 @@ dependencies = [ [[package]] name = "datafusion-pruning" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", - "datafusion-common", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-datasource", - "datafusion-expr-common", - "datafusion-physical-expr", - "datafusion-physical-expr-common", + "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-physical-plan", "log", ] @@ -2584,12 +2798,12 @@ dependencies = [ [[package]] name = "datafusion-session" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "async-trait", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "datafusion-physical-plan", "parking_lot", ] @@ -2597,7 +2811,7 @@ dependencies = [ [[package]] name = "datafusion-spark" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", "bigdecimal", @@ -2605,16 +2819,16 @@ dependencies = [ "crc32fast", "datafusion", "datafusion-catalog", - "datafusion-common", - "datafusion-execution", - "datafusion-expr", - "datafusion-functions", - "datafusion-functions-aggregate", - "datafusion-functions-nested", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-functions 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-functions-aggregate 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-functions-nested 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "log", "num-traits", "percent-encoding", - "rand 0.9.2", + "rand 0.9.4", "serde_json", "sha1 0.11.0", "sha2", @@ -2624,14 +2838,14 @@ dependencies = [ [[package]] name = "datafusion-sql" version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" +source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", "bigdecimal", "chrono", - "datafusion-common", - "datafusion-expr", - "datafusion-functions-nested", + "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-functions-nested 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", "indexmap 2.14.0", "log", "regex", @@ -2853,7 +3067,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -2941,7 +3155,7 @@ version = "25.12.19" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "35f6839d7b3b98adde531effaf34f0c2badc6f4735d26fe74709d8e513a96ef3" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "rustc_version", ] @@ -3150,7 +3364,7 @@ dependencies = [ "cfg-if", "libc", "r-efi 6.0.0", - "rand_core 0.10.0", + "rand_core 0.10.1", "wasip2", "wasip3", ] @@ -3417,16 +3631,15 @@ dependencies = [ [[package]] name = "hyper-rustls" -version = "0.27.7" +version = "0.27.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" +checksum = "33ca68d021ef39cf6463ab54c1d0f5daf03377b70561305bb89a8f83aab66e0f" dependencies = [ "http 1.4.0", "hyper", "hyper-util", "rustls", "rustls-native-certs", - "rustls-pki-types", "tokio", "tokio-rustls", "tower-service", @@ -3759,7 +3972,7 @@ checksum = "3640c1c38b8e4e43584d8df18be5fc6b0aa314ce6ebf51b53313d4306cca8e46" dependencies = [ "hermit-abi", "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3815,7 +4028,7 @@ dependencies = [ "portable-atomic-util", "serde_core", "wasm-bindgen", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3932,9 +4145,9 @@ dependencies = [ [[package]] name = "js-sys" -version = "0.3.94" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2e04e2ef80ce82e13552136fabeef8a5ed1f985a96805761cbb9a2c34e7664d9" +checksum = "2964e92d1d9dc3364cae4d718d93f227e3abb088e747d92e0395bfdedf1c12ca" dependencies = [ "cfg-if", "futures-util", @@ -4162,7 +4375,7 @@ dependencies = [ "log-mdc", "mock_instant", "parking_lot", - "rand 0.9.2", + "rand 0.9.4", "serde", "serde-value", "serde_json", @@ -4951,9 +5164,9 @@ dependencies = [ [[package]] name = "pkg-config" -version = "0.3.32" +version = "0.3.33" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7edddbd0b52d732b21ad9a5fab5c704c14cd949e5e9a1ec5929a24fded1b904c" +checksum = "19f132c84eca552bf34cab8ec81f1c1dcc229b811638f9d283dceabe58c5569e" [[package]] name = "plotters" @@ -5095,7 +5308,7 @@ version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "25485360a54d6861439d60facef26de713b1e126bf015ec8f98239467a2b82f7" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "chrono", "flate2", "procfs-core", @@ -5108,7 +5321,7 @@ version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e6401bf7b6af22f78b563665d15a22e9aef27775b79b149a66ca022468a4e405" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "chrono", "hex", ] @@ -5130,7 +5343,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "343d3bd7056eda839b03204e68deff7d1b13aba7af2b2fd16890697274262ee7" dependencies = [ "heck", - "itertools 0.13.0", + "itertools 0.14.0", "log", "multimap", "petgraph", @@ -5149,7 +5362,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "27c6023962132f4b30eb4c172c91ce92d933da334c59c23cddee82358ddafb0b" dependencies = [ "anyhow", - "itertools 0.13.0", + "itertools 0.14.0", "proc-macro2", "quote", "syn 2.0.117", @@ -5238,7 +5451,7 @@ dependencies = [ "bytes", "getrandom 0.3.4", "lru-slab", - "rand 0.9.2", + "rand 0.9.4", "ring", "rustc-hash 2.1.2", "rustls", @@ -5261,7 +5474,7 @@ dependencies = [ "once_cell", "socket2", "tracing", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -5298,9 +5511,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.2" +version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" +checksum = "44c5af06bb1b7d3216d91932aed5265164bf384dc89cd6ba05cf59a35f5f76ea" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.5", @@ -5314,7 +5527,7 @@ checksum = "d2e8e8bcc7961af1fdac401278c6a831614941f6164ee3bf4ce61b7edb162207" dependencies = [ "chacha20", "getrandom 0.4.2", - "rand_core 0.10.0", + "rand_core 0.10.1", ] [[package]] @@ -5357,15 +5570,15 @@ dependencies = [ [[package]] name = "rand_core" -version = "0.10.0" +version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c8d0fd677905edcbeedbf2edb6494d676f0e98d54d5cf9bda0b061cb8fb8aba" +checksum = "63b8176103e19a2643978565ca18b50549f6101881c443590420e4dc998a3c69" [[package]] name = "rayon" -version = "1.11.0" +version = "1.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" dependencies = [ "either", "rayon-core", @@ -5387,7 +5600,7 @@ version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", ] [[package]] @@ -5639,7 +5852,7 @@ version = "0.38.44" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys 0.4.15", @@ -5652,18 +5865,18 @@ version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b6fe4565b9518b83ef4f91bb47ce29620ca828bd32cb7e408f0062e9930ba190" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "errno", "libc", "linux-raw-sys 0.12.1", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "rustls" -version = "0.23.37" +version = "0.23.38" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "758025cb5fccfd3bc2fd74708fd4682be41d99e5dff73c377c0646c6012c73a4" +checksum = "69f9466fb2c14ea04357e91413efb882e2a6d4a406e625449bc0a5d360d53a21" dependencies = [ "aws-lc-rs", "once_cell", @@ -5698,9 +5911,9 @@ dependencies = [ [[package]] name = "rustls-webpki" -version = "0.103.10" +version = "0.103.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef" +checksum = "8279bb85272c9f10811ae6a6c547ff594d6a7f3c6c6b02ee9726d1d0dcfcdd06" dependencies = [ "aws-lc-rs", "ring", @@ -5794,7 +6007,7 @@ version = "3.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b7f4bc775c73d9a02cde8bf7b2ec4c9d12743edf609006c7facc23998404cd1d" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "core-foundation", "core-foundation-sys", "libc", @@ -6183,9 +6396,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "symbolic-common" -version = "12.17.4" +version = "12.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "803d14d7cb9c6fa5b95a6f3de8af95b356a528d391998fa45a07d320a5573e51" +checksum = "6aba7211a1803a826f108af9f4d86d25abe880712f2a6449479279e861b293f8" dependencies = [ "debugid", "memmap2", @@ -6195,9 +6408,9 @@ dependencies = [ [[package]] name = "symbolic-demangle" -version = "12.17.4" +version = "12.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39505731ae891b2dde47b0e4ae2ec40a7ced3476ab1129f1bf829e3fba62bb83" +checksum = "595bddd9d363c2ef6fc9fb33b98416ff209c5aae8bdca89a7dbbf2ef5e0ecc45" dependencies = [ "cpp_demangle", "rustc-demangle", @@ -6259,10 +6472,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32497e9a4c7b38532efcdebeef879707aa9f794296a4f0244f6f69e9bc8574bd" dependencies = [ "fastrand", - "getrandom 0.3.4", + "getrandom 0.4.2", "once_cell", "rustix 1.1.4", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -6434,9 +6647,9 @@ checksum = "1f3ccbac311fea05f86f61904b462b55fb3df8837a366dfc601a0161d0532f20" [[package]] name = "tokio" -version = "1.51.1" +version = "1.52.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f66bf9585cda4b724d3e78ab34b73fb2bbaba9011b9bfdf69dc836382ea13b8c" +checksum = "a91135f59b1cbf38c91e73cf3386fca9bb77915c45ce2771460c9d92f0f3d776" dependencies = [ "bytes", "libc", @@ -6516,7 +6729,7 @@ version = "0.6.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d4e6559d53cc268e5031cd8429d05415bc4cb4aefc4aa5d6cc35fbf5b924a1f8" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "bytes", "futures-util", "http 1.4.0", @@ -6583,7 +6796,7 @@ version = "2.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" dependencies = [ - "rand 0.9.2", + "rand 0.9.4", ] [[package]] @@ -6811,9 +7024,9 @@ dependencies = [ [[package]] name = "wasm-bindgen" -version = "0.2.117" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0551fc1bb415591e3372d0bc4780db7e587d84e2a7e79da121051c5c4b89d0b0" +checksum = "0bf938a0bacb0469e83c1e148908bd7d5a6010354cf4fb73279b7447422e3a89" dependencies = [ "cfg-if", "once_cell", @@ -6824,9 +7037,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-futures" -version = "0.4.67" +version = "0.4.68" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03623de6905b7206edd0a75f69f747f134b7f0a2323392d664448bf2d3c5d87e" +checksum = "f371d383f2fb139252e0bfac3b81b265689bf45b6874af544ffa4c975ac1ebf8" dependencies = [ "js-sys", "wasm-bindgen", @@ -6834,9 +7047,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.117" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fbdf9a35adf44786aecd5ff89b4563a90325f9da0923236f6104e603c7e86be" +checksum = "eeff24f84126c0ec2db7a449f0c2ec963c6a49efe0698c4242929da037ca28ed" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -6844,9 +7057,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.117" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dca9693ef2bab6d4e6707234500350d8dad079eb508dca05530c85dc3a529ff2" +checksum = "9d08065faf983b2b80a79fd87d8254c409281cf7de75fc4b773019824196c904" dependencies = [ "bumpalo", "proc-macro2", @@ -6857,9 +7070,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-shared" -version = "0.2.117" +version = "0.2.118" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "39129a682a6d2d841b6c429d0c51e5cb0ed1a03829d8b3d1e69a011e62cb3d3b" +checksum = "5fd04d9e306f1907bd13c6361b5c6bfc7b3b3c095ed3f8a9246390f8dbdee129" dependencies = [ "unicode-ident", ] @@ -6905,7 +7118,7 @@ version = "0.244.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" dependencies = [ - "bitflags 2.11.0", + "bitflags 2.11.1", "hashbrown 0.15.5", "indexmap 2.14.0", "semver", @@ -6913,9 +7126,9 @@ dependencies = [ [[package]] name = "web-sys" -version = "0.3.94" +version = "0.3.95" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd70027e39b12f0849461e08ffc50b9cd7688d942c1c8e3c7b22273236b4dd0a" +checksum = "4f2dfbb17949fa2088e5d39408c48368947b86f7834484e87b73de55bc14d97d" dependencies = [ "js-sys", "wasm-bindgen", @@ -6974,7 +7187,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -7069,6 +7282,15 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + [[package]] name = "windows-sys" version = "0.61.2" @@ -7102,13 +7324,30 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" @@ -7121,6 +7360,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" version = "0.42.2" @@ -7133,6 +7378,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" version = "0.42.2" @@ -7145,12 +7396,24 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" version = "0.42.2" @@ -7163,6 +7426,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" version = "0.42.2" @@ -7175,6 +7444,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" version = "0.42.2" @@ -7187,6 +7462,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" version = "0.42.2" @@ -7199,6 +7480,12 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" +[[package]] +name = "windows_x86_64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" + [[package]] name = "wit-bindgen" version = "0.51.0" @@ -7257,7 +7544,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" dependencies = [ "anyhow", - "bitflags 2.11.0", + "bitflags 2.11.1", "indexmap 2.14.0", "log", "serde", diff --git a/native/Cargo.toml b/native/Cargo.toml index 1e1055bfd0..2aeacc8544 100644 --- a/native/Cargo.toml +++ b/native/Cargo.toml @@ -38,10 +38,10 @@ arrow = { version = "58.1.0", features = ["prettyprint", "ffi", "chrono-tz"] } async-trait = { version = "0.1" } bytes = { version = "1.11.1" } parquet = { version = "58.1.0", default-features = false, features = ["experimental"] } -datafusion = { git = "https://github.com/mbutrovich/datafusion.git", rev = "0198302544", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } -datafusion-datasource = { git = "https://github.com/mbutrovich/datafusion.git", rev = "0198302544" } -datafusion-physical-expr-adapter = { git = "https://github.com/mbutrovich/datafusion.git", rev = "0198302544" } -datafusion-spark = { git = "https://github.com/mbutrovich/datafusion.git", rev = "0198302544", features = ["core"] } +datafusion = { git = "https://github.com/mbutrovich/datafusion.git", rev = "9598c96998", default-features = false, features = ["unicode_expressions", "crypto_expressions", "nested_expressions", "parquet"] } +datafusion-datasource = { git = "https://github.com/mbutrovich/datafusion.git", rev = "9598c96998" } +datafusion-physical-expr-adapter = { git = "https://github.com/mbutrovich/datafusion.git", rev = "9598c96998" } +datafusion-spark = { git = "https://github.com/mbutrovich/datafusion.git", rev = "9598c96998", features = ["core"] } datafusion-comet-spark-expr = { path = "spark-expr" } datafusion-comet-common = { path = "common" } datafusion-comet-jni-bridge = { path = "jni-bridge" } From ec18066beb2f37d1fc7d75f4835f40991341b211 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 15 Apr 2026 21:13:06 -0400 Subject: [PATCH 09/12] Fix after merging in latest datafusion main in the feature branch. --- native/spark-expr/src/array_funcs/array_insert.rs | 4 ---- .../spark-expr/src/array_funcs/get_array_struct_fields.rs | 4 ---- native/spark-expr/src/array_funcs/list_extract.rs | 4 ---- native/spark-expr/src/bloom_filter/bloom_filter_agg.rs | 2 +- native/spark-expr/src/conditional_funcs/if_expr.rs | 3 --- native/spark-expr/src/conversion_funcs/cast.rs | 4 ---- native/spark-expr/src/csv_funcs/to_csv.rs | 4 ---- native/spark-expr/src/datetime_funcs/timestamp_trunc.rs | 4 ---- native/spark-expr/src/json_funcs/from_json.rs | 4 ---- native/spark-expr/src/json_funcs/to_json.rs | 4 ---- native/spark-expr/src/math_funcs/internal/checkoverflow.rs | 7 ------- .../src/math_funcs/internal/decimal_rescale_check.rs | 7 ------- native/spark-expr/src/math_funcs/internal/normalize_nan.rs | 4 ---- native/spark-expr/src/math_funcs/negative.rs | 3 --- .../spark-expr/src/math_funcs/wide_decimal_binary_expr.rs | 4 ---- .../nondetermenistic_funcs/monotonically_increasing_id.rs | 4 ---- native/spark-expr/src/nondetermenistic_funcs/rand.rs | 4 ---- native/spark-expr/src/nondetermenistic_funcs/randn.rs | 4 ---- native/spark-expr/src/predicate_funcs/rlike.rs | 4 ---- native/spark-expr/src/string_funcs/substring.rs | 4 ---- native/spark-expr/src/struct_funcs/create_named_struct.rs | 4 ---- native/spark-expr/src/struct_funcs/get_struct_field.rs | 4 ---- native/spark-expr/src/unbound.rs | 3 --- 23 files changed, 1 insertion(+), 92 deletions(-) diff --git a/native/spark-expr/src/array_funcs/array_insert.rs b/native/spark-expr/src/array_funcs/array_insert.rs index bce00483bc..af094fbfd0 100644 --- a/native/spark-expr/src/array_funcs/array_insert.rs +++ b/native/spark-expr/src/array_funcs/array_insert.rs @@ -92,10 +92,6 @@ impl ArrayInsert { } impl PhysicalExpr for ArrayInsert { - fn as_any(&self) -> &dyn Any { - self - } - fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { Display::fmt(self, f) } diff --git a/native/spark-expr/src/array_funcs/get_array_struct_fields.rs b/native/spark-expr/src/array_funcs/get_array_struct_fields.rs index dc05a3b7f0..dd6d340ae5 100644 --- a/native/spark-expr/src/array_funcs/get_array_struct_fields.rs +++ b/native/spark-expr/src/array_funcs/get_array_struct_fields.rs @@ -75,10 +75,6 @@ impl GetArrayStructFields { } impl PhysicalExpr for GetArrayStructFields { - fn as_any(&self) -> &dyn Any { - self - } - fn data_type(&self, input_schema: &Schema) -> DataFusionResult { let struct_field = self.child_field(input_schema)?; match self.child.data_type(input_schema)? { diff --git a/native/spark-expr/src/array_funcs/list_extract.rs b/native/spark-expr/src/array_funcs/list_extract.rs index d3661f496a..26533a75b8 100644 --- a/native/spark-expr/src/array_funcs/list_extract.rs +++ b/native/spark-expr/src/array_funcs/list_extract.rs @@ -113,10 +113,6 @@ impl ListExtract { } impl PhysicalExpr for ListExtract { - fn as_any(&self) -> &dyn Any { - self - } - fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { Display::fmt(self, f) } diff --git a/native/spark-expr/src/bloom_filter/bloom_filter_agg.rs b/native/spark-expr/src/bloom_filter/bloom_filter_agg.rs index 3e9e5f00ad..455e20a019 100644 --- a/native/spark-expr/src/bloom_filter/bloom_filter_agg.rs +++ b/native/spark-expr/src/bloom_filter/bloom_filter_agg.rs @@ -41,7 +41,7 @@ pub struct BloomFilterAgg { #[inline] fn extract_i32_from_literal(expr: Arc) -> i32 { - match expr.as_any().downcast_ref::().unwrap().value() { + match (*expr).downcast_ref::().unwrap().value() { ScalarValue::Int64(scalar_value) => scalar_value.unwrap() as i32, _ => { unreachable!() diff --git a/native/spark-expr/src/conditional_funcs/if_expr.rs b/native/spark-expr/src/conditional_funcs/if_expr.rs index 6b1291fbbe..bedb515c2f 100644 --- a/native/spark-expr/src/conditional_funcs/if_expr.rs +++ b/native/spark-expr/src/conditional_funcs/if_expr.rs @@ -84,9 +84,6 @@ impl IfExpr { impl PhysicalExpr for IfExpr { /// Return a reference to Any that can be used for down-casting - fn as_any(&self) -> &dyn Any { - self - } fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { Display::fmt(self, f) diff --git a/native/spark-expr/src/conversion_funcs/cast.rs b/native/spark-expr/src/conversion_funcs/cast.rs index efba0d17d8..1ba9f4a9e7 100644 --- a/native/spark-expr/src/conversion_funcs/cast.rs +++ b/native/spark-expr/src/conversion_funcs/cast.rs @@ -695,10 +695,6 @@ impl Display for Cast { } impl PhysicalExpr for Cast { - fn as_any(&self) -> &dyn Any { - self - } - fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { Display::fmt(self, f) } diff --git a/native/spark-expr/src/csv_funcs/to_csv.rs b/native/spark-expr/src/csv_funcs/to_csv.rs index f41cb7f918..2d49cde585 100644 --- a/native/spark-expr/src/csv_funcs/to_csv.rs +++ b/native/spark-expr/src/csv_funcs/to_csv.rs @@ -77,10 +77,6 @@ impl Display for ToCsv { } impl PhysicalExpr for ToCsv { - fn as_any(&self) -> &dyn Any { - self - } - fn data_type(&self, _: &Schema) -> Result { Ok(DataType::Utf8) } diff --git a/native/spark-expr/src/datetime_funcs/timestamp_trunc.rs b/native/spark-expr/src/datetime_funcs/timestamp_trunc.rs index 2d7a571b76..9ba281cccc 100644 --- a/native/spark-expr/src/datetime_funcs/timestamp_trunc.rs +++ b/native/spark-expr/src/datetime_funcs/timestamp_trunc.rs @@ -85,10 +85,6 @@ impl Display for TimestampTruncExpr { } impl PhysicalExpr for TimestampTruncExpr { - fn as_any(&self) -> &dyn Any { - self - } - fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { Display::fmt(self, f) } diff --git a/native/spark-expr/src/json_funcs/from_json.rs b/native/spark-expr/src/json_funcs/from_json.rs index 685ea3c8ec..eaca6db016 100644 --- a/native/spark-expr/src/json_funcs/from_json.rs +++ b/native/spark-expr/src/json_funcs/from_json.rs @@ -86,10 +86,6 @@ impl PartialEq for FromJson { } impl PhysicalExpr for FromJson { - fn as_any(&self) -> &dyn Any { - self - } - fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { Display::fmt(self, f) } diff --git a/native/spark-expr/src/json_funcs/to_json.rs b/native/spark-expr/src/json_funcs/to_json.rs index 3cc827f210..6dcd8bc045 100644 --- a/native/spark-expr/src/json_funcs/to_json.rs +++ b/native/spark-expr/src/json_funcs/to_json.rs @@ -79,10 +79,6 @@ impl PartialEq for ToJson { } impl PhysicalExpr for ToJson { - fn as_any(&self) -> &dyn Any { - self - } - fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { Display::fmt(self, f) } diff --git a/native/spark-expr/src/math_funcs/internal/checkoverflow.rs b/native/spark-expr/src/math_funcs/internal/checkoverflow.rs index f1fb9c2f02..2249c9d75f 100644 --- a/native/spark-expr/src/math_funcs/internal/checkoverflow.rs +++ b/native/spark-expr/src/math_funcs/internal/checkoverflow.rs @@ -91,10 +91,6 @@ impl Display for CheckOverflow { } impl PhysicalExpr for CheckOverflow { - fn as_any(&self) -> &dyn Any { - self - } - fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { Display::fmt(self, f) } @@ -274,9 +270,6 @@ mod tests { } impl PhysicalExpr for ScalarChild { - fn as_any(&self) -> &dyn Any { - self - } fn data_type(&self, _: &Schema) -> datafusion::common::Result { Ok(DataType::Decimal128(self.1, self.2)) } diff --git a/native/spark-expr/src/math_funcs/internal/decimal_rescale_check.rs b/native/spark-expr/src/math_funcs/internal/decimal_rescale_check.rs index 1322404951..420c25b321 100644 --- a/native/spark-expr/src/math_funcs/internal/decimal_rescale_check.rs +++ b/native/spark-expr/src/math_funcs/internal/decimal_rescale_check.rs @@ -154,10 +154,6 @@ fn rescale_and_check( } impl PhysicalExpr for DecimalRescaleCheckOverflow { - fn as_any(&self) -> &dyn Any { - self - } - fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { Display::fmt(self, f) } @@ -392,9 +388,6 @@ mod tests { } impl PhysicalExpr for ScalarChild { - fn as_any(&self) -> &dyn Any { - self - } fn data_type(&self, _: &Schema) -> datafusion::common::Result { Ok(DataType::Decimal128(self.1, self.2)) } diff --git a/native/spark-expr/src/math_funcs/internal/normalize_nan.rs b/native/spark-expr/src/math_funcs/internal/normalize_nan.rs index b3838f64f4..84a9b4d0c6 100644 --- a/native/spark-expr/src/math_funcs/internal/normalize_nan.rs +++ b/native/spark-expr/src/math_funcs/internal/normalize_nan.rs @@ -57,10 +57,6 @@ impl NormalizeNaNAndZero { } impl PhysicalExpr for NormalizeNaNAndZero { - fn as_any(&self) -> &dyn Any { - self - } - fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { Display::fmt(self, f) } diff --git a/native/spark-expr/src/math_funcs/negative.rs b/native/spark-expr/src/math_funcs/negative.rs index a268894086..2395520733 100644 --- a/native/spark-expr/src/math_funcs/negative.rs +++ b/native/spark-expr/src/math_funcs/negative.rs @@ -97,9 +97,6 @@ impl std::fmt::Display for NegativeExpr { impl PhysicalExpr for NegativeExpr { /// Return a reference to Any that can be used for downcasting - fn as_any(&self) -> &dyn Any { - self - } fn data_type(&self, input_schema: &Schema) -> Result { self.arg.data_type(input_schema) diff --git a/native/spark-expr/src/math_funcs/wide_decimal_binary_expr.rs b/native/spark-expr/src/math_funcs/wide_decimal_binary_expr.rs index 644252b46b..18821d352c 100644 --- a/native/spark-expr/src/math_funcs/wide_decimal_binary_expr.rs +++ b/native/spark-expr/src/math_funcs/wide_decimal_binary_expr.rs @@ -165,10 +165,6 @@ fn max_for_precision(precision: u8) -> i256 { } impl PhysicalExpr for WideDecimalBinaryExpr { - fn as_any(&self) -> &dyn Any { - self - } - fn data_type(&self, _input_schema: &Schema) -> Result { Ok(DataType::Decimal128( self.output_precision, diff --git a/native/spark-expr/src/nondetermenistic_funcs/monotonically_increasing_id.rs b/native/spark-expr/src/nondetermenistic_funcs/monotonically_increasing_id.rs index 49a5066a38..1daf2e00d4 100644 --- a/native/spark-expr/src/nondetermenistic_funcs/monotonically_increasing_id.rs +++ b/native/spark-expr/src/nondetermenistic_funcs/monotonically_increasing_id.rs @@ -66,10 +66,6 @@ impl Hash for MonotonicallyIncreasingId { } impl PhysicalExpr for MonotonicallyIncreasingId { - fn as_any(&self) -> &dyn Any { - self - } - fn evaluate(&self, batch: &RecordBatch) -> Result { let start = self .current_offset diff --git a/native/spark-expr/src/nondetermenistic_funcs/rand.rs b/native/spark-expr/src/nondetermenistic_funcs/rand.rs index e23a83d84e..3f4272a6f0 100644 --- a/native/spark-expr/src/nondetermenistic_funcs/rand.rs +++ b/native/spark-expr/src/nondetermenistic_funcs/rand.rs @@ -120,10 +120,6 @@ impl Hash for RandExpr { } impl PhysicalExpr for RandExpr { - fn as_any(&self) -> &dyn Any { - self - } - fn data_type(&self, _input_schema: &Schema) -> Result { Ok(DataType::Float64) } diff --git a/native/spark-expr/src/nondetermenistic_funcs/randn.rs b/native/spark-expr/src/nondetermenistic_funcs/randn.rs index 40fafedc20..59376ae064 100644 --- a/native/spark-expr/src/nondetermenistic_funcs/randn.rs +++ b/native/spark-expr/src/nondetermenistic_funcs/randn.rs @@ -131,10 +131,6 @@ impl Hash for RandnExpr { } impl PhysicalExpr for RandnExpr { - fn as_any(&self) -> &dyn Any { - self - } - fn data_type(&self, _input_schema: &Schema) -> datafusion::common::Result { Ok(DataType::Float64) } diff --git a/native/spark-expr/src/predicate_funcs/rlike.rs b/native/spark-expr/src/predicate_funcs/rlike.rs index ed5970a6a2..c76414694b 100644 --- a/native/spark-expr/src/predicate_funcs/rlike.rs +++ b/native/spark-expr/src/predicate_funcs/rlike.rs @@ -102,10 +102,6 @@ impl Display for RLike { } impl PhysicalExpr for RLike { - fn as_any(&self) -> &dyn Any { - self - } - fn data_type(&self, _input_schema: &Schema) -> Result { Ok(DataType::Boolean) } diff --git a/native/spark-expr/src/string_funcs/substring.rs b/native/spark-expr/src/string_funcs/substring.rs index e6f11fc39a..4be306afca 100644 --- a/native/spark-expr/src/string_funcs/substring.rs +++ b/native/spark-expr/src/string_funcs/substring.rs @@ -68,10 +68,6 @@ impl Display for SubstringExpr { } impl PhysicalExpr for SubstringExpr { - fn as_any(&self) -> &dyn Any { - self - } - fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { Display::fmt(self, f) } diff --git a/native/spark-expr/src/struct_funcs/create_named_struct.rs b/native/spark-expr/src/struct_funcs/create_named_struct.rs index 70e03ad0c0..d610648916 100644 --- a/native/spark-expr/src/struct_funcs/create_named_struct.rs +++ b/native/spark-expr/src/struct_funcs/create_named_struct.rs @@ -53,10 +53,6 @@ impl CreateNamedStruct { } impl PhysicalExpr for CreateNamedStruct { - fn as_any(&self) -> &dyn Any { - self - } - fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { Display::fmt(self, f) } diff --git a/native/spark-expr/src/struct_funcs/get_struct_field.rs b/native/spark-expr/src/struct_funcs/get_struct_field.rs index 7929cea483..afafcec53b 100644 --- a/native/spark-expr/src/struct_funcs/get_struct_field.rs +++ b/native/spark-expr/src/struct_funcs/get_struct_field.rs @@ -62,10 +62,6 @@ impl GetStructField { } impl PhysicalExpr for GetStructField { - fn as_any(&self) -> &dyn Any { - self - } - fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { Display::fmt(self, f) } diff --git a/native/spark-expr/src/unbound.rs b/native/spark-expr/src/unbound.rs index cf0adafa91..919701dd1f 100644 --- a/native/spark-expr/src/unbound.rs +++ b/native/spark-expr/src/unbound.rs @@ -60,9 +60,6 @@ impl std::fmt::Display for UnboundColumn { impl PhysicalExpr for UnboundColumn { /// Return a reference to Any that can be used for downcasting - fn as_any(&self) -> &dyn std::any::Any { - self - } fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { Display::fmt(self, f) From 68f1a095b853208a5ee37d13fd2e3a78166a20fb Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 15 Apr 2026 21:26:55 -0400 Subject: [PATCH 10/12] Fix after merging in latest datafusion main in the feature branch. --- native/spark-expr/src/array_funcs/array_insert.rs | 1 - native/spark-expr/src/array_funcs/get_array_struct_fields.rs | 1 - native/spark-expr/src/array_funcs/list_extract.rs | 1 - native/spark-expr/src/conditional_funcs/if_expr.rs | 4 +--- native/spark-expr/src/conversion_funcs/cast.rs | 1 - native/spark-expr/src/csv_funcs/to_csv.rs | 1 - native/spark-expr/src/datetime_funcs/timestamp_trunc.rs | 1 - native/spark-expr/src/math_funcs/internal/checkoverflow.rs | 1 - .../src/math_funcs/internal/decimal_rescale_check.rs | 1 - native/spark-expr/src/math_funcs/internal/normalize_nan.rs | 1 - native/spark-expr/src/math_funcs/negative.rs | 4 +--- native/spark-expr/src/math_funcs/wide_decimal_binary_expr.rs | 2 +- .../src/nondetermenistic_funcs/monotonically_increasing_id.rs | 1 - native/spark-expr/src/nondetermenistic_funcs/rand.rs | 1 - native/spark-expr/src/nondetermenistic_funcs/randn.rs | 1 - native/spark-expr/src/predicate_funcs/rlike.rs | 1 - native/spark-expr/src/string_funcs/substring.rs | 1 - native/spark-expr/src/struct_funcs/create_named_struct.rs | 1 - native/spark-expr/src/struct_funcs/get_struct_field.rs | 1 - native/spark-expr/src/unbound.rs | 2 -- 20 files changed, 3 insertions(+), 25 deletions(-) diff --git a/native/spark-expr/src/array_funcs/array_insert.rs b/native/spark-expr/src/array_funcs/array_insert.rs index af094fbfd0..e638c440fd 100644 --- a/native/spark-expr/src/array_funcs/array_insert.rs +++ b/native/spark-expr/src/array_funcs/array_insert.rs @@ -30,7 +30,6 @@ use datafusion::logical_expr::ColumnarValue; use datafusion::physical_expr::PhysicalExpr; use std::hash::Hash; use std::{ - any::Any, fmt::{Debug, Display, Formatter}, sync::Arc, }; diff --git a/native/spark-expr/src/array_funcs/get_array_struct_fields.rs b/native/spark-expr/src/array_funcs/get_array_struct_fields.rs index dd6d340ae5..a873392499 100644 --- a/native/spark-expr/src/array_funcs/get_array_struct_fields.rs +++ b/native/spark-expr/src/array_funcs/get_array_struct_fields.rs @@ -27,7 +27,6 @@ use datafusion::logical_expr::ColumnarValue; use datafusion::physical_expr::PhysicalExpr; use std::hash::Hash; use std::{ - any::Any, fmt::{Debug, Display, Formatter}, sync::Arc, }; diff --git a/native/spark-expr/src/array_funcs/list_extract.rs b/native/spark-expr/src/array_funcs/list_extract.rs index 26533a75b8..83fa23f757 100644 --- a/native/spark-expr/src/array_funcs/list_extract.rs +++ b/native/spark-expr/src/array_funcs/list_extract.rs @@ -26,7 +26,6 @@ use datafusion::logical_expr::ColumnarValue; use datafusion::physical_expr::PhysicalExpr; use std::hash::Hash; use std::{ - any::Any, fmt::{Debug, Display, Formatter}, sync::Arc, }; diff --git a/native/spark-expr/src/conditional_funcs/if_expr.rs b/native/spark-expr/src/conditional_funcs/if_expr.rs index bedb515c2f..5e57c54a22 100644 --- a/native/spark-expr/src/conditional_funcs/if_expr.rs +++ b/native/spark-expr/src/conditional_funcs/if_expr.rs @@ -24,7 +24,7 @@ use datafusion::logical_expr::ColumnarValue; use datafusion::physical_expr::{expressions::CaseExpr, PhysicalExpr}; use std::fmt::{Display, Formatter}; use std::hash::Hash; -use std::{any::Any, sync::Arc}; +use std::sync::Arc; /// IfExpr is a wrapper around CaseExpr, because `IF(a, b, c)` is semantically equivalent to /// `CASE WHEN a THEN b ELSE c END`. @@ -83,8 +83,6 @@ impl IfExpr { } impl PhysicalExpr for IfExpr { - /// Return a reference to Any that can be used for down-casting - fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { Display::fmt(self, f) } diff --git a/native/spark-expr/src/conversion_funcs/cast.rs b/native/spark-expr/src/conversion_funcs/cast.rs index 1ba9f4a9e7..17c318387d 100644 --- a/native/spark-expr/src/conversion_funcs/cast.rs +++ b/native/spark-expr/src/conversion_funcs/cast.rs @@ -62,7 +62,6 @@ use datafusion::common::{internal_err, DataFusionError, Result as DataFusionResu use datafusion::physical_expr::PhysicalExpr; use datafusion::physical_plan::ColumnarValue; use std::{ - any::Any, fmt::{Debug, Display, Formatter}, hash::Hash, sync::Arc, diff --git a/native/spark-expr/src/csv_funcs/to_csv.rs b/native/spark-expr/src/csv_funcs/to_csv.rs index 2d49cde585..01fdc901cb 100644 --- a/native/spark-expr/src/csv_funcs/to_csv.rs +++ b/native/spark-expr/src/csv_funcs/to_csv.rs @@ -23,7 +23,6 @@ use arrow::datatypes::{DataType, Schema}; use datafusion::common::Result; use datafusion::logical_expr::ColumnarValue; use datafusion::physical_expr::PhysicalExpr; -use std::any::Any; use std::fmt::{Display, Formatter}; use std::hash::Hash; use std::sync::Arc; diff --git a/native/spark-expr/src/datetime_funcs/timestamp_trunc.rs b/native/spark-expr/src/datetime_funcs/timestamp_trunc.rs index 9ba281cccc..1e95eb5141 100644 --- a/native/spark-expr/src/datetime_funcs/timestamp_trunc.rs +++ b/native/spark-expr/src/datetime_funcs/timestamp_trunc.rs @@ -23,7 +23,6 @@ use datafusion::logical_expr::ColumnarValue; use datafusion::physical_expr::PhysicalExpr; use std::hash::Hash; use std::{ - any::Any, fmt::{Debug, Display, Formatter}, sync::Arc, }; diff --git a/native/spark-expr/src/math_funcs/internal/checkoverflow.rs b/native/spark-expr/src/math_funcs/internal/checkoverflow.rs index 2249c9d75f..f867fe0c4d 100644 --- a/native/spark-expr/src/math_funcs/internal/checkoverflow.rs +++ b/native/spark-expr/src/math_funcs/internal/checkoverflow.rs @@ -28,7 +28,6 @@ use std::hash::Hash; use crate::SparkError; use std::{ - any::Any, fmt::{Display, Formatter}, sync::Arc, }; diff --git a/native/spark-expr/src/math_funcs/internal/decimal_rescale_check.rs b/native/spark-expr/src/math_funcs/internal/decimal_rescale_check.rs index 420c25b321..667f76939a 100644 --- a/native/spark-expr/src/math_funcs/internal/decimal_rescale_check.rs +++ b/native/spark-expr/src/math_funcs/internal/decimal_rescale_check.rs @@ -29,7 +29,6 @@ use datafusion::logical_expr::ColumnarValue; use datafusion::physical_expr::PhysicalExpr; use std::hash::Hash; use std::{ - any::Any, fmt::{Display, Formatter}, sync::Arc, }; diff --git a/native/spark-expr/src/math_funcs/internal/normalize_nan.rs b/native/spark-expr/src/math_funcs/internal/normalize_nan.rs index 84a9b4d0c6..165ae3acc9 100644 --- a/native/spark-expr/src/math_funcs/internal/normalize_nan.rs +++ b/native/spark-expr/src/math_funcs/internal/normalize_nan.rs @@ -26,7 +26,6 @@ use datafusion::logical_expr::ColumnarValue; use datafusion::physical_expr::PhysicalExpr; use std::hash::Hash; use std::{ - any::Any, fmt::{Display, Formatter}, sync::Arc, }; diff --git a/native/spark-expr/src/math_funcs/negative.rs b/native/spark-expr/src/math_funcs/negative.rs index 2395520733..650fa401ef 100644 --- a/native/spark-expr/src/math_funcs/negative.rs +++ b/native/spark-expr/src/math_funcs/negative.rs @@ -29,7 +29,7 @@ use datafusion::{ }; use std::fmt::{Display, Formatter}; use std::hash::Hash; -use std::{any::Any, sync::Arc}; +use std::sync::Arc; pub fn create_negate_expr( expr: Arc, @@ -96,8 +96,6 @@ impl std::fmt::Display for NegativeExpr { } impl PhysicalExpr for NegativeExpr { - /// Return a reference to Any that can be used for downcasting - fn data_type(&self, input_schema: &Schema) -> Result { self.arg.data_type(input_schema) } diff --git a/native/spark-expr/src/math_funcs/wide_decimal_binary_expr.rs b/native/spark-expr/src/math_funcs/wide_decimal_binary_expr.rs index 18821d352c..d2479857af 100644 --- a/native/spark-expr/src/math_funcs/wide_decimal_binary_expr.rs +++ b/native/spark-expr/src/math_funcs/wide_decimal_binary_expr.rs @@ -31,7 +31,7 @@ use datafusion::logical_expr::ColumnarValue; use datafusion::physical_expr::PhysicalExpr; use std::fmt::{Display, Formatter}; use std::hash::Hash; -use std::{any::Any, sync::Arc}; +use std::sync::Arc; /// The arithmetic operation to perform. #[derive(Debug, Hash, PartialEq, Eq, Clone, Copy)] diff --git a/native/spark-expr/src/nondetermenistic_funcs/monotonically_increasing_id.rs b/native/spark-expr/src/nondetermenistic_funcs/monotonically_increasing_id.rs index 1daf2e00d4..e7404e00c1 100644 --- a/native/spark-expr/src/nondetermenistic_funcs/monotonically_increasing_id.rs +++ b/native/spark-expr/src/nondetermenistic_funcs/monotonically_increasing_id.rs @@ -20,7 +20,6 @@ use arrow::datatypes::{DataType, Schema}; use datafusion::common::Result; use datafusion::logical_expr::ColumnarValue; use datafusion::physical_expr::PhysicalExpr; -use std::any::Any; use std::fmt::{Debug, Display, Formatter}; use std::hash::{Hash, Hasher}; use std::sync::atomic::{AtomicI64, Ordering}; diff --git a/native/spark-expr/src/nondetermenistic_funcs/rand.rs b/native/spark-expr/src/nondetermenistic_funcs/rand.rs index 3f4272a6f0..8f07f37344 100644 --- a/native/spark-expr/src/nondetermenistic_funcs/rand.rs +++ b/native/spark-expr/src/nondetermenistic_funcs/rand.rs @@ -23,7 +23,6 @@ use arrow::datatypes::{DataType, Schema}; use datafusion::common::Result; use datafusion::logical_expr::ColumnarValue; use datafusion::physical_expr::PhysicalExpr; -use std::any::Any; use std::fmt::{Display, Formatter}; use std::hash::{Hash, Hasher}; use std::sync::{Arc, Mutex}; diff --git a/native/spark-expr/src/nondetermenistic_funcs/randn.rs b/native/spark-expr/src/nondetermenistic_funcs/randn.rs index 59376ae064..7d50d24811 100644 --- a/native/spark-expr/src/nondetermenistic_funcs/randn.rs +++ b/native/spark-expr/src/nondetermenistic_funcs/randn.rs @@ -22,7 +22,6 @@ use arrow::array::RecordBatch; use arrow::datatypes::{DataType, Schema}; use datafusion::logical_expr::ColumnarValue; use datafusion::physical_expr::PhysicalExpr; -use std::any::Any; use std::fmt::{Display, Formatter}; use std::hash::{Hash, Hasher}; use std::sync::{Arc, Mutex}; diff --git a/native/spark-expr/src/predicate_funcs/rlike.rs b/native/spark-expr/src/predicate_funcs/rlike.rs index c76414694b..ee005dd1ac 100644 --- a/native/spark-expr/src/predicate_funcs/rlike.rs +++ b/native/spark-expr/src/predicate_funcs/rlike.rs @@ -25,7 +25,6 @@ use datafusion::common::{internal_err, Result, ScalarValue}; use datafusion::physical_expr::PhysicalExpr; use datafusion::physical_plan::ColumnarValue; use regex::Regex; -use std::any::Any; use std::fmt::{Display, Formatter}; use std::hash::{Hash, Hasher}; use std::sync::Arc; diff --git a/native/spark-expr/src/string_funcs/substring.rs b/native/spark-expr/src/string_funcs/substring.rs index 4be306afca..e8c3f12229 100644 --- a/native/spark-expr/src/string_funcs/substring.rs +++ b/native/spark-expr/src/string_funcs/substring.rs @@ -24,7 +24,6 @@ use datafusion::common::DataFusionError; use datafusion::logical_expr::ColumnarValue; use datafusion::physical_expr::PhysicalExpr; use std::{ - any::Any, fmt::{Display, Formatter}, hash::Hash, sync::Arc, diff --git a/native/spark-expr/src/struct_funcs/create_named_struct.rs b/native/spark-expr/src/struct_funcs/create_named_struct.rs index d610648916..1a63cb1cc9 100644 --- a/native/spark-expr/src/struct_funcs/create_named_struct.rs +++ b/native/spark-expr/src/struct_funcs/create_named_struct.rs @@ -22,7 +22,6 @@ use datafusion::common::Result as DataFusionResult; use datafusion::logical_expr::ColumnarValue; use datafusion::physical_expr::PhysicalExpr; use std::{ - any::Any, fmt::{Display, Formatter}, hash::Hash, sync::Arc, diff --git a/native/spark-expr/src/struct_funcs/get_struct_field.rs b/native/spark-expr/src/struct_funcs/get_struct_field.rs index afafcec53b..6684967c3a 100644 --- a/native/spark-expr/src/struct_funcs/get_struct_field.rs +++ b/native/spark-expr/src/struct_funcs/get_struct_field.rs @@ -22,7 +22,6 @@ use datafusion::common::{DataFusionError, Result as DataFusionResult, ScalarValu use datafusion::logical_expr::ColumnarValue; use datafusion::physical_expr::PhysicalExpr; use std::{ - any::Any, fmt::{Display, Formatter}, hash::Hash, sync::Arc, diff --git a/native/spark-expr/src/unbound.rs b/native/spark-expr/src/unbound.rs index 919701dd1f..69187dbc02 100644 --- a/native/spark-expr/src/unbound.rs +++ b/native/spark-expr/src/unbound.rs @@ -59,8 +59,6 @@ impl std::fmt::Display for UnboundColumn { } impl PhysicalExpr for UnboundColumn { - /// Return a reference to Any that can be used for downcasting - fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { Display::fmt(self, f) } From ab809d1567f0804c38ce1ff25d2a3db010b77a8b Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 15 Apr 2026 21:33:58 -0400 Subject: [PATCH 11/12] Fix after merging in latest datafusion main in the feature branch. --- .../src/execution/expressions/arithmetic.rs | 4 --- .../core/src/execution/expressions/strings.rs | 2 +- .../src/execution/expressions/subquery.rs | 5 --- native/core/src/execution/planner.rs | 33 ++++++------------- native/core/src/parquet/cast_column.rs | 5 --- native/core/src/parquet/schema_adapter.rs | 12 +++---- 6 files changed, 16 insertions(+), 45 deletions(-) diff --git a/native/core/src/execution/expressions/arithmetic.rs b/native/core/src/execution/expressions/arithmetic.rs index 320532d773..8d4c59a010 100644 --- a/native/core/src/execution/expressions/arithmetic.rs +++ b/native/core/src/execution/expressions/arithmetic.rs @@ -77,10 +77,6 @@ impl Hash for CheckedBinaryExpr { } impl PhysicalExpr for CheckedBinaryExpr { - fn as_any(&self) -> &dyn Any { - self - } - fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { self.child.fmt_sql(f) } diff --git a/native/core/src/execution/expressions/strings.rs b/native/core/src/execution/expressions/strings.rs index 7219395963..4a7c44cc3e 100644 --- a/native/core/src/execution/expressions/strings.rs +++ b/native/core/src/execution/expressions/strings.rs @@ -91,7 +91,7 @@ impl ExpressionBuilder for RlikeBuilder { let left = planner.create_expr(expr.left.as_ref().unwrap(), Arc::clone(&input_schema))?; let right = planner.create_expr(expr.right.as_ref().unwrap(), input_schema)?; - match right.as_any().downcast_ref::().unwrap().value() { + match right.downcast_ref::().unwrap().value() { ScalarValue::Utf8(Some(pattern)) => Ok(Arc::new(RLike::try_new(left, pattern)?)), _ => Err(ExecutionError::GeneralError( "RLike only supports scalar patterns".to_string(), diff --git a/native/core/src/execution/expressions/subquery.rs b/native/core/src/execution/expressions/subquery.rs index 9272ede60c..fc7b8104d2 100644 --- a/native/core/src/execution/expressions/subquery.rs +++ b/native/core/src/execution/expressions/subquery.rs @@ -29,7 +29,6 @@ use jni::{ sys::{jboolean, jbyte, jint, jlong, jshort}, }; use std::{ - any::Any, fmt::{Display, Formatter}, hash::Hash, sync::Arc, @@ -63,10 +62,6 @@ impl Display for Subquery { } impl PhysicalExpr for Subquery { - fn as_any(&self) -> &dyn Any { - self - } - fn fmt_sql(&self, f: &mut Formatter<'_>) -> std::fmt::Result { Display::fmt(self, f) } diff --git a/native/core/src/execution/planner.rs b/native/core/src/execution/planner.rs index 7275470db1..9f70db7d70 100644 --- a/native/core/src/execution/planner.rs +++ b/native/core/src/execution/planner.rs @@ -233,7 +233,6 @@ impl PhysicalPlanner { let literal = self.create_expr(partition_value, Arc::::clone(&empty_schema))?; literal - .as_any() .downcast_ref::() .ok_or_else(|| { GeneralError("Expected literal of partition value".to_string()) @@ -424,11 +423,7 @@ impl PhysicalPlanner { // WideDecimalBinaryExpr already handles overflow — skip redundant check // but only if its output type matches CheckOverflow's declared type - if child - .as_any() - .downcast_ref::() - .is_some() - { + if child.downcast_ref::().is_some() { let child_type = child.data_type(&input_schema)?; if child_type == data_type { return Ok(child); @@ -437,7 +432,7 @@ impl PhysicalPlanner { // Fuse Cast(Decimal128→Decimal128) + CheckOverflow into single rescale+check // Only fuse when the Cast target type matches the CheckOverflow output type - if let Some(cast) = child.as_any().downcast_ref::() { + if let Some(cast) = child.downcast_ref::() { if let ( DataType::Decimal128(p_out, s_out), Ok(DataType::Decimal128(_p_in, s_in)), @@ -1166,12 +1161,10 @@ impl PhysicalPlanner { .iter() .map(|expr| { let literal = self.create_expr(expr, Arc::clone(&required_schema))?; - let df_literal = literal - .as_any() - .downcast_ref::() - .ok_or_else(|| { - GeneralError("Expected literal of default value.".to_string()) - })?; + let df_literal = + literal.downcast_ref::().ok_or_else(|| { + GeneralError("Expected literal of default value.".to_string()) + })?; Ok(df_literal.value().clone()) }) .collect(); @@ -2552,8 +2545,7 @@ impl PhysicalPlanner { &boundary_row.partition_bounds[col_idx], Arc::clone(&input_schema), )?; - let literal_expr = - expr.as_any().downcast_ref::().expect("Literal"); + let literal_expr = expr.downcast_ref::().expect("Literal"); col_values.push(literal_expr.value().clone()); } } @@ -2663,12 +2655,7 @@ impl PhysicalPlanner { // TODO this should try and find scalar let arguments = args .iter() - .map(|e| { - e.as_ref() - .as_any() - .downcast_ref::() - .map(|lit| lit.value()) - }) + .map(|e| e.as_ref().downcast_ref::().map(|lit| lit.value())) .collect::>(); let args = ReturnFieldArgs { @@ -2772,7 +2759,7 @@ fn expr_to_columns( expr.apply(&mut |expr: &Arc| { Ok({ - if let Some(column) = expr.as_any().downcast_ref::() { + if let Some(column) = expr.downcast_ref::() { if column.index() > left_field_len + right_field_len { return Err(DataFusionError::Internal(format!( "Column index {} out of range", @@ -2823,7 +2810,7 @@ impl TreeNodeRewriter for JoinFilterRewriter<'_> { type Node = Arc; fn f_down(&mut self, node: Self::Node) -> datafusion::common::Result> { - if let Some(column) = node.as_any().downcast_ref::() { + if let Some(column) = node.downcast_ref::() { if column.index() < self.left_field_len { // left side let new_index = self diff --git a/native/core/src/parquet/cast_column.rs b/native/core/src/parquet/cast_column.rs index 67558b5266..1cc928d1d5 100644 --- a/native/core/src/parquet/cast_column.rs +++ b/native/core/src/parquet/cast_column.rs @@ -31,7 +31,6 @@ use datafusion::common::ScalarValue; use datafusion::logical_expr::ColumnarValue; use datafusion::physical_expr::PhysicalExpr; use std::{ - any::Any, fmt::{self, Display}, hash::Hash, sync::Arc, @@ -250,10 +249,6 @@ impl Display for CometCastColumnExpr { } impl PhysicalExpr for CometCastColumnExpr { - fn as_any(&self) -> &dyn Any { - self - } - fn data_type(&self, _input_schema: &Schema) -> DataFusionResult { Ok(self.target_field.data_type().clone()) } diff --git a/native/core/src/parquet/schema_adapter.rs b/native/core/src/parquet/schema_adapter.rs index 593a0c063a..1703f05807 100644 --- a/native/core/src/parquet/schema_adapter.rs +++ b/native/core/src/parquet/schema_adapter.rs @@ -216,7 +216,7 @@ impl PhysicalExprAdapter for SparkPhysicalExprAdapter { // Walk the expression tree to find Column references let mut duplicate_err: Option = None; let _ = Arc::::clone(&expr).transform(|e| { - if let Some(col) = e.as_any().downcast_ref::() { + if let Some(col) = e.downcast_ref::() { if let Some((req, matched)) = check_column_duplicate(col.name(), orig_physical) { duplicate_err = Some(DataFusionError::External(Box::new( @@ -266,7 +266,7 @@ impl PhysicalExprAdapter for SparkPhysicalExprAdapter { // the actual parquet stream schema, which uses the original physical names. let expr = if let Some(name_map) = &self.logical_to_physical_names { expr.transform(|e| { - if let Some(col) = e.as_any().downcast_ref::() { + if let Some(col) = e.downcast_ref::() { if let Some(physical_name) = name_map.get(col.name()) { return Ok(Transformed::yes(Arc::new(Column::new( physical_name, @@ -295,7 +295,7 @@ impl SparkPhysicalExprAdapter { expr: Arc, ) -> DataFusionResult> { expr.transform(|e| { - if let Some(column) = e.as_any().downcast_ref::() { + if let Some(column) = e.downcast_ref::() { let col_name = column.name(); // Resolve fields by name because this is the fallback path @@ -376,15 +376,13 @@ impl SparkPhysicalExprAdapter { expr: Arc, ) -> DataFusionResult>> { // Check for CastExpr and replace with spark_expr::Cast - if let Some(cast) = expr - .as_any() - .downcast_ref::() + if let Some(cast) = expr.downcast_ref::() { let child = Arc::clone(cast.expr()); let target_type = cast.target_field().data_type(); // Derive input field from the child Column expression and the physical schema - let input_field = if let Some(col) = child.as_any().downcast_ref::() { + let input_field = if let Some(col) = child.downcast_ref::() { Arc::new(self.physical_file_schema.field(col.index()).clone()) } else { // Fallback: synthesize a field from the target field name and child data type From f716fc368e52e5d4f55304c7858b9a2cf7a6f568 Mon Sep 17 00:00:00 2001 From: Matt Butrovich Date: Wed, 15 Apr 2026 21:53:33 -0400 Subject: [PATCH 12/12] Fix after merging in latest datafusion main in the feature branch. --- native/Cargo.lock | 490 ++++++++++++----------------------------- native/core/Cargo.toml | 2 +- 2 files changed, 139 insertions(+), 353 deletions(-) diff --git a/native/Cargo.lock b/native/Cargo.lock index 08e4493ca2..7a75e11487 100644 --- a/native/Cargo.lock +++ b/native/Cargo.lock @@ -1840,25 +1840,25 @@ dependencies = [ "chrono", "datafusion-catalog", "datafusion-catalog-listing", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", "datafusion-datasource-arrow", "datafusion-datasource-csv", "datafusion-datasource-json", "datafusion-datasource-parquet", - "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-functions 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-functions-aggregate 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-functions-nested 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", "datafusion-functions-table", "datafusion-functions-window", "datafusion-optimizer", - "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr", "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr-common", "datafusion-physical-optimizer", "datafusion-physical-plan", "datafusion-session", @@ -1884,12 +1884,12 @@ dependencies = [ "arrow", "async-trait", "dashmap", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-plan", "datafusion-session", "futures", @@ -1908,13 +1908,13 @@ dependencies = [ "arrow", "async-trait", "datafusion-catalog", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", "datafusion-datasource", - "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr-common", "datafusion-physical-plan", "futures", "itertools 0.14.0", @@ -1941,7 +1941,7 @@ dependencies = [ "datafusion-comet-shuffle", "datafusion-comet-spark-expr", "datafusion-datasource", - "datafusion-functions-nested 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", + "datafusion-functions-nested", "datafusion-physical-expr-adapter", "datafusion-spark", "futures", @@ -2093,27 +2093,6 @@ dependencies = [ "twox-hash", ] -[[package]] -name = "datafusion-common" -version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" -dependencies = [ - "arrow", - "arrow-ipc", - "arrow-schema", - "chrono", - "foldhash 0.2.0", - "half", - "hashbrown 0.17.0", - "indexmap 2.14.0", - "itertools 0.14.0", - "libc", - "log", - "tokio", - "uuid", - "web-time", -] - [[package]] name = "datafusion-common" version = "53.0.0" @@ -2160,13 +2139,13 @@ dependencies = [ "bytes", "bzip2", "chrono", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", "datafusion-common-runtime", - "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr", "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", "flate2", @@ -2192,12 +2171,12 @@ dependencies = [ "arrow-ipc", "async-trait", "bytes", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", "futures", @@ -2214,12 +2193,12 @@ dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", "futures", @@ -2236,12 +2215,12 @@ dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-execution", + "datafusion-expr", + "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-session", "futures", @@ -2258,16 +2237,16 @@ dependencies = [ "arrow", "async-trait", "bytes", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", "datafusion-common-runtime", "datafusion-datasource", - "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-functions 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr", "datafusion-physical-expr-adapter", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", "datafusion-session", @@ -2280,37 +2259,11 @@ dependencies = [ "tokio", ] -[[package]] -name = "datafusion-doc" -version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" - [[package]] name = "datafusion-doc" version = "53.0.0" source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" -[[package]] -name = "datafusion-execution" -version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" -dependencies = [ - "arrow", - "arrow-buffer", - "async-trait", - "dashmap", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "futures", - "log", - "object_store", - "parking_lot", - "rand 0.9.4", - "tempfile", - "url", -] - [[package]] name = "datafusion-execution" version = "53.0.0" @@ -2320,9 +2273,9 @@ dependencies = [ "arrow-buffer", "async-trait", "dashmap", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", + "datafusion-expr", + "datafusion-physical-expr-common", "futures", "log", "object_store", @@ -2333,27 +2286,6 @@ dependencies = [ "url", ] -[[package]] -name = "datafusion-expr" -version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" -dependencies = [ - "arrow", - "arrow-schema", - "async-trait", - "chrono", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-doc 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-functions-window-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "indexmap 2.14.0", - "itertools 0.14.0", - "serde_json", - "sqlparser", -] - [[package]] name = "datafusion-expr" version = "53.0.0" @@ -2363,66 +2295,29 @@ dependencies = [ "arrow-schema", "async-trait", "chrono", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-doc 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-functions-window-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", + "datafusion-doc", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr-common", "indexmap 2.14.0", "itertools 0.14.0", "serde_json", "sqlparser", ] -[[package]] -name = "datafusion-expr-common" -version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" -dependencies = [ - "arrow", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "indexmap 2.14.0", - "itertools 0.14.0", -] - [[package]] name = "datafusion-expr-common" version = "53.0.0" source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", "indexmap 2.14.0", "itertools 0.14.0", ] -[[package]] -name = "datafusion-functions" -version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" -dependencies = [ - "arrow", - "arrow-buffer", - "base64", - "chrono", - "chrono-tz", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-doc 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-macros 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "hex", - "itertools 0.14.0", - "log", - "memchr", - "num-traits", - "rand 0.9.4", - "regex", - "uuid", -] - [[package]] name = "datafusion-functions" version = "53.0.0" @@ -2435,12 +2330,12 @@ dependencies = [ "blake3", "chrono", "chrono-tz", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-doc 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-macros 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-macros", "hex", "itertools 0.14.0", "log", @@ -2453,90 +2348,35 @@ dependencies = [ "uuid", ] -[[package]] -name = "datafusion-functions-aggregate" -version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" -dependencies = [ - "arrow", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-doc 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-macros 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "foldhash 0.2.0", - "half", - "log", - "num-traits", -] - [[package]] name = "datafusion-functions-aggregate" version = "53.0.0" source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-doc 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-macros 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "foldhash 0.2.0", "half", "log", "num-traits", ] -[[package]] -name = "datafusion-functions-aggregate-common" -version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" -dependencies = [ - "arrow", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", -] - [[package]] name = "datafusion-functions-aggregate-common" version = "53.0.0" source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", -] - -[[package]] -name = "datafusion-functions-nested" -version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" -dependencies = [ - "arrow", - "arrow-ord", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-doc 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-functions 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-functions-aggregate 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-macros 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "hashbrown 0.17.0", - "itertools 0.14.0", - "itoa", - "log", - "memchr", + "datafusion-common", + "datafusion-expr-common", + "datafusion-physical-expr-common", ] [[package]] @@ -2546,16 +2386,16 @@ source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c9 dependencies = [ "arrow", "arrow-ord", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-doc 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-functions 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-functions-aggregate 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-macros 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", + "datafusion-doc", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-aggregate-common", + "datafusion-macros", + "datafusion-physical-expr-common", "hashbrown 0.17.0", "itertools 0.14.0", "itoa", @@ -2571,8 +2411,8 @@ dependencies = [ "arrow", "async-trait", "datafusion-catalog", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", + "datafusion-expr", "datafusion-physical-plan", "parking_lot", ] @@ -2583,42 +2423,23 @@ version = "53.0.0" source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-doc 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-functions-window-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-macros 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", + "datafusion-doc", + "datafusion-expr", + "datafusion-functions-window-common", + "datafusion-macros", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "log", ] -[[package]] -name = "datafusion-functions-window-common" -version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" -dependencies = [ - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", -] - [[package]] name = "datafusion-functions-window-common" version = "53.0.0" source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", -] - -[[package]] -name = "datafusion-macros" -version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" -dependencies = [ - "datafusion-doc 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "quote", - "syn 2.0.117", + "datafusion-common", + "datafusion-physical-expr-common", ] [[package]] @@ -2626,7 +2447,7 @@ name = "datafusion-macros" version = "53.0.0" source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ - "datafusion-doc 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-doc", "quote", "syn 2.0.117", ] @@ -2638,10 +2459,10 @@ source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c9 dependencies = [ "arrow", "chrono", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", "indexmap 2.14.0", "itertools 0.14.0", "log", @@ -2649,37 +2470,17 @@ dependencies = [ "regex-syntax", ] -[[package]] -name = "datafusion-physical-expr" -version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" -dependencies = [ - "arrow", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "half", - "hashbrown 0.17.0", - "indexmap 2.14.0", - "itertools 0.14.0", - "parking_lot", - "petgraph", - "tokio", -] - [[package]] name = "datafusion-physical-expr" version = "53.0.0" source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-functions-aggregate-common", + "datafusion-physical-expr-common", "half", "hashbrown 0.17.0", "indexmap 2.14.0", @@ -2695,29 +2496,14 @@ version = "53.0.0" source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-functions 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", + "datafusion-expr", + "datafusion-functions", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "itertools 0.14.0", ] -[[package]] -name = "datafusion-physical-expr-common" -version = "53.0.0" -source = "git+https://github.com/mbutrovich/datafusion.git?rev=0198302544#01983025442464d834808a036564f709af58aaff" -dependencies = [ - "arrow", - "chrono", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=0198302544)", - "hashbrown 0.17.0", - "indexmap 2.14.0", - "itertools 0.14.0", - "parking_lot", -] - [[package]] name = "datafusion-physical-expr-common" version = "53.0.0" @@ -2725,8 +2511,8 @@ source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c9 dependencies = [ "arrow", "chrono", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", + "datafusion-expr-common", "hashbrown 0.17.0", "indexmap 2.14.0", "itertools 0.14.0", @@ -2739,12 +2525,12 @@ version = "53.0.0" source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "datafusion-physical-plan", "datafusion-pruning", "itertools 0.14.0", @@ -2759,15 +2545,15 @@ dependencies = [ "arrow-ord", "arrow-schema", "async-trait", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", "datafusion-common-runtime", - "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-functions 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-functions-aggregate-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-functions-window-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate-common", + "datafusion-functions-window-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "futures", "half", "hashbrown 0.17.0", @@ -2786,11 +2572,11 @@ version = "53.0.0" source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "arrow", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", "datafusion-datasource", - "datafusion-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-physical-expr-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-expr-common", + "datafusion-physical-expr", + "datafusion-physical-expr-common", "datafusion-physical-plan", "log", ] @@ -2801,9 +2587,9 @@ version = "53.0.0" source = "git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998#9598c96998eda71a3e4e0ba2e055c418ac4575f7" dependencies = [ "async-trait", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", "datafusion-physical-plan", "parking_lot", ] @@ -2819,12 +2605,12 @@ dependencies = [ "crc32fast", "datafusion", "datafusion-catalog", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-execution 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-functions 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-functions-aggregate 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-functions-nested 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", + "datafusion-execution", + "datafusion-expr", + "datafusion-functions", + "datafusion-functions-aggregate", + "datafusion-functions-nested", "log", "num-traits", "percent-encoding", @@ -2843,9 +2629,9 @@ dependencies = [ "arrow", "bigdecimal", "chrono", - "datafusion-common 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-expr 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", - "datafusion-functions-nested 53.0.0 (git+https://github.com/mbutrovich/datafusion.git?rev=9598c96998)", + "datafusion-common", + "datafusion-expr", + "datafusion-functions-nested", "indexmap 2.14.0", "log", "regex", diff --git a/native/core/Cargo.toml b/native/core/Cargo.toml index 2bfb720379..ed4b11e8b3 100644 --- a/native/core/Cargo.toml +++ b/native/core/Cargo.toml @@ -91,7 +91,7 @@ jni = { version = "0.22.4", features = ["invocation"] } lazy_static = "1.4" assertables = "9" hex = "0.4.3" -datafusion-functions-nested = { git = "https://github.com/mbutrovich/datafusion.git", rev = "0198302544" } +datafusion-functions-nested = { git = "https://github.com/mbutrovich/datafusion.git", rev = "9598c96998" } [features] backtrace = ["datafusion/backtrace"]