diff --git a/Cargo.lock b/Cargo.lock index d1214fc9dc6e4..92996baaf2fac 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -165,7 +165,7 @@ checksum = "7c02d123df017efcdfbd739ef81735b36c5ba83ec3c59c80a9d7ecc718f92e50" [[package]] name = "arrow" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=5d211125b8e8a5784f2caf606cdf88684dec30a5#5d211125b8e8a5784f2caf606cdf88684dec30a5" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=25a9c23d7b9706238b00b1288d27f036c186cd8e#25a9c23d7b9706238b00b1288d27f036c186cd8e" dependencies = [ "arrow-arith", "arrow-array", @@ -187,7 +187,7 @@ dependencies = [ [[package]] name = "arrow-arith" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=5d211125b8e8a5784f2caf606cdf88684dec30a5#5d211125b8e8a5784f2caf606cdf88684dec30a5" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=25a9c23d7b9706238b00b1288d27f036c186cd8e#25a9c23d7b9706238b00b1288d27f036c186cd8e" dependencies = [ "arrow-array", "arrow-buffer", @@ -200,7 +200,7 @@ dependencies = [ [[package]] name = "arrow-array" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=5d211125b8e8a5784f2caf606cdf88684dec30a5#5d211125b8e8a5784f2caf606cdf88684dec30a5" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=25a9c23d7b9706238b00b1288d27f036c186cd8e#25a9c23d7b9706238b00b1288d27f036c186cd8e" dependencies = [ "ahash", "arrow-buffer", @@ -219,7 +219,7 @@ dependencies = [ [[package]] name = "arrow-avro" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=5d211125b8e8a5784f2caf606cdf88684dec30a5#5d211125b8e8a5784f2caf606cdf88684dec30a5" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=25a9c23d7b9706238b00b1288d27f036c186cd8e#25a9c23d7b9706238b00b1288d27f036c186cd8e" dependencies = [ "arrow-array", "arrow-buffer", @@ -242,7 +242,7 @@ dependencies = [ [[package]] name = "arrow-buffer" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=5d211125b8e8a5784f2caf606cdf88684dec30a5#5d211125b8e8a5784f2caf606cdf88684dec30a5" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=25a9c23d7b9706238b00b1288d27f036c186cd8e#25a9c23d7b9706238b00b1288d27f036c186cd8e" dependencies = [ "bytes", "half", @@ -253,7 +253,7 @@ dependencies = [ [[package]] name = "arrow-cast" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=5d211125b8e8a5784f2caf606cdf88684dec30a5#5d211125b8e8a5784f2caf606cdf88684dec30a5" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=25a9c23d7b9706238b00b1288d27f036c186cd8e#25a9c23d7b9706238b00b1288d27f036c186cd8e" dependencies = [ "arrow-array", "arrow-buffer", @@ -274,7 +274,7 @@ dependencies = [ [[package]] name = "arrow-csv" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=5d211125b8e8a5784f2caf606cdf88684dec30a5#5d211125b8e8a5784f2caf606cdf88684dec30a5" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=25a9c23d7b9706238b00b1288d27f036c186cd8e#25a9c23d7b9706238b00b1288d27f036c186cd8e" dependencies = [ "arrow-array", "arrow-cast", @@ -288,7 +288,7 @@ dependencies = [ [[package]] name = "arrow-data" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=5d211125b8e8a5784f2caf606cdf88684dec30a5#5d211125b8e8a5784f2caf606cdf88684dec30a5" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=25a9c23d7b9706238b00b1288d27f036c186cd8e#25a9c23d7b9706238b00b1288d27f036c186cd8e" dependencies = [ "arrow-buffer", "arrow-schema", @@ -300,7 +300,7 @@ dependencies = [ [[package]] name = "arrow-flight" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=5d211125b8e8a5784f2caf606cdf88684dec30a5#5d211125b8e8a5784f2caf606cdf88684dec30a5" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=25a9c23d7b9706238b00b1288d27f036c186cd8e#25a9c23d7b9706238b00b1288d27f036c186cd8e" dependencies = [ "arrow-arith", "arrow-array", @@ -326,7 +326,7 @@ dependencies = [ [[package]] name = "arrow-ipc" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=5d211125b8e8a5784f2caf606cdf88684dec30a5#5d211125b8e8a5784f2caf606cdf88684dec30a5" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=25a9c23d7b9706238b00b1288d27f036c186cd8e#25a9c23d7b9706238b00b1288d27f036c186cd8e" dependencies = [ "arrow-array", "arrow-buffer", @@ -341,7 +341,7 @@ dependencies = [ [[package]] name = "arrow-json" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=5d211125b8e8a5784f2caf606cdf88684dec30a5#5d211125b8e8a5784f2caf606cdf88684dec30a5" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=25a9c23d7b9706238b00b1288d27f036c186cd8e#25a9c23d7b9706238b00b1288d27f036c186cd8e" dependencies = [ "arrow-array", "arrow-buffer", @@ -365,7 +365,7 @@ dependencies = [ [[package]] name = "arrow-ord" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=5d211125b8e8a5784f2caf606cdf88684dec30a5#5d211125b8e8a5784f2caf606cdf88684dec30a5" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=25a9c23d7b9706238b00b1288d27f036c186cd8e#25a9c23d7b9706238b00b1288d27f036c186cd8e" dependencies = [ "arrow-array", "arrow-buffer", @@ -377,7 +377,7 @@ dependencies = [ [[package]] name = "arrow-row" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=5d211125b8e8a5784f2caf606cdf88684dec30a5#5d211125b8e8a5784f2caf606cdf88684dec30a5" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=25a9c23d7b9706238b00b1288d27f036c186cd8e#25a9c23d7b9706238b00b1288d27f036c186cd8e" dependencies = [ "arrow-array", "arrow-buffer", @@ -389,7 +389,7 @@ dependencies = [ [[package]] name = "arrow-schema" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=5d211125b8e8a5784f2caf606cdf88684dec30a5#5d211125b8e8a5784f2caf606cdf88684dec30a5" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=25a9c23d7b9706238b00b1288d27f036c186cd8e#25a9c23d7b9706238b00b1288d27f036c186cd8e" dependencies = [ "bitflags", "serde", @@ -400,7 +400,7 @@ dependencies = [ [[package]] name = "arrow-select" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=5d211125b8e8a5784f2caf606cdf88684dec30a5#5d211125b8e8a5784f2caf606cdf88684dec30a5" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=25a9c23d7b9706238b00b1288d27f036c186cd8e#25a9c23d7b9706238b00b1288d27f036c186cd8e" dependencies = [ "ahash", "arrow-array", @@ -413,7 +413,7 @@ dependencies = [ [[package]] name = "arrow-string" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=5d211125b8e8a5784f2caf606cdf88684dec30a5#5d211125b8e8a5784f2caf606cdf88684dec30a5" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=25a9c23d7b9706238b00b1288d27f036c186cd8e#25a9c23d7b9706238b00b1288d27f036c186cd8e" dependencies = [ "arrow-array", "arrow-buffer", @@ -4484,7 +4484,7 @@ dependencies = [ [[package]] name = "parquet" version = "59.2.0" -source = "git+https://github.com/Embucket/arrow-rs.git?rev=5d211125b8e8a5784f2caf606cdf88684dec30a5#5d211125b8e8a5784f2caf606cdf88684dec30a5" +source = "git+https://github.com/Embucket/arrow-rs.git?rev=25a9c23d7b9706238b00b1288d27f036c186cd8e#25a9c23d7b9706238b00b1288d27f036c186cd8e" dependencies = [ "ahash", "arrow-array", diff --git a/Cargo.toml b/Cargo.toml index 6da9752f918a1..356f65770b09a 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -89,30 +89,30 @@ version = "55.1.0" # # See for more details: https://github.com/rust-lang/cargo/issues/11329 apache-avro = { version = "0.21", default-features = false } -arrow = { git = "https://github.com/Embucket/arrow-rs.git", rev = "5d211125b8e8a5784f2caf606cdf88684dec30a5", features = [ +arrow = { git = "https://github.com/Embucket/arrow-rs.git", rev = "25a9c23d7b9706238b00b1288d27f036c186cd8e", features = [ "prettyprint", "chrono-tz", ] } -arrow-avro = { git = "https://github.com/Embucket/arrow-rs.git", rev = "5d211125b8e8a5784f2caf606cdf88684dec30a5", default-features = false, features = [ +arrow-avro = { git = "https://github.com/Embucket/arrow-rs.git", rev = "25a9c23d7b9706238b00b1288d27f036c186cd8e", default-features = false, features = [ "deflate", "snappy", "zstd", "bzip2", "xz", ] } -arrow-buffer = { git = "https://github.com/Embucket/arrow-rs.git", rev = "5d211125b8e8a5784f2caf606cdf88684dec30a5", default-features = false } -arrow-data = { git = "https://github.com/Embucket/arrow-rs.git", rev = "5d211125b8e8a5784f2caf606cdf88684dec30a5", default-features = false } -arrow-flight = { git = "https://github.com/Embucket/arrow-rs.git", rev = "5d211125b8e8a5784f2caf606cdf88684dec30a5", features = [ +arrow-buffer = { git = "https://github.com/Embucket/arrow-rs.git", rev = "25a9c23d7b9706238b00b1288d27f036c186cd8e", default-features = false } +arrow-data = { git = "https://github.com/Embucket/arrow-rs.git", rev = "25a9c23d7b9706238b00b1288d27f036c186cd8e", default-features = false } +arrow-flight = { git = "https://github.com/Embucket/arrow-rs.git", rev = "25a9c23d7b9706238b00b1288d27f036c186cd8e", features = [ "flight-sql-experimental", ] } # Both codecs are required here to make sure that code paths like # file-spilling have access to all compression codecs. -arrow-ipc = { git = "https://github.com/Embucket/arrow-rs.git", rev = "5d211125b8e8a5784f2caf606cdf88684dec30a5", default-features = false, features = [ +arrow-ipc = { git = "https://github.com/Embucket/arrow-rs.git", rev = "25a9c23d7b9706238b00b1288d27f036c186cd8e", default-features = false, features = [ "lz4", "zstd", ] } -arrow-ord = { git = "https://github.com/Embucket/arrow-rs.git", rev = "5d211125b8e8a5784f2caf606cdf88684dec30a5", default-features = false } -arrow-schema = { git = "https://github.com/Embucket/arrow-rs.git", rev = "5d211125b8e8a5784f2caf606cdf88684dec30a5", default-features = false } +arrow-ord = { git = "https://github.com/Embucket/arrow-rs.git", rev = "25a9c23d7b9706238b00b1288d27f036c186cd8e", default-features = false } +arrow-schema = { git = "https://github.com/Embucket/arrow-rs.git", rev = "25a9c23d7b9706238b00b1288d27f036c186cd8e", default-features = false } async-trait = "0.1.89" bigdecimal = "0.4.8" bytes = "1.11" @@ -178,7 +178,7 @@ memchr = "2.8.1" num-traits = { version = "0.2" } object_store = { version = "0.13.2", default-features = false } parking_lot = "0.12" -parquet = { git = "https://github.com/Embucket/arrow-rs.git", rev = "5d211125b8e8a5784f2caf606cdf88684dec30a5", default-features = false, features = [ +parquet = { git = "https://github.com/Embucket/arrow-rs.git", rev = "25a9c23d7b9706238b00b1288d27f036c186cd8e", default-features = false, features = [ "arrow", "async", "object_store", diff --git a/datafusion/datasource-csv/src/file_format.rs b/datafusion/datasource-csv/src/file_format.rs index 4a110e6e50a6b..7588d2f80a5cd 100644 --- a/datafusion/datasource-csv/src/file_format.rs +++ b/datafusion/datasource-csv/src/file_format.rs @@ -136,6 +136,7 @@ impl GetExt for CsvFormatFactory { pub struct CsvFormat { options: CsvOptions, record_error_handler: Option>, + numeric_boolean_values: bool, } impl CsvFormat { @@ -202,6 +203,12 @@ impl CsvFormat { self } + /// Accept exact `0` and `1` values in Boolean CSV columns. + pub fn with_numeric_boolean_values(mut self, allow: bool) -> Self { + self.numeric_boolean_values = allow; + self + } + /// Skip malformed field-count records and report them to `handler`. /// /// This recovery path retains source record bytes and disables file range @@ -492,7 +499,9 @@ impl FileFormat for CsvFormat { if csv_options.has_header.is_none() { csv_options.has_header = Some(true); } - let mut source = CsvSource::new(table_schema).with_csv_options(csv_options); + let mut source = CsvSource::new(table_schema) + .with_csv_options(csv_options) + .with_numeric_boolean_values(self.numeric_boolean_values); if let Some(handler) = &self.record_error_handler { source = source.with_record_error_handler(Arc::clone(handler)); } diff --git a/datafusion/datasource-csv/src/source.rs b/datafusion/datasource-csv/src/source.rs index 7c8fb4f9fe76b..2c3e11c7dfc80 100644 --- a/datafusion/datasource-csv/src/source.rs +++ b/datafusion/datasource-csv/src/source.rs @@ -87,6 +87,7 @@ use tokio::io::AsyncWriteExt; #[derive(Debug, Clone)] pub struct CsvSource { options: CsvOptions, + numeric_boolean_values: bool, record_error_handler: Option>, batch_size: Option, table_schema: TableSchema, @@ -100,6 +101,7 @@ impl CsvSource { let table_schema = table_schema.into(); Self { options: CsvOptions::default(), + numeric_boolean_values: false, record_error_handler: None, projection: SplitProjection::unprojected(&table_schema), table_schema, @@ -114,6 +116,17 @@ impl CsvSource { self } + /// Accept exact numeric Boolean literals (`0` and `1`) while scanning CSV. + pub fn with_numeric_boolean_values(mut self, allow: bool) -> Self { + self.numeric_boolean_values = allow; + self + } + + /// Whether exact `0` and `1` values are accepted in Boolean CSV columns. + pub fn numeric_boolean_values(&self) -> bool { + self.numeric_boolean_values + } + /// Skip malformed field-count records and report them to `handler`. pub fn with_record_error_handler( mut self, @@ -217,6 +230,7 @@ impl CsvSource { ) .with_header(self.has_header()) .with_quote(self.quote()) + .with_numeric_boolean_values(self.numeric_boolean_values) .with_preserve_quoted_empty(self.preserve_quoted_empty()) .with_truncated_rows(self.truncate_rows()); if let Some(terminator) = self.terminator() { @@ -404,6 +418,7 @@ impl FileSource for CsvSource { truncate_rows: self.truncate_rows(), null_regex: self.options.null_regex.clone(), preserve_quoted_empty: self.preserve_quoted_empty(), + numeric_boolean_values: self.numeric_boolean_values(), }; Ok(Some(protobuf::PhysicalPlanNode { physical_plan_type: Some(PhysicalPlanType::CsvScan(node)), @@ -684,6 +699,7 @@ impl CsvSource { let source = Arc::new( CsvSource::new(table_schema) .with_csv_options(csv_options) + .with_numeric_boolean_values(scan.numeric_boolean_values) .with_escape(escape) .with_comment(comment), ); @@ -702,13 +718,50 @@ impl CsvSource { #[cfg(test)] mod tests { use super::*; - use arrow::array::{Int64Array, StringArray}; + use arrow::array::{AsArray, Int64Array, StringArray}; use arrow::csv::{CsvRecordError, CsvRecordErrorHandler}; use arrow::datatypes::{DataType, Field, Schema}; use arrow::error::ArrowError; use std::io::Cursor; use std::sync::Mutex; + #[test] + fn csv_source_numeric_boolean_values_are_opt_in() { + let schema = Arc::new(Schema::new(vec![Field::new( + "flag", + DataType::Boolean, + false, + )])); + let mut source = CsvSource::new(schema).with_csv_options(CsvOptions { + has_header: Some(false), + ..Default::default() + }); + source.batch_size = Some(1024); + let strict_error = source + .open(Cursor::new(b"true\n1\n0\nfalse\n")) + .unwrap() + .next() + .unwrap() + .unwrap_err(); + assert!( + strict_error + .to_string() + .contains("value '1' as type 'Boolean'") + ); + + let batches = source + .with_numeric_boolean_values(true) + .open(Cursor::new(b"true\n1\n0\nfalse\n")) + .unwrap() + .collect::, _>>() + .unwrap(); + let values = batches[0].column(0).as_boolean(); + assert_eq!( + values.values().iter().collect::>(), + [true, true, false, false] + ); + } + #[test] fn csv_source_applies_null_regex_to_loaded_rows() { let schema = Arc::new(Schema::new(vec![ diff --git a/datafusion/proto-models/proto/datafusion.proto b/datafusion/proto-models/proto/datafusion.proto index e99c53d148623..dc797defc2920 100644 --- a/datafusion/proto-models/proto/datafusion.proto +++ b/datafusion/proto-models/proto/datafusion.proto @@ -1288,6 +1288,7 @@ message CsvScanExecNode { bool truncate_rows = 8; optional string null_regex = 9; bool preserve_quoted_empty = 10; + bool numeric_boolean_values = 11; } message JsonScanExecNode { diff --git a/datafusion/proto-models/src/generated/pbjson.rs b/datafusion/proto-models/src/generated/pbjson.rs index 406713d9f6545..71a837fe46c4f 100644 --- a/datafusion/proto-models/src/generated/pbjson.rs +++ b/datafusion/proto-models/src/generated/pbjson.rs @@ -4387,6 +4387,9 @@ impl serde::Serialize for CsvScanExecNode { if self.preserve_quoted_empty { len += 1; } + if self.numeric_boolean_values { + len += 1; + } if self.optional_escape.is_some() { len += 1; } @@ -4418,6 +4421,9 @@ impl serde::Serialize for CsvScanExecNode { if self.preserve_quoted_empty { struct_ser.serialize_field("preserveQuotedEmpty", &self.preserve_quoted_empty)?; } + if self.numeric_boolean_values { + struct_ser.serialize_field("numericBooleanValues", &self.numeric_boolean_values)?; + } if let Some(v) = self.optional_escape.as_ref() { match v { csv_scan_exec_node::OptionalEscape::Escape(v) => { @@ -4456,6 +4462,8 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode { "nullRegex", "preserve_quoted_empty", "preserveQuotedEmpty", + "numeric_boolean_values", + "numericBooleanValues", "escape", "comment", ]; @@ -4470,6 +4478,7 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode { TruncateRows, NullRegex, PreserveQuotedEmpty, + NumericBooleanValues, Escape, Comment, } @@ -4501,6 +4510,7 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode { "truncateRows" | "truncate_rows" => Ok(GeneratedField::TruncateRows), "nullRegex" | "null_regex" => Ok(GeneratedField::NullRegex), "preserveQuotedEmpty" | "preserve_quoted_empty" => Ok(GeneratedField::PreserveQuotedEmpty), + "numericBooleanValues" | "numeric_boolean_values" => Ok(GeneratedField::NumericBooleanValues), "escape" => Ok(GeneratedField::Escape), "comment" => Ok(GeneratedField::Comment), _ => Err(serde::de::Error::unknown_field(value, FIELDS)), @@ -4530,6 +4540,7 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode { let mut truncate_rows__ = None; let mut null_regex__ = None; let mut preserve_quoted_empty__ = None; + let mut numeric_boolean_values__ = None; let mut optional_escape__ = None; let mut optional_comment__ = None; while let Some(k) = map_.next_key()? { @@ -4582,6 +4593,12 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode { } preserve_quoted_empty__ = Some(map_.next_value()?); } + GeneratedField::NumericBooleanValues => { + if numeric_boolean_values__.is_some() { + return Err(serde::de::Error::duplicate_field("numericBooleanValues")); + } + numeric_boolean_values__ = Some(map_.next_value()?); + } GeneratedField::Escape => { if optional_escape__.is_some() { return Err(serde::de::Error::duplicate_field("escape")); @@ -4605,6 +4622,7 @@ impl<'de> serde::Deserialize<'de> for CsvScanExecNode { truncate_rows: truncate_rows__.unwrap_or_default(), null_regex: null_regex__, preserve_quoted_empty: preserve_quoted_empty__.unwrap_or_default(), + numeric_boolean_values: numeric_boolean_values__.unwrap_or_default(), optional_escape: optional_escape__, optional_comment: optional_comment__, }) diff --git a/datafusion/proto-models/src/generated/prost.rs b/datafusion/proto-models/src/generated/prost.rs index e41d47343776e..b529239bde720 100644 --- a/datafusion/proto-models/src/generated/prost.rs +++ b/datafusion/proto-models/src/generated/prost.rs @@ -1968,6 +1968,8 @@ pub struct CsvScanExecNode { pub null_regex: ::core::option::Option<::prost::alloc::string::String>, #[prost(bool, tag = "10")] pub preserve_quoted_empty: bool, + #[prost(bool, tag = "11")] + pub numeric_boolean_values: bool, #[prost(oneof = "csv_scan_exec_node::OptionalEscape", tags = "5")] pub optional_escape: ::core::option::Option, #[prost(oneof = "csv_scan_exec_node::OptionalComment", tags = "6")] diff --git a/datafusion/proto/tests/cases/plans/sources.rs b/datafusion/proto/tests/cases/plans/sources.rs index 6dfd4b748ebef..f3e46118cf22e 100644 --- a/datafusion/proto/tests/cases/plans/sources.rs +++ b/datafusion/proto/tests/cases/plans/sources.rs @@ -220,19 +220,22 @@ fn roundtrip_csv_scan_preserves_format_options() -> Result<()> { let file_schema = Arc::new(Schema::new(vec![Field::new("col", DataType::Utf8, false)])); let table_schema = TableSchema::from(&file_schema); - let file_source = - Arc::new(CsvSource::new(table_schema).with_csv_options(CsvOptions { - has_header: Some(false), - delimiter: b'|', - quote: b'\'', - escape: Some(b'\\'), - comment: Some(b'#'), - newlines_in_values: Some(true), - truncated_rows: Some(true), - null_regex: Some(r"\A(?:NA|)\z".to_owned()), - preserve_quoted_empty: Some(true), - ..Default::default() - })); + let file_source = Arc::new( + CsvSource::new(table_schema) + .with_csv_options(CsvOptions { + has_header: Some(false), + delimiter: b'|', + quote: b'\'', + escape: Some(b'\\'), + comment: Some(b'#'), + newlines_in_values: Some(true), + truncated_rows: Some(true), + null_regex: Some(r"\A(?:NA|)\z".to_owned()), + preserve_quoted_empty: Some(true), + ..Default::default() + }) + .with_numeric_boolean_values(true), + ); let scan_config = FileScanConfigBuilder::new(ObjectStoreUrl::local_filesystem(), file_source) @@ -270,6 +273,7 @@ fn roundtrip_csv_scan_preserves_format_options() -> Result<()> { assert!(csv_source.truncate_rows()); assert_eq!(csv_source.null_regex(), Some(r"\A(?:NA|)\z")); assert!(csv_source.preserve_quoted_empty()); + assert!(csv_source.numeric_boolean_values()); Ok(()) }