diff --git a/datafusion/expr/src/utils.rs b/datafusion/expr/src/utils.rs index 84185ffecff72..66821179116a3 100644 --- a/datafusion/expr/src/utils.rs +++ b/datafusion/expr/src/utils.rs @@ -367,18 +367,18 @@ fn get_excluded_columns( }; idents.extend(exclude_owned.iter()); } - // Excluded columns should be unique + // Lookup uses the resolved name, so quoting differences cannot distinguish duplicates. let n_elem = idents.len(); - let unique_idents = idents.into_iter().collect::>(); - // If HashSet size, and vector length are different, this means that some of the excluded columns - // are not unique. In this case return error. - if n_elem != unique_idents.len() { + let unique_names = idents + .into_iter() + .map(|ident| ident.value.as_str()) + .collect::>(); + if n_elem != unique_names.len() { return plan_err!("EXCLUDE or EXCEPT contains duplicate column names"); } let mut result = vec![]; - for ident in unique_idents.into_iter() { - let col_name = ident.value.as_str(); + for col_name in unique_names { let (qualifier, field) = schema.qualified_field_with_name(qualifier, col_name)?; result.push(Column::from((qualifier, field))); } diff --git a/datafusion/sql/src/expr/function.rs b/datafusion/sql/src/expr/function.rs index b9a0a80fadc5a..783a2cfd4e476 100644 --- a/datafusion/sql/src/expr/function.rs +++ b/datafusion/sql/src/expr/function.rs @@ -40,25 +40,6 @@ use sqlparser::ast::{ WindowType, }; -fn function_wildcard_options( - options: WildcardAdditionalOptions, -) -> Result> { - if options.opt_alias.is_some() { - return not_impl_err!("wildcard function argument with AS alias"); - } - if options.opt_replace.is_some() { - return not_impl_err!("wildcard function argument with REPLACE"); - } - - Ok(Box::new(WildcardOptions { - ilike: options.opt_ilike, - exclude: options.opt_exclude, - except: options.opt_except, - replace: None, - rename: options.opt_rename, - })) -} - /// Suggest a valid function based on an invalid input function name /// /// Returns `None` if no valid matches are found. This happens when there are no @@ -247,6 +228,28 @@ impl FunctionArgs { type WithinGroupExtraction = (Vec, Vec, Vec>); impl SqlToRel<'_, S> { + fn function_wildcard_options( + &self, + options: WildcardAdditionalOptions, + ) -> Result> { + if options.opt_alias.is_some() { + return not_impl_err!("wildcard function argument with AS alias"); + } + if options.opt_replace.is_some() { + return not_impl_err!("wildcard function argument with REPLACE"); + } + + Ok(Box::new(WildcardOptions { + ilike: options.opt_ilike, + exclude: options + .opt_exclude + .map(|exclude| self.normalize_wildcard_exclude(exclude)), + except: options.opt_except, + replace: None, + rename: options.opt_rename, + })) + } + pub(super) fn sql_function_to_expr( &self, function: SQLFunction, @@ -1109,7 +1112,7 @@ impl SqlToRel<'_, S> { #[expect(deprecated)] let expr = Expr::Wildcard { qualifier: None, - options: function_wildcard_options(options)?, + options: self.function_wildcard_options(options)?, }; Ok((expr, None)) } @@ -1140,7 +1143,7 @@ impl SqlToRel<'_, S> { #[expect(deprecated)] let expr = Expr::Wildcard { qualifier: qualifier.into(), - options: function_wildcard_options(options)?, + options: self.function_wildcard_options(options)?, }; Ok((expr, None)) } diff --git a/datafusion/sql/src/select.rs b/datafusion/sql/src/select.rs index 1d69e14225493..10e6ba0925a44 100644 --- a/datafusion/sql/src/select.rs +++ b/datafusion/sql/src/select.rs @@ -54,9 +54,9 @@ use datafusion_expr::{ use indexmap::IndexMap; use sqlparser::ast::{ - Distinct, Expr as SQLExpr, GroupByExpr, NamedWindowExpr, OrderBy, RenameSelectItem, - SelectItemQualifiedWildcardKind, WildcardAdditionalOptions, WindowType, - visit_expressions_mut, + Distinct, ExcludeSelectItem, Expr as SQLExpr, GroupByExpr, NamedWindowExpr, + ObjectNamePart, OrderBy, RenameSelectItem, SelectItemQualifiedWildcardKind, + WildcardAdditionalOptions, WindowType, visit_expressions_mut, }; use sqlparser::ast::{NamedWindowDefinition, Select, SelectItem, TableWithJoins}; @@ -1246,6 +1246,22 @@ impl SqlToRel<'_, S> { } } + pub(crate) fn normalize_wildcard_exclude( + &self, + mut exclude: ExcludeSelectItem, + ) -> ExcludeSelectItem { + let names = match &mut exclude { + ExcludeSelectItem::Single(name) => std::slice::from_mut(name), + ExcludeSelectItem::Multiple(names) => names, + }; + for name in names { + if let [ObjectNamePart::Identifier(ident)] = name.0.as_mut_slice() { + ident.value = self.ident_normalizer.normalize(ident.clone()); + } + } + exclude + } + /// If there is a REPLACE statement in the projected expression in the form of /// "REPLACE (some_column_within_an_expr AS some_column)", we should plan the /// replace expressions first. @@ -1256,6 +1272,9 @@ impl SqlToRel<'_, S> { planner_context: &mut PlannerContext, options: WildcardAdditionalOptions, ) -> Result { + let exclude = options + .opt_exclude + .map(|exclude| self.normalize_wildcard_exclude(exclude)); let rename = options.opt_rename.map(|mut rename| { let items = match &mut rename { RenameSelectItem::Single(item) => std::slice::from_mut(item), @@ -1269,7 +1288,7 @@ impl SqlToRel<'_, S> { }); let planned_option = WildcardOptions { ilike: options.opt_ilike, - exclude: options.opt_exclude, + exclude, except: options.opt_except, replace: None, rename, diff --git a/datafusion/sql/tests/sql_integration.rs b/datafusion/sql/tests/sql_integration.rs index 0ac97600a9823..e824399de98d6 100644 --- a/datafusion/sql/tests/sql_integration.rs +++ b/datafusion/sql/tests/sql_integration.rs @@ -2477,6 +2477,27 @@ fn scalar_expr_planner_receives_wildcard_options() { ); } +#[test] +fn scalar_expr_planner_normalizes_wildcard_exclude() { + let state = mock_session_state().with_expr_planner(Arc::new(ScalarWildcardPlanner)); + let plan = logical_plan_from_state( + "SELECT concat(* EXCLUDE FIRST_NAME) FROM person AS p", + &GenericDialect {}, + ParserOptions::default(), + state, + ) + .unwrap(); + + assert_snapshot!( + plan, + @r" + Projection: concat(p.last_name, p.state) + SubqueryAlias: p + TableScan: person + " + ); +} + #[test] fn scalar_expr_planner_applies_wildcard_ilike() { let state = mock_session_state().with_expr_planner(Arc::new(ScalarWildcardPlanner)); diff --git a/datafusion/sqllogictest/test_files/select.slt b/datafusion/sqllogictest/test_files/select.slt index 4107921d2fda5..86ef4dfb0fc18 100644 --- a/datafusion/sqllogictest/test_files/select.slt +++ b/datafusion/sqllogictest/test_files/select.slt @@ -1289,6 +1289,24 @@ LIMIT 5 100 1000 200 2000 +# Unquoted EXCLUDE targets follow the same normalization as input columns. +query III +SELECT * EXCLUDE B FROM table1 ORDER BY a +---- +1 100 1000 +2 200 2000 + +# Quoted targets preserve case and match quoted aliases. +query III +SELECT * EXCLUDE "B" FROM (SELECT a, b AS "B", c, d FROM table1) ORDER BY a +---- +1 100 1000 +2 200 2000 + +# Quoted and unquoted spellings that resolve to one field are duplicates. +statement error DataFusion error: Error during planning: EXCLUDE or EXCEPT contains duplicate column names +SELECT * EXCLUDE (B, "b") FROM table1 + # when wildcard is prepended with table name, exclude should still work # below query should emit all the columns except a and b query II