diff --git a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 index 0508bcad579409..d1df1ad68d1ad2 100644 --- a/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 +++ b/fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4 @@ -1101,7 +1101,7 @@ refreshSchedule ; refreshMethod - : COMPLETE | AUTO + : COMPLETE | AUTO | INCREMENTAL ; mvPartition diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java index 94a46864eb9a77..39d9d76de144d1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/Column.java @@ -59,12 +59,15 @@ public class Column implements GsonPostProcessable { public static final String HIDDEN_COLUMN_PREFIX = "__DORIS_"; // all shadow indexes should have this prefix in name public static final String SHADOW_NAME_PREFIX = "__doris_shadow_"; + public static final String IVM_HIDDEN_COLUMN_PREFIX = "__DORIS_IVM_"; // NOTE: you should name hidden column start with '__DORIS_' !!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! public static final String DELETE_SIGN = "__DORIS_DELETE_SIGN__"; public static final String WHERE_SIGN = "__DORIS_WHERE_SIGN__"; public static final String SEQUENCE_COL = "__DORIS_SEQUENCE_COL__"; public static final String ROWID_COL = "__DORIS_ROWID_COL__"; public static final String GLOBAL_ROWID_COL = "__DORIS_GLOBAL_ROWID_COL__"; + public static final String IVM_ROW_ID_COL = "__DORIS_IVM_ROW_ID_COL__"; + public static final String IVM_AGG_COUNT_COL = "__DORIS_IVM_AGG_COUNT_COL__"; public static final String ROW_STORE_COL = "__DORIS_ROW_STORE_COL__"; public static final String VERSION_COL = "__DORIS_VERSION_COL__"; public static final String SKIP_BITMAP_COL = "__DORIS_SKIP_BITMAP_COL__"; @@ -217,6 +220,10 @@ public Column(String name, Type type, boolean isKey, AggregateType aggregateType false, null, null, Sets.newHashSet(), null); } + public static boolean isIvmHiddenColumn(String columnName) { + return StringUtils.startsWith(columnName, IVM_HIDDEN_COLUMN_PREFIX); + } + public Column(String name, Type type, boolean isKey, AggregateType aggregateType, boolean isAllowNull, String defaultValue, String comment, boolean visible, int colUniqueId) { this(name, type, isKey, aggregateType, isAllowNull, -1, defaultValue, comment, visible, null, colUniqueId, null, diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java index 729c68e5e4f31f..2ca9beafd3280b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java @@ -44,9 +44,12 @@ import org.apache.doris.mtmv.MTMVRelation; import org.apache.doris.mtmv.MTMVSnapshotIf; import org.apache.doris.mtmv.MTMVStatus; +import org.apache.doris.mtmv.ivm.IvmInfo; +import org.apache.doris.mtmv.ivm.IvmUtil; import org.apache.doris.nereids.rules.analysis.SessionVarGuardRewriter; import org.apache.doris.qe.ConnectContext; +import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import com.google.gson.annotations.SerializedName; @@ -56,6 +59,7 @@ import org.apache.logging.log4j.Logger; import java.io.IOException; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Optional; @@ -87,6 +91,8 @@ public class MTMV extends OlapTable { private MTMVPartitionInfo mvPartitionInfo; @SerializedName("rs") private MTMVRefreshSnapshot refreshSnapshot; + @SerializedName("ii") + private IvmInfo ivmInfo; // Should update after every fresh, not persist // Cache with SessionVarGuardExpr: used when query session variables differ from MV creation variables private MTMVCache cacheWithGuard; @@ -120,6 +126,7 @@ public MTMV() { this.mvPartitionInfo = params.mvPartitionInfo; this.relation = params.relation; this.refreshSnapshot = new MTMVRefreshSnapshot(); + this.ivmInfo = new IvmInfo(); this.envInfo = new EnvInfo(-1L, -1L); this.sessionVariables = params.sessionVariables; mvRwLock = new ReentrantReadWriteLock(true); @@ -437,6 +444,21 @@ public MTMVRefreshSnapshot getRefreshSnapshot() { return refreshSnapshot; } + public IvmInfo getIvmInfo() { + return ivmInfo; + } + + public List getInsertedColumnNames() { + List columns = getBaseSchema(true); + List columnNames = Lists.newArrayListWithExpectedSize(columns.size()); + for (Column column : columns) { + if (column.isVisible() || IvmUtil.isIvmHiddenColumn(column.getName())) { + columnNames.add(column.getName()); + } + } + return columnNames; + } + public long getSchemaChangeVersion() { readMvLock(); try { @@ -609,6 +631,9 @@ private void compatibleInternal(CatalogMgr catalogMgr) throws Exception { @Override public void gsonPostProcess() throws IOException { super.gsonPostProcess(); + if (ivmInfo == null) { + ivmInfo = new IvmInfo(); + } Map partitionSnapshots = refreshSnapshot.getPartitionSnapshots(); compatiblePctSnapshot(partitionSnapshots); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index ccf1e9be8993fb..e46fd348cf34b1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -57,18 +57,16 @@ import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.mtmv.MTMVRelation; import org.apache.doris.mtmv.MTMVUtil; +import org.apache.doris.mtmv.ivm.IvmRefreshManager; +import org.apache.doris.mtmv.ivm.IvmRefreshResult; import org.apache.doris.nereids.StatementContext; -import org.apache.doris.nereids.glue.LogicalPlanAdapter; import org.apache.doris.nereids.trees.plans.commands.UpdateMvByPartitionCommand; -import org.apache.doris.qe.AuditLogHelper; import org.apache.doris.qe.ConnectContext; -import org.apache.doris.qe.QueryState.MysqlStateType; import org.apache.doris.qe.StmtExecutor; import org.apache.doris.system.SystemInfoService; import org.apache.doris.thrift.TCell; import org.apache.doris.thrift.TRow; import org.apache.doris.thrift.TStatusCode; -import org.apache.doris.thrift.TUniqueId; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; @@ -244,6 +242,21 @@ public void run() throws JobException { if (refreshMode == MTMVTaskRefreshMode.NOT_REFRESH) { return; } + // Attempt IVM refresh for incremental MVs and fall back when the plan is unsupported. + if (mtmv.getRefreshInfo().getRefreshMethod() == RefreshMethod.INCREMENTAL) { + IvmRefreshManager ivmRefreshManager = new IvmRefreshManager(); + IvmRefreshResult ivmResult = ivmRefreshManager.doRefresh(mtmv); + if (ivmResult.isSuccess()) { + LOG.info("IVM incremental refresh succeeded for mv={}, taskId={}", + mtmv.getName(), getTaskId()); + return; + } + LOG.warn("IVM refresh fell back for mv={}, reason={}, detail={}, taskId={}. " + + "Continuing with partition-based refresh.", + mtmv.getName(), ivmResult.getFallbackReason(), + ivmResult.getDetailMessage(), getTaskId()); + // TODO: it may cause too many full refresh, need limit full refresh here + } Map tableWithPartKey = getIncrementalTableMap(); this.completedPartitions = Lists.newCopyOnWriteArrayList(); int refreshPartitionNum = mtmv.getRefreshPartitionNum(); @@ -321,36 +334,23 @@ private void executeWithRetry(Set execPartitionNames, Map refreshPartitionNames, Map tableWithPartKey) throws Exception { - ConnectContext ctx = MTMVPlanUtil.createMTMVContext(mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_RUN_MTMV_TASK); + // Create MTMV context first so that new StatementContext() captures the + // correct thread-local ConnectContext (with MTMV disabled rules, etc.). + ConnectContext mtmvCtx = MTMVPlanUtil.createMTMVContext(mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_RUN_MTMV_TASK); StatementContext statementContext = new StatementContext(); for (Entry entry : snapshots.entrySet()) { statementContext.setSnapshot(entry.getKey(), entry.getValue()); } - ctx.setStatementContext(statementContext); - TUniqueId queryId = generateQueryId(); - lastQueryId = DebugUtil.printId(queryId); // if SELF_MANAGE mv, only have default partition, will not have partitionItem, so we give empty set UpdateMvByPartitionCommand command = UpdateMvByPartitionCommand .from(mtmv, mtmv.getMvPartitionInfo().getPartitionType() != MTMVPartitionType.SELF_MANAGE ? refreshPartitionNames : Sets.newHashSet(), tableWithPartKey, statementContext); - try { - executor = new StmtExecutor(ctx, new LogicalPlanAdapter(command, ctx.getStatementContext())); - ctx.setExecutor(executor); - ctx.setQueryId(queryId); - ctx.getState().setNereids(true); - command.run(ctx, executor); - if (getStatus() == TaskStatus.CANCELED) { - // Throwing an exception to interrupt subsequent partition update tasks - throw new JobException("task is CANCELED"); - } - if (ctx.getState().getStateType() != MysqlStateType.OK) { - throw new JobException(ctx.getState().getErrorMessage()); - } - } finally { - if (executor != null) { - AuditLogHelper.logAuditLog(ctx, getDummyStmt(refreshPartitionNames), - executor.getParsedStmt(), executor.getQueryStatisticsForAuditLog(), true); - } + boolean enableIvmNormalMTMVPlan = mtmv.getRefreshInfo().getRefreshMethod() == RefreshMethod.INCREMENTAL; + executor = MTMVPlanUtil.executeCommand(mtmvCtx, command, statementContext, + getDummyStmt(refreshPartitionNames), enableIvmNormalMTMVPlan); + lastQueryId = DebugUtil.printId(executor.getContext().queryId()); + if (getStatus() == TaskStatus.CANCELED) { + throw new JobException("task is CANCELED"); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVAnalyzeQueryInfo.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVAnalyzeQueryInfo.java index f9ac4f18c1b588..247f0eae745b7c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVAnalyzeQueryInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVAnalyzeQueryInfo.java @@ -17,6 +17,8 @@ package org.apache.doris.mtmv; +import org.apache.doris.mtmv.ivm.IvmNormalizeResult; +import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition; import java.util.List; @@ -25,6 +27,8 @@ public class MTMVAnalyzeQueryInfo { private MTMVRelation relation; private MTMVPartitionInfo mvPartitionInfo; private List columnDefinitions; + // set when IVM normalization is enabled; carries normalizedPlan + aggMeta + private IvmNormalizeResult ivmNormalizeResult; public MTMVAnalyzeQueryInfo(List columnDefinitions, MTMVPartitionInfo mvPartitionInfo, MTMVRelation relation) { @@ -44,4 +48,17 @@ public MTMVPartitionInfo getMvPartitionInfo() { public MTMVRelation getRelation() { return relation; } + + public IvmNormalizeResult getIvmNormalizeResult() { + return ivmNormalizeResult; + } + + public void setIvmNormalizeResult(IvmNormalizeResult ivmNormalizeResult) { + this.ivmNormalizeResult = ivmNormalizeResult; + } + + /** Convenience accessor — returns the normalized plan, or null if IVM is not active. */ + public Plan getIvmNormalizedPlan() { + return ivmNormalizeResult != null ? ivmNormalizeResult.getNormalizedPlan() : null; + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java index 4be21db3008033..a567dd296280f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPlanUtil.java @@ -45,7 +45,10 @@ import org.apache.doris.datasource.CatalogIf; import org.apache.doris.datasource.ExternalTable; import org.apache.doris.job.exception.JobException; +import org.apache.doris.job.task.AbstractTask; import org.apache.doris.mtmv.MTMVPartitionInfo.MTMVPartitionType; +import org.apache.doris.mtmv.MTMVRefreshEnum.RefreshMethod; +import org.apache.doris.mtmv.ivm.IvmUtil; import org.apache.doris.nereids.NereidsPlanner; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundResultSink; @@ -60,6 +63,7 @@ import org.apache.doris.nereids.trees.expressions.Slot; import org.apache.doris.nereids.trees.expressions.SlotReference; import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.Command; import org.apache.doris.nereids.trees.plans.commands.ExplainCommand.ExplainLevel; import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition; import org.apache.doris.nereids.trees.plans.commands.info.CreateMTMVInfo; @@ -78,8 +82,11 @@ import org.apache.doris.nereids.types.VarcharType; import org.apache.doris.nereids.types.coercion.CharacterType; import org.apache.doris.nereids.util.TypeCoercionUtils; +import org.apache.doris.qe.AuditLogHelper; import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.QueryState.MysqlStateType; import org.apache.doris.qe.SessionVariable; +import org.apache.doris.qe.StmtExecutor; import com.google.common.collect.ImmutableList; import com.google.common.collect.Lists; @@ -89,6 +96,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -130,6 +138,49 @@ public static ConnectContext createMTMVContext(MTMV mtmv, List disable return ctx; } + /** + * Execute a Nereids command in an MTMV context with optional audit logging. + * Creates a new MTMV ConnectContext internally. Callers that need the ConnectContext + * to exist before the StatementContext is constructed (so that {@code new StatementContext()} + * captures the correct thread-local) should use + * {@link #executeCommand(ConnectContext, Command, StatementContext, String, boolean)} instead. + */ + public static StmtExecutor executeCommand(MTMV mtmv, Command command, + StatementContext stmtCtx, @Nullable String auditStmt, boolean enableIvmNormalMTMVPlan) throws Exception { + ConnectContext ctx = createMTMVContext(mtmv, DISABLE_RULES_WHEN_RUN_MTMV_TASK); + stmtCtx.setConnectContext(ctx); + return executeCommand(ctx, command, stmtCtx, auditStmt, enableIvmNormalMTMVPlan); + } + + /** + * Execute a Nereids command using a pre-created ConnectContext. + * Use this overload when the ConnectContext must be created before the StatementContext + * so that {@code new StatementContext()} captures the correct thread-local ConnectContext. + */ + public static StmtExecutor executeCommand(ConnectContext ctx, Command command, + StatementContext stmtCtx, @Nullable String auditStmt, boolean enableIvmNormalMTMVPlan) throws Exception { + ctx.setStatementContext(stmtCtx); + ctx.getState().setNereids(true); + ctx.getSessionVariable().setEnableMaterializedViewRewrite(false); + ctx.getSessionVariable().setEnableDmlMaterializedViewRewrite(false); + ctx.getSessionVariable().setEnableIvmNormalRewrite(enableIvmNormalMTMVPlan); + StmtExecutor executor = new StmtExecutor(ctx, new LogicalPlanAdapter(command, stmtCtx)); + ctx.setExecutor(executor); + ctx.setQueryId(AbstractTask.generateQueryId()); + try { + command.run(ctx, executor); + if (ctx.getState().getStateType() != MysqlStateType.OK) { + throw new UserException(ctx.getState().getErrorMessage()); + } + } finally { + if (auditStmt != null) { + AuditLogHelper.logAuditLog(ctx, auditStmt, + executor.getParsedStmt(), executor.getQueryStatisticsForAuditLog(), true); + } + } + return executor; + } + public static ConnectContext createBasicMvContext(@Nullable ConnectContext parentContext, List disableRules, Map sessionVariables) { ConnectContext ctx = new ConnectContext(); @@ -289,13 +340,35 @@ public static List generateColumns(Plan plan, ConnectContext c if (slots.isEmpty()) { throw new org.apache.doris.nereids.exceptions.AnalysisException("table should contain at least one column"); } - if (!CollectionUtils.isEmpty(simpleColumnDefinitions) && simpleColumnDefinitions.size() != slots.size()) { + // Separate IVM hidden columns from user-visible columns. + // Schema layout must match normalized plan output: [row_id, user visible, trailing hidden agg cols] + Slot rowIdSlot = null; + List trailingHiddenSlots = new ArrayList<>(); + List userSlots = new ArrayList<>(); + for (Slot slot : slots) { + if (Column.IVM_ROW_ID_COL.equals(slot.getName())) { + rowIdSlot = slot; + } else if (IvmUtil.isIvmHiddenColumn(slot.getName())) { + trailingHiddenSlots.add(slot); + } else { + userSlots.add(slot); + } + } + int userSlotSize = userSlots.size(); + if (!CollectionUtils.isEmpty(simpleColumnDefinitions) && simpleColumnDefinitions.size() != userSlotSize) { throw new org.apache.doris.nereids.exceptions.AnalysisException( "simpleColumnDefinitions size is not equal to the query's"); } + // 1. Row-id column first (if present) + if (rowIdSlot != null) { + columns.add(IvmUtil.newIvmRowIdColumnDefinition( + rowIdSlot.getDataType().conversion(), rowIdSlot.nullable())); + } + // 2. User-visible column definitions Set colNames = Sets.newHashSet(); - for (int i = 0; i < slots.size(); i++) { - String colName = CollectionUtils.isEmpty(simpleColumnDefinitions) ? slots.get(i).getName() + for (int i = 0; i < userSlots.size(); i++) { + Slot userSlot = userSlots.get(i); + String colName = CollectionUtils.isEmpty(simpleColumnDefinitions) ? userSlot.getName() : simpleColumnDefinitions.get(i).getName(); try { FeNameFormat.checkColumnName(colName); @@ -307,18 +380,23 @@ public static List generateColumns(Plan plan, ConnectContext c } else { colNames.add(colName); } - DataType dataType = getDataType(slots.get(i), i, ctx, partitionCol, distributionColumnNames); + DataType dataType = getDataType(userSlot, i, ctx, partitionCol, distributionColumnNames); // If datatype is AggStateType, AggregateType should be generic, or column definition check will fail columns.add(new ColumnDefinition( colName, dataType, false, - slots.get(i).getDataType() instanceof AggStateType ? AggregateType.GENERIC : null, - slots.get(i).nullable(), + userSlot.getDataType() instanceof AggStateType ? AggregateType.GENERIC : null, + userSlot.nullable(), Optional.empty(), CollectionUtils.isEmpty(simpleColumnDefinitions) ? null : simpleColumnDefinitions.get(i).getComment())); } + // 3. Trailing hidden agg state columns (after user-visible) + for (Slot hiddenSlot : trailingHiddenSlots) { + columns.add(IvmUtil.newIvmAggHiddenColumnDefinition( + hiddenSlot.getName(), hiddenSlot.getDataType().conversion(), hiddenSlot.nullable())); + } // add a hidden column as row store if (properties != null) { try { @@ -382,7 +460,8 @@ public static DataType getDataType(Slot s, int i, ConnectContext ctx, String par return dataType; } - public static MTMVAnalyzeQueryInfo analyzeQueryWithSql(MTMV mtmv, ConnectContext ctx) throws UserException { + public static MTMVAnalyzeQueryInfo analyzeQueryWithSql(MTMV mtmv, ConnectContext ctx, + boolean enableIvmNormalize) throws UserException { String querySql = mtmv.getQuerySql(); MTMVPartitionInfo mvPartitionInfo = mtmv.getMvPartitionInfo(); MTMVPartitionDefinition mtmvPartitionDefinition = new MTMVPartitionDefinition(); @@ -409,16 +488,14 @@ public static MTMVAnalyzeQueryInfo analyzeQueryWithSql(MTMV mtmv, ConnectContext DistributionDescriptor distribution = new DistributionDescriptor(defaultDistributionInfo.getType().equals( DistributionInfoType.HASH), defaultDistributionInfo.getAutoBucket(), defaultDistributionInfo.getBucketNum(), Lists.newArrayList(mtmv.getDistributionColumnNames())); - return analyzeQuery(ctx, mtmv.getMvProperties(), querySql, mtmvPartitionDefinition, distribution, null, - mtmv.getTableProperty().getProperties(), keys, logicalPlan); + return analyzeQuery(ctx, mtmv.getMvProperties(), mtmvPartitionDefinition, distribution, null, + mtmv.getTableProperty().getProperties(), keys, logicalPlan, enableIvmNormalize); } public static MTMVAnalyzeQueryInfo analyzeQuery(ConnectContext ctx, Map mvProperties, - String querySql, MTMVPartitionDefinition mvPartitionDefinition, DistributionDescriptor distribution, List simpleColumnDefinitions, Map properties, List keys, - LogicalPlan - logicalQuery) throws UserException { + LogicalPlan logicalQuery, boolean enableIvmNormalize) throws UserException { try (StatementContext statementContext = ctx.getStatementContext()) { NereidsPlanner planner = new NereidsPlanner(statementContext); // this is for expression column name infer when not use alias @@ -432,17 +509,24 @@ public static MTMVAnalyzeQueryInfo analyzeQuery(ConnectContext ctx, Map keysSet = Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); keysSet.addAll(keys); validateColumns(columns, keysSet, finalEnableMergeOnWrite); - return new MTMVAnalyzeQueryInfo(columns, mvPartitionInfo, relation); + MTMVAnalyzeQueryInfo queryInfo = new MTMVAnalyzeQueryInfo(columns, mvPartitionInfo, relation); + if (enableIvmNormalize) { + planner.getCascadesContext().getIvmNormalizeResult().ifPresent( + queryInfo::setIvmNormalizeResult); + } + return queryInfo; } } @@ -494,7 +583,19 @@ private static void validateColumns(List columns, Set } } - private static void analyzeKeys(List keys, Map properties, List columns) { + private static List analyzeKeys(List keys, Map properties, + List columns, boolean isIvm) { + if (isIvm) { + // for IVM, the hidden row-id column is the sole unique key + for (ColumnDefinition col : columns) { + if (Column.IVM_ROW_ID_COL.equals(col.getName())) { + col.setIsKey(true); + return Lists.newArrayList(col.getName()); + } + } + throw new org.apache.doris.nereids.exceptions.AnalysisException( + "IVM row-id column not found in generated columns; IVM normalization may have failed."); + } boolean enableDuplicateWithoutKeysByDefault = false; try { if (properties != null) { @@ -532,6 +633,7 @@ private static void analyzeKeys(List keys, Map propertie } } } + return keys; } private static void analyzeExpressions(Plan plan, Map mvProperties) { @@ -540,7 +642,7 @@ private static void analyzeExpressions(Plan plan, Map mvProperti if (enableNondeterministicFunction) { return; } - List functionCollectResult = MaterializedViewUtils.extractNondeterministicFunction(plan); + List functionCollectResult = MaterializedViewUtils.extractMvNondeterministicFunction(plan); if (!CollectionUtils.isEmpty(functionCollectResult)) { throw new AnalysisException(String.format( "can not contain nonDeterministic expression, the expression is %s. " @@ -553,7 +655,8 @@ private static void analyzeExpressions(Plan plan, Map mvProperti public static void ensureMTMVQueryUsable(MTMV mtmv, ConnectContext ctx) throws JobException { MTMVAnalyzeQueryInfo mtmvAnalyzedQueryInfo; try { - mtmvAnalyzedQueryInfo = MTMVPlanUtil.analyzeQueryWithSql(mtmv, ctx); + boolean enableIvmNormalize = mtmv.getRefreshInfo().getRefreshMethod() == RefreshMethod.INCREMENTAL; + mtmvAnalyzedQueryInfo = MTMVPlanUtil.analyzeQueryWithSql(mtmv, ctx, enableIvmNormalize); } catch (Exception e) { throw new JobException(e.getMessage(), e); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshEnum.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshEnum.java index b9d27db9c22045..9aab260d338b23 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshEnum.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRefreshEnum.java @@ -27,7 +27,8 @@ public class MTMVRefreshEnum { */ public enum RefreshMethod { COMPLETE, //complete - AUTO //try to update incrementally, if not possible, update in full + AUTO, // existing auto refresh behavior + INCREMENTAL // opt in nereids ivm rewrite flow } /** diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/AbstractDeltaStrategy.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/AbstractDeltaStrategy.java new file mode 100644 index 00000000000000..043153f3a44bd8 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/AbstractDeltaStrategy.java @@ -0,0 +1,117 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv.ivm; + +import org.apache.doris.catalog.MTMV; +import org.apache.doris.datasource.InternalCatalog; +import org.apache.doris.mtmv.BaseTableInfo; +import org.apache.doris.nereids.analyzer.UnboundTableSink; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.literal.TinyIntLiteral; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.Command; +import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType; +import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink; +import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; + +import com.google.common.collect.ImmutableList; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +/** + * Shared helpers for IVM delta rewrite strategies. + * + *

Provides common operations used by both scan-only and aggregate delta strategies: + * stripping result sinks, extracting scan nodes, building insert commands, etc. + */ +public abstract class AbstractDeltaStrategy implements IvmDeltaStrategy { + + /** Name used for the mock dml_factor column (1 for insert, -1 for delete). */ + protected static final String DML_FACTOR_NAME = "__dml_factor__"; + + /** Strips {@link LogicalResultSink} wrappers from the top of a plan tree. */ + protected static Plan stripResultSink(Plan plan) { + while (plan instanceof LogicalResultSink) { + plan = ((LogicalResultSink) plan).child(); + } + return plan; + } + + /** + * Walks down through Project / Aggregate nodes to find the leaf OlapScan. + * Throws if an unsupported node type is encountered. + */ + protected static LogicalOlapScan extractScan(Plan plan) { + if (plan instanceof LogicalOlapScan) { + return (LogicalOlapScan) plan; + } + if (plan instanceof LogicalProject) { + return extractScan(((LogicalProject) plan).child()); + } + if (plan instanceof LogicalAggregate) { + return extractScan(((LogicalAggregate) plan).child()); + } + throw new AnalysisException( + "IVM delta rewrite does not yet support: " + plan.getClass().getSimpleName()); + } + + /** Builds a {@link BaseTableInfo} from a scan node. */ + protected static BaseTableInfo extractBaseTableInfo(LogicalOlapScan scan) { + return new BaseTableInfo(scan.getTable(), 0L); + } + + /** + * Wraps a query plan with an {@link UnboundTableSink} and {@link InsertIntoTableCommand} + * targeting the given MTMV. + */ + protected static Command buildInsertCommand(Plan queryPlan, IvmDeltaRewriteContext ctx) { + MTMV mtmv = ctx.getMtmv(); + List mvNameParts = ImmutableList.of( + InternalCatalog.INTERNAL_CATALOG_NAME, + mtmv.getQualifiedDbName(), + mtmv.getName()); + UnboundTableSink sink = new UnboundTableSink<>( + mvNameParts, mtmv.getInsertedColumnNames(), ImmutableList.of(), + false, ImmutableList.of(), false, + TPartialUpdateNewRowPolicy.APPEND, DMLCommandType.INSERT, + Optional.empty(), Optional.empty(), (LogicalPlan) queryPlan); + return new InsertIntoTableCommand(sink, Optional.empty(), Optional.empty(), Optional.empty()); + } + + /** + * Returns a new project that appends a mock {@code dml_factor = 1} column to the given + * bottom project's output list. The original child plan is preserved. + * + *

The mock factor will be replaced with a real stream-sourced value once stream + * integration is ready. + */ + protected static LogicalProject appendMockDmlFactor(LogicalProject bottomProject) { + List outputs = new ArrayList<>(bottomProject.getProjects()); + outputs.add(new Alias(new TinyIntLiteral((byte) 1), DML_FACTOR_NAME)); + return new LogicalProject<>(ImmutableList.copyOf(outputs), bottomProject.child()); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/DeltaCommandBundle.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/DeltaCommandBundle.java new file mode 100644 index 00000000000000..9cf612c276af78 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/DeltaCommandBundle.java @@ -0,0 +1,55 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv.ivm; + +import org.apache.doris.mtmv.BaseTableInfo; +import org.apache.doris.nereids.trees.plans.commands.Command; + +import java.util.Objects; + +/** + * One delta write command for a single changed base table. + * Produced by a per-pattern IVM Nereids rule and consumed by IvmDeltaExecutor. + */ +public class DeltaCommandBundle { + // the base table whose changes this bundle handles + private final BaseTableInfo baseTableInfo; + // the logical delta write command (INSERT / DELETE / MERGE INTO) + private final Command command; + + public DeltaCommandBundle(BaseTableInfo baseTableInfo, Command command) { + this.baseTableInfo = Objects.requireNonNull(baseTableInfo, "baseTableInfo can not be null"); + this.command = Objects.requireNonNull(command, "command can not be null"); + } + + public BaseTableInfo getBaseTableInfo() { + return baseTableInfo; + } + + public Command getCommand() { + return command; + } + + @Override + public String toString() { + return "DeltaCommandBundle{" + + "baseTableInfo=" + baseTableInfo + + ", command=" + command.getClass().getSimpleName() + + '}'; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/FallbackReason.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/FallbackReason.java new file mode 100644 index 00000000000000..ad8a42b2e23426 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/FallbackReason.java @@ -0,0 +1,32 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv.ivm; + +/** Reasons an incremental refresh may fall back to partition or full refresh. */ +public enum FallbackReason { + BINLOG_BROKEN, + STREAM_UNSUPPORTED, + SNAPSHOT_ALIGNMENT_UNSUPPORTED, + PLAN_PATTERN_UNSUPPORTED, + NON_DETERMINISTIC_ROW_ID, + OUTER_JOIN_RETRACTION_UNSUPPORTED, + PREVIOUS_RUN_INCOMPLETE, + INCREMENTAL_EXECUTION_FAILED, + AGG_UNSUPPORTED, + MIN_MAX_BOUNDARY_HIT +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmAggMeta.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmAggMeta.java new file mode 100644 index 00000000000000..a4b15e9991170f --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmAggMeta.java @@ -0,0 +1,145 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv.ivm; + +import org.apache.doris.nereids.trees.expressions.Slot; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; + +import java.util.List; +import java.util.Map; +import java.util.Objects; + +/** + * Metadata describing the aggregate IVM structure of a materialized view. + * Produced by IvmNormalizeMtmv when it processes a LogicalAggregate. + * Consumed by IvmDeltaRewriter to generate the delta computation + apply commands. + */ +public class IvmAggMeta { + + /** Supported aggregate types for IVM. */ + public enum AggType { + COUNT_STAR, + COUNT_EXPR, + SUM, + AVG, + MIN, + MAX + } + + /** + * Describes one aggregate target in the MV and its associated hidden state columns. + */ + public static class AggTarget { + private final int ordinal; + private final AggType aggType; + private final Slot visibleSlot; + // hidden state column slots, keyed by state type (e.g. "SUM", "COUNT") + private final Map hiddenStateSlots; + // the expression slots from the base scan that feed this aggregate + // (empty for COUNT_STAR) + private final List exprSlots; + + public AggTarget(int ordinal, AggType aggType, Slot visibleSlot, + Map hiddenStateSlots, List exprSlots) { + this.ordinal = ordinal; + this.aggType = Objects.requireNonNull(aggType); + this.visibleSlot = Objects.requireNonNull(visibleSlot); + this.hiddenStateSlots = ImmutableMap.copyOf(hiddenStateSlots); + this.exprSlots = ImmutableList.copyOf(exprSlots); + } + + public int getOrdinal() { + return ordinal; + } + + public AggType getAggType() { + return aggType; + } + + public Slot getVisibleSlot() { + return visibleSlot; + } + + public Map getHiddenStateSlots() { + return hiddenStateSlots; + } + + public Slot getHiddenStateSlot(String stateType) { + return hiddenStateSlots.get(stateType); + } + + public List getExprSlots() { + return exprSlots; + } + + @Override + public String toString() { + return "AggTarget{ordinal=" + ordinal + ", type=" + aggType + + ", visible=" + visibleSlot.getName() + + ", hidden=" + hiddenStateSlots.keySet() + "}"; + } + } + + private final boolean scalarAgg; + private final List groupKeySlots; + private final Slot groupCountSlot; + private final List aggTargets; + + public IvmAggMeta(boolean scalarAgg, List groupKeySlots, + Slot groupCountSlot, List aggTargets) { + this.scalarAgg = scalarAgg; + this.groupKeySlots = ImmutableList.copyOf(groupKeySlots); + this.groupCountSlot = Objects.requireNonNull(groupCountSlot); + this.aggTargets = ImmutableList.copyOf(aggTargets); + } + + /** True if this is a scalar aggregate (no GROUP BY). */ + public boolean isScalarAgg() { + return scalarAgg; + } + + /** The group-by key slots (empty for scalar aggregate). */ + public List getGroupKeySlots() { + return groupKeySlots; + } + + /** The hidden slot for group-level count (__DORIS_IVM_AGG_COUNT_COL__). */ + public Slot getGroupCountSlot() { + return groupCountSlot; + } + + /** All aggregate targets with their hidden state mappings. */ + public List getAggTargets() { + return aggTargets; + } + + /** Returns true if any aggregate target is MIN or MAX. */ + public boolean hasMinMax() { + return aggTargets.stream().anyMatch(t -> + t.getAggType() == AggType.MIN || t.getAggType() == AggType.MAX); + } + + @Override + public String toString() { + return "IvmAggMeta{scalar=" + scalarAgg + + ", groupKeys=" + groupKeySlots.size() + + ", targets=" + aggTargets + "}"; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmDeltaExecutor.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmDeltaExecutor.java new file mode 100644 index 00000000000000..13614f75345fcf --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmDeltaExecutor.java @@ -0,0 +1,52 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv.ivm; + +import org.apache.doris.common.AnalysisException; +import org.apache.doris.mtmv.MTMVPlanUtil; +import org.apache.doris.nereids.StatementContext; + +import java.util.List; + +/** + * Executes IVM delta command bundles against the MV target table. + */ +public class IvmDeltaExecutor { + + public void execute(IvmRefreshContext context, List bundles) + throws AnalysisException { + for (DeltaCommandBundle bundle : bundles) { + executeBundle(context, bundle); + } + } + + private void executeBundle(IvmRefreshContext context, DeltaCommandBundle bundle) + throws AnalysisException { + StatementContext stmtCtx = new StatementContext(); + String auditStmt = String.format("IVM delta refresh, mvName: %s, baseTable: %s", + context.getMtmv().getName(), bundle.getBaseTableInfo()); + try { + // normalPlan had applied ivm normal mtmv plan rule, so no need enable this rule then. + MTMVPlanUtil.executeCommand(context.getMtmv(), bundle.getCommand(), + stmtCtx, auditStmt, false); + } catch (Exception e) { + throw new AnalysisException("IVM delta execution failed for " + + bundle.getBaseTableInfo() + ": " + e.getMessage(), e); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmDeltaRewriteContext.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmDeltaRewriteContext.java new file mode 100644 index 00000000000000..a244755d8f49c7 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmDeltaRewriteContext.java @@ -0,0 +1,51 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv.ivm; + +import org.apache.doris.catalog.MTMV; +import org.apache.doris.qe.ConnectContext; + +import java.util.Objects; + +/** + * Context passed to {@link IvmDeltaRewriter} during delta command construction. + */ +public class IvmDeltaRewriteContext { + private final MTMV mtmv; + private final ConnectContext connectContext; + private final IvmNormalizeResult normalizeResult; + + public IvmDeltaRewriteContext(MTMV mtmv, ConnectContext connectContext, IvmNormalizeResult normalizeResult) { + this.mtmv = Objects.requireNonNull(mtmv, "mtmv can not be null"); + this.connectContext = Objects.requireNonNull(connectContext, "connectContext can not be null"); + this.normalizeResult = normalizeResult; + } + + public MTMV getMtmv() { + return mtmv; + } + + public ConnectContext getConnectContext() { + return connectContext; + } + + /** Returns the IVM normalize result, or null if this is a non-agg scan-only MV. */ + public IvmNormalizeResult getNormalizeResult() { + return normalizeResult; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmDeltaRewriter.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmDeltaRewriter.java new file mode 100644 index 00000000000000..adb72c5601a291 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmDeltaRewriter.java @@ -0,0 +1,65 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv.ivm; + +import org.apache.doris.mtmv.BaseTableInfo; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.Command; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; + +import java.util.Collections; +import java.util.List; + +/** + * Transforms a normalized MV plan into delta INSERT commands. + * + *

Supported patterns: + *

    + *
  • SCAN_ONLY: ResultSink → Project → OlapScan
    • + *
  • PROJECT_SCAN: ResultSink → Project → Project → OlapScan
    • + *
+ * + *

Aggregate plans are not yet supported and will be routed to + * {@code AggDeltaStrategy} once strategy routing is implemented. + */ +public class IvmDeltaRewriter { + + /** + * Rewrites the normalized plan into a list of delta command bundles. + * Currently produces exactly one INSERT bundle for the single base table scan. + */ + public List rewrite(Plan normalizedPlan, IvmDeltaRewriteContext ctx) { + Plan queryPlan = AbstractDeltaStrategy.stripResultSink(normalizedPlan); + rejectAggPlan(queryPlan); + LogicalOlapScan scan = AbstractDeltaStrategy.extractScan(queryPlan); + BaseTableInfo baseTableInfo = AbstractDeltaStrategy.extractBaseTableInfo(scan); + Command insertCommand = AbstractDeltaStrategy.buildInsertCommand(queryPlan, ctx); + return Collections.singletonList(new DeltaCommandBundle(baseTableInfo, insertCommand)); + } + + /** Guard: reject aggregate plans until AggDeltaStrategy routing is wired in. */ + private void rejectAggPlan(Plan plan) { + if (plan.containsType(LogicalAggregate.class)) { + throw new AnalysisException( + "IVM delta rewrite does not yet support aggregate plans; " + + "AggDeltaStrategy routing is not yet implemented"); + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmDeltaStrategy.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmDeltaStrategy.java new file mode 100644 index 00000000000000..15048c2470ca18 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmDeltaStrategy.java @@ -0,0 +1,38 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv.ivm; + +import org.apache.doris.nereids.trees.plans.Plan; + +import java.util.List; + +/** + * Strategy interface for IVM delta rewriting. + * Each strategy handles a specific normalized plan pattern (e.g. scan-only, agg). + */ +public interface IvmDeltaStrategy { + + /** + * Rewrites a normalized MV plan into delta command bundles. + * + * @param normalizedPlan the plan produced by IvmNormalizeMtmv (with ResultSink stripped) + * @param ctx rewrite context carrying MTMV metadata and normalize result + * @return one or more delta command bundles for execution + */ + List rewrite(Plan normalizedPlan, IvmDeltaRewriteContext ctx); +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmInfo.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmInfo.java new file mode 100644 index 00000000000000..117c6f1675c399 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmInfo.java @@ -0,0 +1,64 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv.ivm; + +import org.apache.doris.mtmv.BaseTableInfo; + +import com.google.common.collect.Maps; +import com.google.gson.annotations.SerializedName; + +import java.util.Map; + +/** + * Thin persistent IVM metadata stored on MTMV. + */ +public class IvmInfo { + @SerializedName("bb") + private boolean binlogBroken = false; + + @SerializedName("bs") + private Map baseTableStreams; + + public IvmInfo() { + this.baseTableStreams = Maps.newHashMap(); + } + + public boolean isBinlogBroken() { + return binlogBroken; + } + + public void setBinlogBroken(boolean binlogBroken) { + this.binlogBroken = binlogBroken; + } + + public Map getBaseTableStreams() { + return baseTableStreams; + } + + public void setBaseTableStreams(Map baseTableStreams) { + this.baseTableStreams = baseTableStreams; + } + + @Override + public String toString() { + return "IvmInfo{" + + "binlogBroken=" + binlogBroken + + ", baseTableStreams=" + baseTableStreams + + '}'; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmNormalizeResult.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmNormalizeResult.java new file mode 100644 index 00000000000000..f73b30e89cbf24 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmNormalizeResult.java @@ -0,0 +1,72 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv.ivm; + +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.Plan; + +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Holds IVM-related state produced during a Nereids run for an incremental MV. + * Stored as Optional in CascadesContext — absent when IVM rewrite is not active. + * + * rowIdDeterminism: maps each injected row-id slot to whether it is deterministic. + * - deterministic (true): MOW table — row-id = hash(unique keys), stable across refreshes + * - non-deterministic (false): DUP_KEYS table — row-id = random 128-bit per insert + * + * normalizedPlan: the plan tree after IvmNormalizeMtmv has injected row-id columns. + * Stored here so that IvmRefreshManager can retrieve it for external delta rewriting. + */ +public class IvmNormalizeResult { + // insertion-ordered so row-ids appear in scan order + private final Map rowIdDeterminism = new LinkedHashMap<>(); + private Plan normalizedPlan; + private IvmAggMeta aggMeta; + + public void addRowId(Slot rowIdSlot, boolean deterministic) { + rowIdDeterminism.put(rowIdSlot, deterministic); + } + + public Map getRowIdDeterminism() { + return rowIdDeterminism; + } + + public Plan getNormalizedPlan() { + return normalizedPlan; + } + + public void setNormalizedPlan(Plan normalizedPlan) { + this.normalizedPlan = normalizedPlan; + } + + /** Returns the aggregate IVM metadata, or null if the MV is not an agg MV. */ + public IvmAggMeta getAggMeta() { + return aggMeta; + } + + public void setAggMeta(IvmAggMeta aggMeta) { + this.aggMeta = aggMeta; + } + + /** Returns true if this MV uses aggregate IVM. */ + public boolean isAggMv() { + return aggMeta != null; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmRefreshContext.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmRefreshContext.java new file mode 100644 index 00000000000000..b1c256f0dca9e2 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmRefreshContext.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv.ivm; + +import org.apache.doris.catalog.MTMV; +import org.apache.doris.mtmv.MTMVRefreshContext; +import org.apache.doris.qe.ConnectContext; + +import java.util.Objects; + +/** + * Shared immutable context for one FE-side incremental refresh attempt. + */ +public class IvmRefreshContext { + private final MTMV mtmv; + private final ConnectContext connectContext; + private final MTMVRefreshContext mtmvRefreshContext; + + public IvmRefreshContext(MTMV mtmv, ConnectContext connectContext, MTMVRefreshContext mtmvRefreshContext) { + this.mtmv = Objects.requireNonNull(mtmv, "mtmv can not be null"); + this.connectContext = Objects.requireNonNull(connectContext, "connectContext can not be null"); + this.mtmvRefreshContext = Objects.requireNonNull(mtmvRefreshContext, "mtmvRefreshContext can not be null"); + } + + public MTMV getMtmv() { + return mtmv; + } + + public ConnectContext getConnectContext() { + return connectContext; + } + + public MTMVRefreshContext getMtmvRefreshContext() { + return mtmvRefreshContext; + } + + @Override + public boolean equals(Object o) { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + IvmRefreshContext that = (IvmRefreshContext) o; + return Objects.equals(mtmv, that.mtmv) + && Objects.equals(connectContext, that.connectContext) + && Objects.equals(mtmvRefreshContext, that.mtmvRefreshContext); + } + + @Override + public int hashCode() { + return Objects.hash(mtmv, connectContext, mtmvRefreshContext); + } + + @Override + public String toString() { + return "IvmRefreshContext{" + + "mtmv=" + mtmv.getName() + + ", mtmvRefreshContext=" + mtmvRefreshContext + + '}'; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmRefreshManager.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmRefreshManager.java new file mode 100644 index 00000000000000..16f5ff368fbc4a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmRefreshManager.java @@ -0,0 +1,183 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv.ivm; + +import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.mtmv.BaseTableInfo; +import org.apache.doris.mtmv.MTMVAnalyzeQueryInfo; +import org.apache.doris.mtmv.MTMVPlanUtil; +import org.apache.doris.mtmv.MTMVRefreshContext; +import org.apache.doris.mtmv.MTMVRelation; +import org.apache.doris.mtmv.MTMVUtil; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.annotations.VisibleForTesting; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.Set; + +/** + * Minimal orchestration entry point for incremental refresh. + */ +public class IvmRefreshManager { + private static final Logger LOG = LogManager.getLogger(IvmRefreshManager.class); + private final IvmDeltaExecutor deltaExecutor; + + public IvmRefreshManager() { + this(new IvmDeltaExecutor()); + } + + @VisibleForTesting + IvmRefreshManager(IvmDeltaExecutor deltaExecutor) { + this.deltaExecutor = Objects.requireNonNull(deltaExecutor, "deltaExecutor can not be null"); + } + + public IvmRefreshResult doRefresh(MTMV mtmv) { + Objects.requireNonNull(mtmv, "mtmv can not be null"); + IvmRefreshResult precheckResult = precheck(mtmv); + if (!precheckResult.isSuccess()) { + LOG.warn("IVM precheck failed for mv={}, result={}", mtmv.getName(), precheckResult); + return precheckResult; + } + final IvmRefreshContext context; + try { + context = buildRefreshContext(mtmv); + } catch (Exception e) { + IvmRefreshResult result = IvmRefreshResult.fallback( + FallbackReason.SNAPSHOT_ALIGNMENT_UNSUPPORTED, e.getMessage()); + LOG.warn("IVM context build failed for mv={}, result={}", mtmv.getName(), result); + return result; + } + return doRefreshInternal(context); + } + + @VisibleForTesting + IvmRefreshResult precheck(MTMV mtmv) { + Objects.requireNonNull(mtmv, "mtmv can not be null"); + if (mtmv.getIvmInfo().isBinlogBroken()) { + return IvmRefreshResult.fallback(FallbackReason.BINLOG_BROKEN, + "Stream binlog is marked as broken"); + } + // return checkStreamSupport(mtmv); + return IvmRefreshResult.success(); + } + + @VisibleForTesting + IvmRefreshContext buildRefreshContext(MTMV mtmv) throws Exception { + ConnectContext connectContext = MTMVPlanUtil.createMTMVContext(mtmv, + MTMVPlanUtil.DISABLE_RULES_WHEN_RUN_MTMV_TASK); + MTMVRefreshContext mtmvRefreshContext = MTMVRefreshContext.buildContext(mtmv); + return new IvmRefreshContext(mtmv, connectContext, mtmvRefreshContext); + } + + @VisibleForTesting + List analyzeDeltaCommandBundles(IvmRefreshContext context) throws Exception { + MTMVAnalyzeQueryInfo queryInfo = MTMVPlanUtil.analyzeQueryWithSql( + context.getMtmv(), context.getConnectContext(), true); + IvmNormalizeResult normalizeResult = queryInfo.getIvmNormalizeResult(); + Plan normalizedPlan = queryInfo.getIvmNormalizedPlan(); + if (normalizedPlan == null) { + return Collections.emptyList(); + } + IvmDeltaRewriteContext rewriteCtx = new IvmDeltaRewriteContext( + context.getMtmv(), context.getConnectContext(), normalizeResult); + return new IvmDeltaRewriter().rewrite(normalizedPlan, rewriteCtx); + } + + private IvmRefreshResult doRefreshInternal(IvmRefreshContext context) { + Objects.requireNonNull(context, "context can not be null"); + + // Run Nereids with IVM rewrite enabled — per-pattern delta rules write bundles to CascadesContext + List bundles; + try { + bundles = analyzeDeltaCommandBundles(context); + } catch (Exception e) { + IvmRefreshResult result = IvmRefreshResult.fallback( + FallbackReason.PLAN_PATTERN_UNSUPPORTED, e.getMessage()); + LOG.warn("IVM plan analysis failed for mv={}, result={}", context.getMtmv().getName(), result); + return result; + } + + if (bundles == null || bundles.isEmpty()) { + IvmRefreshResult result = IvmRefreshResult.fallback( + FallbackReason.PLAN_PATTERN_UNSUPPORTED, "No IVM delta rule matched the MV define plan"); + LOG.warn("IVM no delta command bundles for mv={}, result={}", context.getMtmv().getName(), result); + return result; + } + + try { + deltaExecutor.execute(context, bundles); + return IvmRefreshResult.success(); + } catch (Exception e) { + IvmRefreshResult result = IvmRefreshResult.fallback( + FallbackReason.INCREMENTAL_EXECUTION_FAILED, e.getMessage()); + LOG.warn("IVM execution failed for mv={}, result={}", context.getMtmv().getName(), result, e); + return result; + } + } + + private IvmRefreshResult checkStreamSupport(MTMV mtmv) { + MTMVRelation relation = mtmv.getRelation(); + if (relation == null) { + return IvmRefreshResult.fallback(FallbackReason.STREAM_UNSUPPORTED, + "No base table relation found for incremental refresh"); + } + Set baseTables = relation.getBaseTablesOneLevelAndFromView(); + if (baseTables == null || baseTables.isEmpty()) { + return IvmRefreshResult.fallback(FallbackReason.STREAM_UNSUPPORTED, + "No base tables found for incremental refresh"); + } + Map baseTableStreams = mtmv.getIvmInfo().getBaseTableStreams(); + if (baseTableStreams == null || baseTableStreams.isEmpty()) { + return IvmRefreshResult.fallback(FallbackReason.STREAM_UNSUPPORTED, + "No stream bindings are registered for this materialized view"); + } + for (BaseTableInfo baseTableInfo : baseTables) { + IvmStreamRef streamRef = baseTableStreams.get(baseTableInfo); + if (streamRef == null) { + return IvmRefreshResult.fallback(FallbackReason.STREAM_UNSUPPORTED, + "No stream binding found for base table: " + baseTableInfo); + } + if (streamRef.getStreamType() != StreamType.OLAP) { + return IvmRefreshResult.fallback(FallbackReason.STREAM_UNSUPPORTED, + "Only OLAP base table streams are supported for incremental refresh: " + baseTableInfo); + } + final TableIf table; + try { + table = MTMVUtil.getTable(baseTableInfo); + } catch (Exception e) { + return IvmRefreshResult.fallback(FallbackReason.STREAM_UNSUPPORTED, + "Failed to resolve base table metadata for incremental refresh: " + + baseTableInfo + ", reason=" + e.getMessage()); + } + if (!(table instanceof OlapTable)) { + return IvmRefreshResult.fallback(FallbackReason.STREAM_UNSUPPORTED, + "Only OLAP base tables are supported for incremental refresh: " + baseTableInfo); + } + } + return IvmRefreshResult.success(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmRefreshResult.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmRefreshResult.java new file mode 100644 index 00000000000000..36b229b07429f2 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmRefreshResult.java @@ -0,0 +1,67 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv.ivm; + +import java.util.Objects; + +/** Result of one FE-side incremental refresh attempt. */ +public class IvmRefreshResult { + private final boolean success; + private final FallbackReason fallbackReason; + private final String detailMessage; + + private IvmRefreshResult(boolean success, FallbackReason fallbackReason, String detailMessage) { + this.success = success; + this.fallbackReason = fallbackReason; + this.detailMessage = detailMessage; + } + + public static IvmRefreshResult success() { + return new IvmRefreshResult(true, null, null); + } + + public static IvmRefreshResult fallback(FallbackReason fallbackReason, String detailMessage) { + return new IvmRefreshResult(false, + Objects.requireNonNull(fallbackReason, "fallbackReason can not be null"), + Objects.requireNonNull(detailMessage, "detailMessage can not be null")); + } + + public boolean isSuccess() { + return success; + } + + public FallbackReason getFallbackReason() { + return fallbackReason; + } + + public String getDetailMessage() { + return detailMessage; + } + + @Override + public String toString() { + if (success) { + return "IvmRefreshResult{success=true}"; + } + return "IvmRefreshResult{" + + "success=false" + + ", fallbackReason=" + fallbackReason + + ", detailMessage='" + detailMessage + '\'' + + '}'; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmStreamRef.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmStreamRef.java new file mode 100644 index 00000000000000..5bf7879f6faa2a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmStreamRef.java @@ -0,0 +1,76 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv.ivm; + +import com.google.common.collect.Maps; +import com.google.gson.annotations.SerializedName; + +import java.util.Map; + +/** + * Thin persistent binding between one base table and its stream. + */ +public class IvmStreamRef { + @SerializedName("st") + private StreamType streamType; + + @SerializedName("cid") + private String consumerId; + + @SerializedName("p") + private Map properties; + + public IvmStreamRef() { + this.properties = Maps.newHashMap(); + } + + public IvmStreamRef(StreamType streamType, String consumerId, Map properties) { + this.streamType = streamType; + this.consumerId = consumerId; + this.properties = properties != null ? properties : Maps.newHashMap(); + } + + public StreamType getStreamType() { + return streamType; + } + + public void setStreamType(StreamType streamType) { + this.streamType = streamType; + } + + public String getConsumerId() { + return consumerId; + } + + public void setConsumerId(String consumerId) { + this.consumerId = consumerId; + } + + public Map getProperties() { + return properties; + } + + @Override + public String toString() { + return "IvmStreamRef{" + + "streamType=" + streamType + + ", consumerId='" + consumerId + '\'' + + ", properties=" + properties + + '}'; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmUtil.java new file mode 100644 index 00000000000000..5e1db5d0e9fbc5 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/IvmUtil.java @@ -0,0 +1,85 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv.ivm; + +import org.apache.doris.catalog.Column; +import org.apache.doris.nereids.trees.plans.commands.info.ColumnDefinition; +import org.apache.doris.nereids.types.BigIntType; +import org.apache.doris.nereids.types.DataType; + +import java.util.Optional; + +/** + * IVM (Incremental View Maintenance) utility class. + * Centralizes IVM hidden column detection, naming, and ColumnDefinition factories. + * Column name constants are defined in {@link Column}. + */ +public class IvmUtil { + + public static boolean isIvmHiddenColumn(String columnName) { + return columnName != null && columnName.startsWith(Column.IVM_HIDDEN_COLUMN_PREFIX); + } + + /** + * Generates a hidden column name for an IVM aggregate state. + * Format: __DORIS_IVM_AGG_{ordinal}_{stateType}_COL__ + * Example: __DORIS_IVM_AGG_2_SUM_COL__, __DORIS_IVM_AGG_2_COUNT_COL__ + * + * @param ordinal the 0-based ordinal of the aggregate target in the MV query + * @param stateType the state type (SUM, COUNT, etc.) + */ + public static String ivmAggHiddenColumnName(int ordinal, String stateType) { + return Column.IVM_HIDDEN_COLUMN_PREFIX + "AGG_" + ordinal + "_" + stateType + "_COL__"; + } + + /** Creates a hidden ColumnDefinition for the IVM row-id column. */ + public static ColumnDefinition newIvmRowIdColumnDefinition(DataType type, boolean isNullable) { + ColumnDefinition columnDefinition = new ColumnDefinition( + Column.IVM_ROW_ID_COL, type, false, null, isNullable, Optional.empty(), + "ivm row id hidden column", false); + columnDefinition.setEnableAddHiddenColumn(true); + return columnDefinition; + } + + /** + * Creates a hidden ColumnDefinition for the IVM group-level count. + * Type is always BigInt (same as COUNT result), non-nullable. + */ + public static ColumnDefinition newIvmCountColumnDefinition() { + ColumnDefinition columnDefinition = new ColumnDefinition( + Column.IVM_AGG_COUNT_COL, BigIntType.INSTANCE, false, null, false, Optional.empty(), + "ivm group count hidden column", false); + columnDefinition.setEnableAddHiddenColumn(true); + return columnDefinition; + } + + /** + * Creates a hidden ColumnDefinition for an IVM aggregate state. + * + * @param name the hidden column name (e.g. __DORIS_IVM_AGG_0_SUM_COL__) + * @param type the data type of this state column + * @param isNullable whether this state column can be null + */ + public static ColumnDefinition newIvmAggHiddenColumnDefinition(String name, DataType type, boolean isNullable) { + ColumnDefinition columnDefinition = new ColumnDefinition( + name, type, false, null, isNullable, Optional.empty(), + "ivm aggregate hidden column", false); + columnDefinition.setEnableAddHiddenColumn(true); + return columnDefinition; + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/StreamType.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/StreamType.java new file mode 100644 index 00000000000000..ea7a5a9793f99a --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/ivm/StreamType.java @@ -0,0 +1,25 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv.ivm; + +/** Type of change stream backing a base table for IVM. */ +public enum StreamType { + OLAP, + PAIMON, + ICEBERG +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java index aa98abdd4b0c22..1e55ebb36ae065 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/CascadesContext.java @@ -19,6 +19,7 @@ import org.apache.doris.common.IdGenerator; import org.apache.doris.common.Pair; +import org.apache.doris.mtmv.ivm.IvmNormalizeResult; import org.apache.doris.nereids.analyzer.Scope; import org.apache.doris.nereids.hint.Hint; import org.apache.doris.nereids.jobs.Job; @@ -91,6 +92,8 @@ public class CascadesContext implements ScheduleContext { // in analyze/rewrite stage, the plan will storage in this field private Plan plan; + // present when IVM rewrite is active; absent otherwise + private Optional ivmNormalizeResult = Optional.empty(); private Optional currentRootRewriteJobContext; // in optimize stage, the plan will storage in the memo private Memo memo; @@ -365,6 +368,14 @@ public Plan getRewritePlan() { return plan; } + public Optional getIvmNormalizeResult() { + return ivmNormalizeResult; + } + + public void setIvmNormalizeResult(IvmNormalizeResult ivmNormalizeResult) { + this.ivmNormalizeResult = Optional.ofNullable(ivmNormalizeResult); + } + public void setRewritePlan(Plan plan) { this.plan = plan; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index cb2ecdba15bf95..8a8ed4435863f3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -91,6 +91,7 @@ import org.apache.doris.nereids.rules.rewrite.MergeLimits; import org.apache.doris.nereids.stats.StatsErrorEstimator; import org.apache.doris.nereids.trees.expressions.AggregateExpression; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.CTEId; import org.apache.doris.nereids.trees.expressions.EqualPredicate; import org.apache.doris.nereids.trees.expressions.ExprId; @@ -307,11 +308,13 @@ public PhysicalPlanTranslator(PlanTranslatorContext context, StatsErrorEstimator */ public PlanFragment translatePlan(PhysicalPlan physicalPlan) { PlanFragment rootFragment = physicalPlan.accept(this, context); - if (CollectionUtils.isEmpty(rootFragment.getOutputExprs())) { - List outputExprs = Lists.newArrayList(); - physicalPlan.getOutput().stream().map(Slot::getExprId) - .forEach(exprId -> outputExprs.add(context.findSlotRef(exprId))); - rootFragment.setOutputExprs(outputExprs); + boolean canTranslateRootOutput = physicalPlan.getOutput().stream() + .allMatch(slot -> context.findSlotRef(slot.getExprId()) != null); + // Prefer the final physical output slots when they are fully bound. + // If they are not bound, preserve the explicit root fragment output exprs installed by + // child translation, e.g. for defer materialize topn followed by a projection. + if (canTranslateRootOutput || CollectionUtils.isEmpty(rootFragment.getOutputExprs())) { + rootFragment.setOutputExprs(translateOutputExprs(physicalPlan.getOutput())); } Collections.reverse(context.getPlanFragments()); if (context.getSessionVariable() != null && context.getSessionVariable().forbidUnknownColStats) { @@ -401,6 +404,7 @@ public PlanFragment visitPhysicalDistribute(PhysicalDistribute d // its source partition is targetDataPartition. and outputPartition is UNPARTITIONED now, will be set when // visit its SinkNode PlanFragment downstreamFragment = new PlanFragment(context.nextFragmentId(), exchangeNode, targetDataPartition); + downstreamFragment.setOutputExprs(translateOutputExprs(distribute.getOutput())); if (targetDistribution instanceof DistributionSpecGather || targetDistribution instanceof DistributionSpecStorageGather) { // gather to one instance @@ -679,9 +683,7 @@ public PlanFragment visitPhysicalFileSink(PhysicalFileSink fileS fileSink.getProperties() ); - List outputExprs = Lists.newArrayList(); - fileSink.getOutput().stream().map(Slot::getExprId) - .forEach(exprId -> outputExprs.add(context.findSlotRef(exprId))); + List outputExprs = translateOutputExprs(fileSink.getOutput()); sinkFragment.setOutputExprs(outputExprs); // generate colLabels @@ -3074,6 +3076,20 @@ private List collectGroupBySlots(List groupByExpressi for (Expression e : groupByExpressions) { if (e instanceof SlotReference && outputExpressions.stream().anyMatch(o -> o.anyMatch(e::equals))) { groupSlots.add((SlotReference) e); + } else if (!(e instanceof SlotReference)) { + SlotReference outputAliasSlot = outputExpressions.stream() + .filter(Alias.class::isInstance) + .map(Alias.class::cast) + .filter(outputAlias -> outputAlias.child().equals(e)) + .map(Alias::toSlot) + .map(SlotReference.class::cast) + .findFirst() + .orElse(null); + if (outputAliasSlot != null) { + groupSlots.add(outputAliasSlot); + continue; + } + groupSlots.add(new SlotReference(e.toSql(), e.getDataType(), e.nullable(), ImmutableList.of())); } else { groupSlots.add(new SlotReference(e.toSql(), e.getDataType(), e.nullable(), ImmutableList.of())); } @@ -3271,6 +3287,18 @@ private boolean checkPushSort(SortNode sortNode, OlapTable olapTable) { return true; } + private List translateOutputExprs(List outputSlots) { + List outputExprs = Lists.newArrayListWithCapacity(outputSlots.size()); + for (Slot slot : outputSlots) { + SlotRef slotRef = context.findSlotRef(slot.getExprId()); + Preconditions.checkNotNull(slotRef, + "missing SlotRef for ExprId %s (%s) during output expr translation", + slot.getExprId(), slot); + outputExprs.add(slotRef); + } + return outputExprs; + } + private boolean isComplexDataType(DataType dataType) { return dataType instanceof ArrayType || dataType instanceof MapType || dataType instanceof JsonType || dataType instanceof StructType; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java index 0400e50c792b10..cb706259b913eb 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Analyzer.java @@ -51,6 +51,7 @@ import org.apache.doris.nereids.rules.analysis.SubqueryToApply; import org.apache.doris.nereids.rules.analysis.VariableToLiteral; import org.apache.doris.nereids.rules.rewrite.AdjustNullable; +import org.apache.doris.nereids.rules.rewrite.IvmNormalizeMtmv; import org.apache.doris.nereids.rules.rewrite.MergeFilters; import org.apache.doris.nereids.rules.rewrite.SemiJoinCommute; import org.apache.doris.nereids.rules.rewrite.SimplifyAggGroupBy; @@ -220,6 +221,7 @@ private static List buildAnalyzerJobs() { // merge normal filter and hidden column filter new MergeFilters() ), + custom(RuleType.IVM_NORMALIZE_MTMV, IvmNormalizeMtmv::new), // for cte: analyze producer -> analyze consumer -> rewrite consumer -> rewrite producer, // in order to ensure cte consumer had right nullable attribute, need adjust nullable at analyze phase. custom(RuleType.ADJUST_NULLABLE, () -> new AdjustNullable(true)) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java index 2c4b6593d5d206..efd9220562c96b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/RuleType.java @@ -299,6 +299,7 @@ public enum RuleType { ELIMINATE_NOT_NULL(RuleTypeClass.REWRITE), ELIMINATE_UNNECESSARY_PROJECT(RuleTypeClass.REWRITE), RECORD_PLAN_FOR_MV_PRE_REWRITE(RuleTypeClass.REWRITE), + IVM_NORMALIZE_MTMV(RuleTypeClass.REWRITE), ELIMINATE_OUTER_JOIN(RuleTypeClass.REWRITE), ELIMINATE_MARK_JOIN(RuleTypeClass.REWRITE), ELIMINATE_GROUP_BY(RuleTypeClass.REWRITE), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java index 4e02766ef146e8..518fe848d5faf5 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java @@ -26,6 +26,7 @@ import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.GeneratedColumnInfo; import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.MaterializedIndexMeta; import org.apache.doris.catalog.OlapTable; import org.apache.doris.catalog.Partition; @@ -44,6 +45,7 @@ import org.apache.doris.datasource.maxcompute.MaxComputeExternalDatabase; import org.apache.doris.datasource.maxcompute.MaxComputeExternalTable; import org.apache.doris.dictionary.Dictionary; +import org.apache.doris.mtmv.MTMVRefreshEnum.RefreshMethod; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.Scope; @@ -189,9 +191,11 @@ private Plan bindOlapTableSink(MatchingContext> ctx) { boolean needExtraSeqCol = isPartialUpdate && !childHasSeqCol && table.hasSequenceCol() && table.getSequenceMapCol() != null && sink.getColNames().contains(table.getSequenceMapCol()); + Set missingIvmHiddenColumns = getMissingIvmHiddenColumns(table, sink.getColNames(), child); // 1. bind target columns: from sink's column names to target tables' Columns Pair, Integer> bindColumnsResult = bindTargetColumns(table, sink.getColNames(), childHasSeqCol, needExtraSeqCol, + missingIvmHiddenColumns, sink.getDMLCommandType() == DMLCommandType.GROUP_COMMIT); List bindColumns = bindColumnsResult.first; int extraColumnsNum = bindColumnsResult.second; @@ -277,7 +281,7 @@ private Plan bindOlapTableSink(MatchingContext> ctx) { } Map columnToOutput = getColumnToOutput( - ctx, table, isPartialUpdate, boundSink, child); + ctx, table, isPartialUpdate, boundSink, child, missingIvmHiddenColumns); LogicalProject fullOutputProject = getOutputProjectByCoercion( table.getFullSchema(), child, columnToOutput); List columns = new ArrayList<>(table.getFullSchema().size()); @@ -365,14 +369,13 @@ private LogicalProject getOutputProjectByCoercion(List tableSchema, L private static Map getColumnToOutput( MatchingContext> ctx, - TableIf table, boolean isPartialUpdate, LogicalTableSink boundSink, LogicalPlan child) { + TableIf table, boolean isPartialUpdate, LogicalTableSink boundSink, LogicalPlan child, + Set missingIvmHiddenColumns) { // we need to insert all the columns of the target table // although some columns are not mentions. // so we add a projects to supply the default value. - Map columnToChildOutput = Maps.newHashMap(); - for (int i = 0; i < child.getOutput().size(); ++i) { - columnToChildOutput.put(boundSink.getCols().get(i), child.getOutput().get(i)); - } + Map columnToChildOutput = getColumnToChildOutput(boundSink, child, + missingIvmHiddenColumns); Map columnToOutput = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); Map columnToReplaced = Maps.newTreeMap(String.CASE_INSENSITIVE_ORDER); Map replaceMap = Maps.newHashMap(); @@ -425,6 +428,12 @@ private static Map getColumnToOutput( columnToReplaced.put(column.getName(), seqColumn.toSlot()); replaceMap.put(seqColumn.toSlot(), seqColumn.child(0)); } + } else if (missingIvmHiddenColumns.contains(column.getName())) { + Alias output = new Alias(new NullLiteral(DataType.fromCatalogType(column.getType())), + column.getName()); + columnToOutput.put(column.getName(), output); + columnToReplaced.put(column.getName(), output.toSlot()); + replaceMap.put(output.toSlot(), output.child()); } else if (isPartialUpdate) { // If the current load is a partial update, the values of unmentioned // columns will be filled in SegmentWriter. And the output of sink node @@ -559,6 +568,34 @@ private static Map getColumnToOutput( return columnToOutput; } + private static Map getColumnToChildOutput(LogicalTableSink boundSink, LogicalPlan child, + Set missingIvmHiddenColumns) { + Map columnToChildOutput = Maps.newHashMap(); + if (missingIvmHiddenColumns.isEmpty()) { + for (int i = 0; i < child.getOutput().size(); ++i) { + columnToChildOutput.put(boundSink.getCols().get(i), child.getOutput().get(i)); + } + return columnToChildOutput; + } + + int childIdx = 0; + for (Column column : boundSink.getCols()) { + if (childIdx >= child.getOutput().size()) { + break; + } + if (missingIvmHiddenColumns.contains(column.getName())) { + continue; + } + NamedExpression childOutput = child.getOutput().get(childIdx); + columnToChildOutput.put(column, childOutput); + childIdx++; + } + if (childIdx != child.getOutput().size()) { + throw new AnalysisException("insert into cols should be corresponding to the query output"); + } + return columnToChildOutput; + } + private Plan bindBlackHoleSink(MatchingContext> ctx) { UnboundBlackholeSink sink = ctx.root; LogicalPlan child = ((LogicalPlan) sink.child()); @@ -687,7 +724,7 @@ private Plan bindHiveTableSink(MatchingContext> ctx) throw new AnalysisException("insert into cols should be corresponding to the query output"); } Map columnToOutput = getColumnToOutput(ctx, table, false, - boundSink, child); + boundSink, child, Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER)); LogicalProject fullOutputProject = getOutputProjectByCoercion(table.getFullSchema(), child, columnToOutput); return boundSink.withChildAndUpdateOutput(fullOutputProject); } @@ -763,7 +800,7 @@ private Plan bindIcebergTableSink(MatchingContext> } Map columnToOutput = getColumnToOutput(ctx, table, false, - boundSink, child); + boundSink, child, Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER)); // For static partition columns, add constant expressions from PARTITION clause // This ensures partition column values are written to the data file @@ -888,7 +925,7 @@ private Plan bindMaxComputeTableSink(MatchingContext columnToOutput = getColumnToOutput(ctx, table, false, - boundSink, child); + boundSink, child, Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER)); LogicalProject fullOutputProject = getOutputProjectByCoercion(table.getFullSchema(), child, columnToOutput); return boundSink.withChildAndUpdateOutput(fullOutputProject); } @@ -1108,7 +1145,8 @@ private List bindPartitionIds(OlapTable table, List partitions, bo // bindTargetColumns means bind sink node's target columns' names to target table's columns private Pair, Integer> bindTargetColumns(OlapTable table, List colsName, - boolean childHasSeqCol, boolean needExtraSeqCol, boolean isGroupCommit) { + boolean childHasSeqCol, boolean needExtraSeqCol, Set missingIvmHiddenColumns, + boolean isGroupCommit) { // if the table set sequence column in stream load phase, the sequence map column is null, we query it. if (colsName.isEmpty()) { // ATTN: group commit without column list should return all base index column @@ -1117,7 +1155,7 @@ private Pair, Integer> bindTargetColumns(OlapTable table, List isGroupCommit || validColumn(c, childHasSeqCol)) .collect(ImmutableList.toImmutableList()), 0); } else { - int extraColumnsNum = (needExtraSeqCol ? 1 : 0); + int extraColumnsNum = (needExtraSeqCol ? 1 : 0) + missingIvmHiddenColumns.size(); List processedColsName = Lists.newArrayList(colsName); for (Column col : table.getFullSchema()) { if (col.hasOnUpdateDefaultValue()) { @@ -1154,6 +1192,23 @@ private boolean validColumn(Column column, boolean isNeedSequenceCol) { && !column.isMaterializedViewColumn(); } + private boolean isIncrementalIvmTable(OlapTable table) { + return table instanceof MTMV && ((MTMV) table).getRefreshInfo().getRefreshMethod() == RefreshMethod.INCREMENTAL; + } + + private Set getMissingIvmHiddenColumns(OlapTable table, List sinkColumns, LogicalPlan child) { + if (!isIncrementalIvmTable(table)) { + return Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER); + } + Set childOutputNames = child.getOutput().stream() + .map(NamedExpression::getName) + .collect(Collectors.toCollection(() -> Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER))); + return sinkColumns.stream() + .filter(Column::isIvmHiddenColumn) + .filter(columnName -> !childOutputNames.contains(columnName)) + .collect(Collectors.toCollection(() -> Sets.newTreeSet(String.CASE_INSENSITIVE_ORDER))); + } + private static class CustomExpressionAnalyzer extends ExpressionAnalyzer { private Map slotBinder; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java index 4c3194703f5153..5c6d4ba09dc9f7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewUtils.java @@ -56,7 +56,7 @@ import org.apache.doris.nereids.trees.plans.physical.PhysicalRelation; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanVisitor; -import org.apache.doris.nereids.trees.plans.visitor.NondeterministicFunctionCollector; +import org.apache.doris.nereids.trees.plans.visitor.MvNondeterministicFunctionCollector; import org.apache.doris.qe.SessionVariable; import com.google.common.collect.ImmutableList; @@ -519,9 +519,9 @@ public static List removeMaterializedViewHooks(StatementContext sta * the function would be considered as deterministic function and will not return * in the result expression result */ - public static List extractNondeterministicFunction(Plan plan) { + public static List extractMvNondeterministicFunction(Plan plan) { List nondeterministicFunctions = new ArrayList<>(); - plan.accept(NondeterministicFunctionCollector.INSTANCE, nondeterministicFunctions); + plan.accept(MvNondeterministicFunctionCollector.INSTANCE, nondeterministicFunctions); return nondeterministicFunctions; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/IvmNormalizeMtmv.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/IvmNormalizeMtmv.java new file mode 100644 index 00000000000000..dc86768abe3d4e --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/IvmNormalizeMtmv.java @@ -0,0 +1,544 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.common.Pair; +import org.apache.doris.mtmv.ivm.IvmAggMeta; +import org.apache.doris.mtmv.ivm.IvmAggMeta.AggTarget; +import org.apache.doris.mtmv.ivm.IvmAggMeta.AggType; +import org.apache.doris.mtmv.ivm.IvmNormalizeResult; +import org.apache.doris.mtmv.ivm.IvmUtil; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.Cast; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.functions.agg.AggregateFunction; +import org.apache.doris.nereids.trees.expressions.functions.agg.Avg; +import org.apache.doris.nereids.trees.expressions.functions.agg.Count; +import org.apache.doris.nereids.trees.expressions.functions.agg.Max; +import org.apache.doris.nereids.trees.expressions.functions.agg.Min; +import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; +import org.apache.doris.nereids.trees.expressions.functions.scalar.MurmurHash364; +import org.apache.doris.nereids.trees.expressions.functions.scalar.UuidNumeric; +import org.apache.doris.nereids.trees.expressions.literal.LargeIntLiteral; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink; +import org.apache.doris.nereids.trees.plans.visitor.CustomRewriter; +import org.apache.doris.nereids.trees.plans.visitor.DefaultPlanRewriter; +import org.apache.doris.nereids.types.LargeIntType; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.ImmutableSet; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.stream.Collectors; + +/** + * Normalizes the MV define plan for IVM at both CREATE MV and REFRESH MV time. + * - Injects __DORIS_IVM_ROW_ID_COL__ at index 0 of each OlapScan output via a wrapping LogicalProject: + * - MOW (UNIQUE_KEYS + merge-on-write): Alias(cast(murmur_hash3_64(uk...) as LargeInt), + * "__DORIS_IVM_ROW_ID_COL__") + * → deterministic (stable across refreshes) + * - DUP_KEYS: Alias(uuid_numeric(), "__DORIS_IVM_ROW_ID_COL__") → non-deterministic (random per insert) + * - Other key types: not supported, throws. + * - Records (rowIdSlot → isDeterministic) in IvmNormalizeResult on CascadesContext. + * - visitLogicalProject propagates child's row-id slot if not already in outputs. + * - visitLogicalFilter recurses into the child and preserves filter predicates/output shape. + * - visitLogicalResultSink recurses into the child and prepends the row-id to output exprs. + * - Whitelists supported plan nodes; throws AnalysisException for unsupported nodes. + * Supported: OlapScan, filter, project, result sink, logical olap table sink. + * TODO: avg rewrite, join support. + */ +public class IvmNormalizeMtmv extends DefaultPlanRewriter implements CustomRewriter { + + private static final Set> SUPPORTED_AGG_FUNCTIONS = + ImmutableSet.of(Count.class, Sum.class, Avg.class, Min.class, Max.class); + + private final IvmNormalizeResult normalizeResult = new IvmNormalizeResult(); + + @Override + public Plan rewriteRoot(Plan plan, JobContext jobContext) { + ConnectContext connectContext = jobContext.getCascadesContext().getConnectContext(); + if (connectContext == null || !connectContext.getSessionVariable().isEnableIvmNormalRewrite()) { + return plan; + } + // Idempotency: if already normalized (e.g. rewritten plan re-entering), skip. + if (jobContext.getCascadesContext().getIvmNormalizeResult().isPresent()) { + return plan; + } + jobContext.getCascadesContext().setIvmNormalizeResult(normalizeResult); + Plan result = plan.accept(this, true); + normalizeResult.setNormalizedPlan(result); + return result; + } + + // unsupported: any plan node not explicitly whitelisted below + @Override + public Plan visit(Plan plan, Boolean isFirstNonSink) { + throw new AnalysisException("IVM does not support plan node: " + + plan.getClass().getSimpleName()); + } + + // whitelisted: only OlapScan — inject IVM row-id at index 0 + @Override + public Plan visitLogicalOlapScan(LogicalOlapScan scan, Boolean isFirstNonSink) { + OlapTable table = scan.getTable(); + Pair rowId = buildRowId(table, scan); + Alias rowIdAlias = new Alias(rowId.first, Column.IVM_ROW_ID_COL); + normalizeResult.addRowId(rowIdAlias.toSlot(), rowId.second); + List outputs = ImmutableList.builder() + .add(rowIdAlias) + .addAll(scan.getOutput()) + .build(); + return new LogicalProject<>(outputs, scan); + } + + // whitelisted: project — recurse into child, then propagate row-id if not already present + @Override + public Plan visitLogicalProject(LogicalProject project, Boolean isFirstNonSink) { + Plan newChild = project.child().accept(this, isFirstNonSink); + List newOutputs = rewriteOutputsWithIvmHiddenColumns(newChild, project.getProjects()); + if (newChild == project.child() && newOutputs.equals(project.getProjects())) { + return project; + } + return project.withProjectsAndChild(newOutputs, newChild); + } + + @Override + public Plan visitLogicalFilter(LogicalFilter filter, Boolean isFirstNonSink) { + Plan newChild = filter.child().accept(this, false); + return newChild == filter.child() ? filter : filter.withChildren(ImmutableList.of(newChild)); + } + + /** + * Handles aggregate MV normalization. Post-NormalizeAggregate plan shape: + * {@code Project(top) → Aggregate(normalized) → Project(bottom) → ... → Scan} + * + *

This method: + *

    + *
  1. Recurses into child (injects base scan row-id, unused at agg level)
    2. + *
  2. Validates all aggregate functions via {@link #checkAggFunctions}
    3. + *
  3. Adds hidden state aggregate columns to the Aggregate output
    4. + *
  4. Wraps with a Project that computes row-id = hash(group keys) or constant
    5. + *
  5. Stores {@link IvmAggMeta} in {@link IvmNormalizeResult}
    6. + *
+ * + *

Returns: {@code Project(ivm hidden cols + original agg outputs) → Aggregate(with hidden aggs)} + */ + @Override + public Plan visitLogicalAggregate(LogicalAggregate agg, Boolean isFirstNonSink) { + if (!isFirstNonSink) { + throw new AnalysisException( + "IVM aggregate must be the top-level operator (only sinks and projects allowed above it)"); + } + Plan newChild = agg.child().accept(this, false); + + // After NormalizeAggregate, outputs are: group-by key Slots + Alias(AggFunc) + List origOutputs = agg.getOutputExpressions(); + List groupByExprs = agg.getGroupByExpressions(); + boolean scalarAgg = groupByExprs.isEmpty(); + + List aggAliases = new ArrayList<>(); + for (NamedExpression output : origOutputs) { + if (output instanceof Slot) { + // group-by key slot — validated but not collected separately + } else if (output instanceof Alias && ((Alias) output).child() instanceof AggregateFunction) { + aggAliases.add((Alias) output); + } else { + throw new AnalysisException( + "IVM: unexpected expression in normalized aggregate output: " + output); + } + } + + // Validate aggregate functions + List aggFunctions = new ArrayList<>(); + for (Alias alias : aggAliases) { + aggFunctions.add((AggregateFunction) alias.child()); + } + checkAggFunctions(aggFunctions); + + // Build hidden aggregate expressions and AggTarget metadata + // __DORIS_IVM_AGG_COUNT_COL__ = COUNT(*) for group multiplicity + Alias groupCountAlias = new Alias(new Count(), Column.IVM_AGG_COUNT_COL); + + List hiddenAggOutputs = new ArrayList<>(); + hiddenAggOutputs.add(groupCountAlias); + + List aggTargets = new ArrayList<>(); + for (int i = 0; i < aggAliases.size(); i++) { + Alias origAlias = aggAliases.get(i); + AggregateFunction aggFunc = (AggregateFunction) origAlias.child(); + buildHiddenStateForAgg(i, aggFunc, origAlias, hiddenAggOutputs, aggTargets); + } + + // Build new Aggregate with hidden agg outputs AFTER original outputs + ImmutableList.Builder newAggOutputs = ImmutableList.builder(); + newAggOutputs.addAll(origOutputs); + newAggOutputs.addAll(hiddenAggOutputs); + LogicalAggregate newAgg = agg.withAggOutputChild(newAggOutputs.build(), newChild); + + // Build wrapping Project that computes row-id and exposes all slots + // Layout: [row_id, original visible outputs, hidden state outputs] + // groupByExprs are already Slots after NormalizeAggregate + Expression rowIdExpr = scalarAgg + ? new LargeIntLiteral(BigInteger.ZERO) + : buildRowIdHash(groupByExprs); + Alias rowIdAlias = new Alias(rowIdExpr, Column.IVM_ROW_ID_COL); + + // Replace base scan row-id in IvmNormalizeResult with the agg-level row-id + normalizeResult.getRowIdDeterminism().clear(); + normalizeResult.addRowId(rowIdAlias.toSlot(), !scalarAgg); + + // Project output: row_id first, then all Aggregate output slots (original + hidden) + ImmutableList.Builder projectOutputs = ImmutableList.builder(); + projectOutputs.add(rowIdAlias); + for (NamedExpression aggOutput : newAgg.getOutputExpressions()) { + projectOutputs.add(aggOutput.toSlot()); + } + + // Resolve AggTarget slots from the new Aggregate output + List newAggSlots = newAgg.getOutput(); + // groupCountSlot is at origOutputs.size() (first hidden output after original outputs) + Slot groupCountSlot = newAggSlots.get(origOutputs.size()); + List resolvedTargets = resolveAggTargetSlots(aggTargets, hiddenAggOutputs, newAggSlots); + + // Resolve group key slots from the new Aggregate output by matching groupByExprs names + List resolvedGroupKeys = new ArrayList<>(); + for (Expression groupByExpr : groupByExprs) { + String name = ((Slot) groupByExpr).getName(); + for (Slot newSlot : newAggSlots) { + if (newSlot.getName().equals(name)) { + resolvedGroupKeys.add(newSlot); + break; + } + } + } + if (resolvedGroupKeys.size() != groupByExprs.size()) { + throw new AnalysisException("IVM: failed to resolve all group-by key slots from rebuilt aggregate. " + + "Expected " + groupByExprs.size() + " but resolved " + resolvedGroupKeys.size()); + } + + IvmAggMeta aggMeta = new IvmAggMeta(scalarAgg, resolvedGroupKeys, + groupCountSlot, resolvedTargets); + normalizeResult.setAggMeta(aggMeta); + + return new LogicalProject<>(projectOutputs.build(), newAgg); + } + + /** + * For each user-visible aggregate, creates the hidden state columns needed for IVM delta. + * Appends hidden Alias expressions to {@code hiddenAggOutputs} and builds an AggTarget + * (with placeholder slots that will be resolved later from the new Aggregate output). + */ + private void buildHiddenStateForAgg(int ordinal, AggregateFunction aggFunc, Alias origAlias, + List hiddenAggOutputs, List aggTargets) { + AggType aggType; + Map hiddenAliases = new LinkedHashMap<>(); + + if (aggFunc instanceof Count) { + Count countFunc = (Count) aggFunc; + if (countFunc.isStar()) { + aggType = AggType.COUNT_STAR; + hiddenAliases.put("COUNT", new Alias(new Count(), + IvmUtil.ivmAggHiddenColumnName(ordinal, "COUNT"))); + } else { + aggType = AggType.COUNT_EXPR; + hiddenAliases.put("COUNT", new Alias( + new Count(aggFunc.child(0)), + IvmUtil.ivmAggHiddenColumnName(ordinal, "COUNT"))); + } + } else if (aggFunc instanceof Sum) { + aggType = AggType.SUM; + hiddenAliases.put("SUM", new Alias( + new Sum(aggFunc.child(0)), + IvmUtil.ivmAggHiddenColumnName(ordinal, "SUM"))); + hiddenAliases.put("COUNT", new Alias( + new Count(aggFunc.child(0)), + IvmUtil.ivmAggHiddenColumnName(ordinal, "COUNT"))); + } else if (aggFunc instanceof Avg) { + aggType = AggType.AVG; + hiddenAliases.put("SUM", new Alias( + new Sum(aggFunc.child(0)), + IvmUtil.ivmAggHiddenColumnName(ordinal, "SUM"))); + hiddenAliases.put("COUNT", new Alias( + new Count(aggFunc.child(0)), + IvmUtil.ivmAggHiddenColumnName(ordinal, "COUNT"))); + } else if (aggFunc instanceof Min) { + aggType = AggType.MIN; + hiddenAliases.put("MIN", new Alias( + new Min(aggFunc.child(0)), + IvmUtil.ivmAggHiddenColumnName(ordinal, "MIN"))); + hiddenAliases.put("COUNT", new Alias( + new Count(aggFunc.child(0)), + IvmUtil.ivmAggHiddenColumnName(ordinal, "COUNT"))); + } else if (aggFunc instanceof Max) { + aggType = AggType.MAX; + hiddenAliases.put("MAX", new Alias( + new Max(aggFunc.child(0)), + IvmUtil.ivmAggHiddenColumnName(ordinal, "MAX"))); + hiddenAliases.put("COUNT", new Alias( + new Count(aggFunc.child(0)), + IvmUtil.ivmAggHiddenColumnName(ordinal, "COUNT"))); + } else { + throw new AnalysisException("IVM: unsupported aggregate function: " + aggFunc.getName()); + } + + hiddenAggOutputs.addAll(hiddenAliases.values()); + + // Build AggTarget with placeholder slots (to be resolved after Aggregate is rebuilt) + ImmutableMap.Builder placeholderHiddenSlots = ImmutableMap.builder(); + for (Map.Entry entry : hiddenAliases.entrySet()) { + placeholderHiddenSlots.put(entry.getKey(), entry.getValue().toSlot()); + } + + List exprSlots = ImmutableList.of(); + if (!(aggFunc instanceof Count && ((Count) aggFunc).isStar())) { + Expression child0 = aggFunc.child(0); + if (child0 instanceof Slot) { + exprSlots = ImmutableList.of((Slot) child0); + } + } + + aggTargets.add(new AggTarget(ordinal, aggType, origAlias.toSlot(), + placeholderHiddenSlots.build(), exprSlots)); + } + + /** + * Resolves placeholder AggTarget slots to actual slots from the rebuilt Aggregate output. + * Matching is done by column name. + */ + private List resolveAggTargetSlots(List placeholderTargets, + List hiddenAggOutputs, List newAggSlots) { + // Build name→slot map from the new Aggregate output + Map slotByName = new LinkedHashMap<>(); + for (Slot slot : newAggSlots) { + slotByName.put(slot.getName(), slot); + } + + List resolved = new ArrayList<>(); + for (AggTarget target : placeholderTargets) { + // Resolve visible slot + Slot resolvedVisible = slotByName.get(target.getVisibleSlot().getName()); + if (resolvedVisible == null) { + throw new AnalysisException("IVM: failed to resolve visible slot '" + + target.getVisibleSlot().getName() + "' from rebuilt aggregate output"); + } + + // Resolve hidden state slots + ImmutableMap.Builder resolvedHidden = ImmutableMap.builder(); + for (Map.Entry entry : target.getHiddenStateSlots().entrySet()) { + Slot resolvedSlot = slotByName.get(entry.getValue().getName()); + if (resolvedSlot == null) { + throw new AnalysisException("IVM: failed to resolve hidden state slot '" + + entry.getValue().getName() + "' from rebuilt aggregate output"); + } + resolvedHidden.put(entry.getKey(), resolvedSlot); + } + + resolved.add(new AggTarget(target.getOrdinal(), target.getAggType(), + resolvedVisible, resolvedHidden.build(), target.getExprSlots())); + } + return resolved; + } + + // whitelisted: result sink — recurse into child, then prepend row-id to output exprs + @Override + public Plan visitLogicalResultSink(LogicalResultSink sink, Boolean isFirstNonSink) { + Plan newChild = sink.child().accept(this, isFirstNonSink); + List newOutputs = rewriteOutputsWithIvmHiddenColumns(newChild, sink.getOutputExprs()); + if (newChild == sink.child() && newOutputs.equals(sink.getOutputExprs())) { + return sink; + } + return sink.withOutputExprs(newOutputs).withChildren(ImmutableList.of(newChild)); + } + + @Override + public Plan visitLogicalOlapTableSink(LogicalOlapTableSink sink, + Boolean isFirstNonSink) { + Plan newChild = sink.child().accept(this, isFirstNonSink); + if (newChild == sink.child()) { + return sink; + } + return sink.withChildAndUpdateOutput(newChild, sink.getPartitionExprList(), + sink.getSyncMvWhereClauses(), sink.getTargetTableSlots()); + } + + private boolean hasIvmHiddenOutputInOutputs(List outputs) { + return outputs.stream() + .anyMatch(this::isIvmHiddenOutput); + } + + private boolean isIvmHiddenOutput(NamedExpression expression) { + return IvmUtil.isIvmHiddenColumn(expression.getName()); + } + + /** + * Rewrites output expressions to include IVM hidden columns from the child. + * Layout: [row_id, original visible outputs, other hidden cols (count, per-agg states)]. + */ + private List rewriteOutputsWithIvmHiddenColumns( + Plan normalizedChild, List outputs) { + Map ivmHiddenSlotsByName = collectIvmHiddenSlots(normalizedChild); + if (!ivmHiddenSlotsByName.containsKey(Column.IVM_ROW_ID_COL)) { + throw new AnalysisException("IVM normalization error: child plan has no row-id slot after normalization"); + } + + // Separate row-id from other hidden slots + Slot rowIdSlot = ivmHiddenSlotsByName.get(Column.IVM_ROW_ID_COL); + Map otherHiddenSlots = new LinkedHashMap<>(ivmHiddenSlotsByName); + otherHiddenSlots.remove(Column.IVM_ROW_ID_COL); + + ImmutableList.Builder rewrittenOutputs = ImmutableList.builder(); + if (!hasIvmHiddenOutputInOutputs(outputs)) { + // No hidden outputs in original list: prepend row_id, then originals, then other hidden + rewrittenOutputs.add(rowIdSlot); + rewrittenOutputs.addAll(outputs); + rewrittenOutputs.addAll(otherHiddenSlots.values()); + return rewrittenOutputs.build(); + } + + // Outputs already contain some hidden columns (e.g. BindSink placeholders). + // Replace hidden outputs in-place to preserve positions and ExprIds. + for (NamedExpression output : outputs) { + if (isIvmHiddenOutput(output)) { + rewrittenOutputs.add(rewriteIvmHiddenOutput(output, ivmHiddenSlotsByName)); + } else { + rewrittenOutputs.add(output); + } + } + // Append any new hidden slots from child that weren't in the original outputs + for (Map.Entry entry : ivmHiddenSlotsByName.entrySet()) { + String name = entry.getKey(); + if (outputs.stream().noneMatch(o -> name.equals(o.getName()))) { + rewrittenOutputs.add(entry.getValue()); + } + } + return rewrittenOutputs.build(); + } + + private Map collectIvmHiddenSlots(Plan normalizedChild) { + return normalizedChild.getOutput().stream() + .filter(slot -> IvmUtil.isIvmHiddenColumn(slot.getName())) + .collect(Collectors.toMap(Slot::getName, slot -> slot, (left, right) -> left, LinkedHashMap::new)); + } + + private NamedExpression rewriteIvmHiddenOutput(NamedExpression output, Map ivmHiddenSlotsByName) { + Slot ivmHiddenSlot = ivmHiddenSlotsByName.get(output.getName()); + if (ivmHiddenSlot == null) { + throw new AnalysisException("IVM normalization error: child plan has no hidden slot named " + + output.getName() + " after normalization"); + } + if (output instanceof Slot) { + return ivmHiddenSlot; + } + if (output instanceof Alias) { + Alias alias = (Alias) output; + return new Alias(alias.getExprId(), ImmutableList.of(ivmHiddenSlot), alias.getName(), + alias.getQualifier(), alias.isNameFromChild()); + } + throw new AnalysisException("IVM normalization error: unsupported hidden output expression: " + + output.getClass().getSimpleName()); + } + + /** + * Builds the row-id expression and returns whether it is deterministic as a pair. + * - MOW: (buildRowIdHash(uk...), true) — stable across refreshes + * - DUP_KEYS: (UuidNumeric(), false) — random per insert + * - Other key types: throws AnalysisException + */ + private Pair buildRowId(OlapTable table, LogicalOlapScan scan) { + KeysType keysType = table.getKeysType(); + if (keysType == KeysType.UNIQUE_KEYS && table.getEnableUniqueKeyMergeOnWrite()) { + List keyColNames = table.getBaseSchemaKeyColumns().stream() + .map(Column::getName) + .collect(Collectors.toList()); + List keySlots = scan.getOutput().stream() + .filter(s -> keyColNames.contains(s.getName())) + .collect(Collectors.toList()); + if (keySlots.isEmpty()) { + throw new AnalysisException("IVM: no unique key columns found for MOW table: " + + table.getName()); + } + return Pair.of(buildRowIdHash(keySlots), true); + } + if (keysType == KeysType.DUP_KEYS) { + return Pair.of(new UuidNumeric(), false); + } + throw new AnalysisException("IVM does not support table key type: " + keysType + + " for table: " + table.getName() + + ". Only MOW (UNIQUE_KEYS with merge-on-write) and DUP_KEYS are supported."); + } + + /** + * Builds a hash expression over the given key slots for use as a deterministic row-id. + * Currently uses murmur_hash3_64 (64-bit) which is not collision-safe for large tables. + * TODO: replace with a 128-bit hash once BE supports it or a Java UDF is available. + */ + private Expression buildRowIdHash(List keySlots) { + Expression first = keySlots.get(0); + Expression[] rest = keySlots.subList(1, keySlots.size()).toArray(new Expression[0]); + return new Cast(new MurmurHash364(first, rest), LargeIntType.INSTANCE); + } + + /** + * Validates that all aggregate functions are supported for IVM. + * + *

Rules enforced: + *

    + *
  1. At least one aggregate function must be present (bare GROUP BY is not supported).
    2. + *
  2. DISTINCT aggregates are not supported.
    3. + *
  3. Only count, sum, avg, min, and max are supported.
    4. + *
+ * + * @throws AnalysisException if validation fails + */ + private static void checkAggFunctions(List aggFunctions) { + if (aggFunctions.isEmpty()) { + throw new AnalysisException( + "GROUP BY without aggregate functions is not supported for IVM"); + } + for (AggregateFunction aggFunc : aggFunctions) { + if (aggFunc.isDistinct()) { + throw new AnalysisException( + "Aggregate DISTINCT is not supported for IVM: " + aggFunc.toSql()); + } + if (!SUPPORTED_AGG_FUNCTIONS.contains(aggFunc.getClass())) { + throw new AnalysisException( + "Unsupported aggregate function for IVM: " + aggFunc.getName()); + } + } + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java index 74f2bd1bfec007..baab63e3f9c7f8 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommand.java @@ -32,6 +32,7 @@ import org.apache.doris.datasource.mvcc.MvccUtil; import org.apache.doris.mtmv.BaseColInfo; import org.apache.doris.mtmv.BaseTableInfo; +import org.apache.doris.mtmv.MTMVRefreshEnum.RefreshMethod; import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.nereids.StatementContext; import org.apache.doris.nereids.analyzer.UnboundRelation; @@ -115,8 +116,11 @@ public static UpdateMvByPartitionCommand from(MTMV mv, Set partitionName if (plan instanceof Sink) { plan = plan.child(0); } + List sinkColumns = mv.getRefreshInfo().getRefreshMethod() == RefreshMethod.INCREMENTAL + ? mv.getInsertedColumnNames() + : ImmutableList.of(); LogicalSink sink = UnboundTableSinkCreator.createUnboundTableSink(mv.getFullQualifiers(), - ImmutableList.of(), ImmutableList.of(), parts, plan); + sinkColumns, ImmutableList.of(), parts, plan); if (LOG.isDebugEnabled()) { LOG.debug("MTMVTask plan for mvName: {}, partitionNames: {}, plan: {}", mv.getName(), partitionNames, sink.treeString()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java index e03e6907b2766a..6b53a917d8e534 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/BaseViewInfo.java @@ -84,6 +84,7 @@ import java.util.Map; import java.util.Set; import java.util.TreeMap; +import java.util.stream.Collectors; /** BaseViewInfo */ public class BaseViewInfo { @@ -140,12 +141,24 @@ public static String rewriteSql(TreeMap, String> indexStr } protected String rewriteProjectsToUserDefineAlias(String resSql) { - IndexFinder finder = new IndexFinder(); - ParserRuleContext tree = NereidsParser.toAst(resSql, DorisParser::singleStatement); - finder.visit(tree); if (simpleColumnDefinitions.isEmpty()) { return resSql; } + return rewriteProjectsToUserDefineAlias(resSql, finalCols.stream() + .map(Column::getName) + .collect(Collectors.toList())); + } + + /** + * rewrite projects to user define alias by column names list + */ + public static String rewriteProjectsToUserDefineAlias(String resSql, List finalColNames) { + if (finalColNames.isEmpty()) { + return resSql; + } + IndexFinder finder = new IndexFinder(); + ParserRuleContext tree = NereidsParser.toAst(resSql, DorisParser::singleStatement); + finder.visit(tree); List namedExpressionContexts = finder.getNamedExpressionContexts(); StringBuilder replaceWithColsBuilder = new StringBuilder(); for (int i = 0; i < namedExpressionContexts.size(); ++i) { @@ -154,7 +167,7 @@ protected String rewriteProjectsToUserDefineAlias(String resSql) { int stop = namedExpressionContext.expression().stop.getStopIndex(); replaceWithColsBuilder.append(resSql, start, stop + 1); replaceWithColsBuilder.append(" AS `"); - String escapeBacktick = finalCols.get(i).getName().replace("`", "``"); + String escapeBacktick = finalColNames.get(i).replace("`", "``"); replaceWithColsBuilder.append(escapeBacktick); replaceWithColsBuilder.append('`'); if (i != namedExpressionContexts.size() - 1) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java index 96b9981819f97d..94eada3f401ba6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateMTMVInfo.java @@ -28,6 +28,7 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeNameFormat; +import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.util.DynamicPartitionUtil; import org.apache.doris.common.util.PropertyAnalyzer; @@ -39,6 +40,7 @@ import org.apache.doris.mtmv.MTMVPartitionUtil; import org.apache.doris.mtmv.MTMVPlanUtil; import org.apache.doris.mtmv.MTMVPropertyUtil; +import org.apache.doris.mtmv.MTMVRefreshEnum.RefreshMethod; import org.apache.doris.mtmv.MTMVRefreshInfo; import org.apache.doris.mtmv.MTMVRelatedTableIf; import org.apache.doris.mtmv.MTMVRelation; @@ -145,6 +147,12 @@ public void analyze(ConnectContext ctx) throws Exception { throw new AnalysisException(message); } analyzeProperties(); + // IVM MVs must not have user-specified keys — the unique key is the hidden row-id + if (refreshInfo.getRefreshMethod() == RefreshMethod.INCREMENTAL && !keys.isEmpty()) { + throw new AnalysisException( + "Incremental materialized view does not allow specifying key columns. " + + "The unique key is the hidden row-id column managed by IVM."); + } analyzeQuery(ctx); this.partitionDesc = generatePartitionDesc(ctx); if (distribution == null) { @@ -172,8 +180,21 @@ public void analyze(ConnectContext ctx) throws Exception { } private void rewriteQuerySql(ConnectContext ctx) { - analyzeAndFillRewriteSqlMap(querySql, ctx); - querySql = BaseViewInfo.rewriteSql(ctx.getStatementContext().getIndexInSqlToString(), querySql); + TreeMap, String> rewriteMap = ctx.getStatementContext().getIndexInSqlToString(); + TreeMap, String> snapshot = new TreeMap<>(rewriteMap); + rewriteMap.clear(); + try { + analyzeAndFillRewriteSqlMap(querySql, ctx); + querySql = BaseViewInfo.rewriteSql(rewriteMap, querySql); + if (refreshInfo.getRefreshMethod() == RefreshMethod.INCREMENTAL && !simpleColumnDefinitions.isEmpty()) { + querySql = BaseViewInfo.rewriteProjectsToUserDefineAlias(querySql, simpleColumnDefinitions.stream() + .map(SimpleColumnDefinition::getName) + .collect(Collectors.toList())); + } + } finally { + rewriteMap.clear(); + rewriteMap.putAll(snapshot); + } } private void analyzeAndFillRewriteSqlMap(String sql, ConnectContext ctx) { @@ -213,9 +234,10 @@ private void analyzeProperties() { * analyzeQuery */ public void analyzeQuery(ConnectContext ctx) throws UserException { - MTMVAnalyzeQueryInfo mtmvAnalyzeQueryInfo = MTMVPlanUtil.analyzeQuery(ctx, this.mvProperties, this.querySql, + boolean enableIvmNormalize = this.refreshInfo.getRefreshMethod() == RefreshMethod.INCREMENTAL; + MTMVAnalyzeQueryInfo mtmvAnalyzeQueryInfo = MTMVPlanUtil.analyzeQuery(ctx, this.mvProperties, this.mvPartitionDefinition, this.distribution, this.simpleColumnDefinitions, this.properties, this.keys, - this.logicalQuery); + this.logicalQuery, enableIvmNormalize); this.mvPartitionInfo = mtmvAnalyzeQueryInfo.getMvPartitionInfo(); this.columns = mtmvAnalyzeQueryInfo.getColumnDefinitions(); this.relation = mtmvAnalyzeQueryInfo.getRelation(); @@ -281,7 +303,15 @@ private void setTableInformation(ConnectContext ctx) { this.setTableName(tableNameInfo.getTbl()); this.setCtasColumns(ctasColumns.isEmpty() ? null : ctasColumns); this.setEngineName(CreateTableInfo.ENGINE_OLAP); - this.setKeysType(KeysType.DUP_KEYS); + if (refreshInfo.getRefreshMethod() == RefreshMethod.INCREMENTAL) { + this.setKeysType(KeysType.UNIQUE_KEYS); + if (properties == null) { + properties = Maps.newHashMap(); + } + properties.put(PropertyAnalyzer.ENABLE_UNIQUE_KEY_MERGE_ON_WRITE, "true"); + } else { + this.setKeysType(KeysType.DUP_KEYS); + } this.setPartitionTableInfo(partitionDesc == null ? PartitionTableInfo.EMPTY : partitionDesc.convertToPartitionTableInfo()); this.setRollups(Lists.newArrayList()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java index 202b5b58e4cbff..3adacb02880bf7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/info/CreateViewInfo.java @@ -22,6 +22,7 @@ import org.apache.doris.common.ErrorCode; import org.apache.doris.common.ErrorReport; import org.apache.doris.common.FeNameFormat; +import org.apache.doris.common.Pair; import org.apache.doris.common.UserException; import org.apache.doris.common.util.Util; import org.apache.doris.info.TableNameInfo; @@ -33,6 +34,7 @@ import com.google.common.base.Strings; import java.util.List; +import java.util.TreeMap; /** * CreateViewInfo @@ -64,16 +66,24 @@ public void init(ConnectContext ctx) throws UserException { ErrorReport.reportAnalysisException(ErrorCode.ERR_TABLE_ACCESS_DENIED_ERROR, PrivPredicate.CREATE.getPrivs().toString(), viewName.getTbl()); } - analyzeAndFillRewriteSqlMap(querySql, ctx); - PlanUtils.OutermostPlanFinderContext outermostPlanFinderContext = new PlanUtils.OutermostPlanFinderContext(); - analyzedPlan.accept(PlanUtils.OutermostPlanFinder.INSTANCE, outermostPlanFinderContext); - List outputs = outermostPlanFinderContext.outermostPlan.getOutput(); - createFinalCols(outputs); - - // expand star(*) in project list and replace table name with qualifier - String rewrittenSql = rewriteSql(ctx.getStatementContext().getIndexInSqlToString(), querySql); - // rewrite project alias - rewrittenSql = rewriteProjectsToUserDefineAlias(rewrittenSql); + TreeMap, String> rewriteMap = ctx.getStatementContext().getIndexInSqlToString(); + TreeMap, String> snapshot = new TreeMap<>(rewriteMap); + String rewrittenSql; + try { + rewriteMap.clear(); + analyzeAndFillRewriteSqlMap(querySql, ctx); + PlanUtils.OutermostPlanFinderContext outermostPlanFinderContext = + new PlanUtils.OutermostPlanFinderContext(); + analyzedPlan.accept(PlanUtils.OutermostPlanFinder.INSTANCE, outermostPlanFinderContext); + List outputs = outermostPlanFinderContext.outermostPlan.getOutput(); + createFinalCols(outputs); + // expand star(*) in project list and replace table name with qualifier + rewrittenSql = rewriteSql(rewriteMap, querySql); + rewrittenSql = rewriteProjectsToUserDefineAlias(rewrittenSql); + } finally { + rewriteMap.clear(); + rewriteMap.putAll(snapshot); + } checkViewSql(rewrittenSql); this.inlineViewDef = rewrittenSql; } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/NondeterministicFunctionCollector.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/MvNondeterministicFunctionCollector.java similarity index 75% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/NondeterministicFunctionCollector.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/MvNondeterministicFunctionCollector.java index 5b2601445751e6..931c2c8d85bbb9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/NondeterministicFunctionCollector.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/MvNondeterministicFunctionCollector.java @@ -17,6 +17,8 @@ package org.apache.doris.nereids.trees.plans.visitor; +import org.apache.doris.catalog.Column; +import org.apache.doris.nereids.trees.expressions.Alias; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.functions.ExpressionTrait; import org.apache.doris.nereids.trees.expressions.functions.FunctionTrait; @@ -26,12 +28,12 @@ import java.util.Set; /** - * Collect the nondeterministic expr in plan, these expressions will be put into context + * Collect the nondeterministic expr in MV plan, skipping hidden IVM row-id aliases. */ -public class NondeterministicFunctionCollector +public class MvNondeterministicFunctionCollector extends DefaultPlanVisitor> { - public static final NondeterministicFunctionCollector INSTANCE = new NondeterministicFunctionCollector(); + public static final MvNondeterministicFunctionCollector INSTANCE = new MvNondeterministicFunctionCollector(); @Override public Void visit(Plan plan, List collectedExpressions) { @@ -40,6 +42,9 @@ public Void visit(Plan plan, List collectedExpressions) { return super.visit(plan, collectedExpressions); } for (Expression expression : expressions) { + if (isMvRowIdAlias(expression)) { + continue; + } Set nondeterministicFunctions = expression.collect(expr -> !((ExpressionTrait) expr).isDeterministic() && expr instanceof FunctionTrait); @@ -47,4 +52,9 @@ public Void visit(Plan plan, List collectedExpressions) { } return super.visit(plan, collectedExpressions); } + + private boolean isMvRowIdAlias(Expression expression) { + return expression instanceof Alias + && Column.IVM_ROW_ID_COL.equals(((Alias) expression).getName()); + } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index f5c80600178904..d735c42d2272c1 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -411,6 +411,7 @@ public class SessionVariable implements Serializable, Writable { public static final String NEREIDS_CBO_PENALTY_FACTOR = "nereids_cbo_penalty_factor"; public static final String ENABLE_NEREIDS_TRACE = "enable_nereids_trace"; + public static final String ENABLE_IVM_NORMAL_REWRITE = "enable_ivm_normal_rewrite"; public static final String ENABLE_EXPR_TRACE = "enable_expr_trace"; public static final String ENABLE_DPHYP_TRACE = "enable_dphyp_trace"; @@ -2003,6 +2004,9 @@ public boolean isEnableHboNonStrictMatchingMode() { @VariableMgr.VarAttr(name = ENABLE_NEREIDS_TRACE) private boolean enableNereidsTrace = false; + @VariableMgr.VarAttr(name = ENABLE_IVM_NORMAL_REWRITE) + private boolean enableIvmNormalRewrite = false; + @VariableMgr.VarAttr(name = ENABLE_EXPR_TRACE) private boolean enableExprTrace = false; @@ -3844,6 +3848,10 @@ public void setEnableNereidsTrace(boolean enableNereidsTrace) { this.enableNereidsTrace = enableNereidsTrace; } + public void setEnableIvmNormalRewrite(boolean enableIvmNormalRewrite) { + this.enableIvmNormalRewrite = enableIvmNormalRewrite; + } + public void setNereidsTraceEventMode(String nereidsTraceEventMode) { checkNereidsTraceEventMode(nereidsTraceEventMode); this.nereidsTraceEventMode = nereidsTraceEventMode; @@ -5075,6 +5083,10 @@ public boolean isEnableNereidsTrace() { return enableNereidsTrace; } + public boolean isEnableIvmNormalRewrite() { + return enableIvmNormalRewrite; + } + public void setEnableExprTrace(boolean enableExprTrace) { this.enableExprTrace = enableExprTrace; } @@ -6089,6 +6101,10 @@ public boolean isEnableDmlMaterializedViewRewrite() { return enableDmlMaterializedViewRewrite; } + public void setEnableDmlMaterializedViewRewrite(boolean enableDmlMaterializedViewRewrite) { + this.enableDmlMaterializedViewRewrite = enableDmlMaterializedViewRewrite; + } + public boolean isEnableDmlMaterializedViewRewriteWhenBaseTableUnawareness() { return enableDmlMaterializedViewRewriteWhenBaseTableUnawareness; } diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateViewTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateViewTest.java index ec17ca18a07e12..c112680cbac874 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateViewTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/CreateViewTest.java @@ -212,6 +212,32 @@ public void testAlterView() throws Exception { alter1.getInlineViewDef()); } + @Test + public void testCreateViewWithoutDefinedColumnsDoesNotInjectAliases() throws Exception { + ExceptionChecker.expectThrowsNoException( + () -> createView("create view test.no_alias_view as select k1, k2 from test.tbl1;")); + + Database db = Env.getCurrentInternalCatalog().getDbOrDdlException("test"); + View view = (View) db.getTableOrDdlException("no_alias_view"); + Assert.assertEquals( + "select `internal`.`test`.`tbl1`.`k1`, `internal`.`test`.`tbl1`.`k2` " + + "from `internal`.`test`.`tbl1`", + view.getInlineViewDef()); + } + + @Test + public void testCreateViewWithDefinedColumnsRewritesAliases() throws Exception { + ExceptionChecker.expectThrowsNoException( + () -> createView("create view test.with_alias_view(c1, c2) as select k1, k2 from test.tbl1;")); + + Database db = Env.getCurrentInternalCatalog().getDbOrDdlException("test"); + View view = (View) db.getTableOrDdlException("with_alias_view"); + Assert.assertEquals( + "select `internal`.`test`.`tbl1`.`k1` AS `c1`, `internal`.`test`.`tbl1`.`k2` AS `c2` " + + "from `internal`.`test`.`tbl1`", + view.getInlineViewDef()); + } + @Test public void testViewRejectVarbinary() throws Exception { ExceptionChecker.expectThrowsWithMsg( diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java index ca33a4d69973d6..1a8cbdf0ce99a8 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPlanUtilTest.java @@ -45,13 +45,14 @@ import org.apache.doris.nereids.types.StringType; import org.apache.doris.nereids.types.TinyIntType; import org.apache.doris.nereids.types.VarcharType; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; import mockit.Expectations; import mockit.Mocked; -import org.junit.Assert; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; @@ -137,10 +138,10 @@ public void testGenerateColumnsBySql() throws Exception { } private void checkRes(List expect, List actual) { - Assert.assertEquals(expect.size(), actual.size()); + Assertions.assertEquals(expect.size(), actual.size()); for (int i = 0; i < expect.size(); i++) { - Assert.assertEquals(expect.get(i).getName(), actual.get(i).getName()); - Assert.assertEquals(expect.get(i).getType(), actual.get(i).getType()); + Assertions.assertEquals(expect.get(i).getName(), actual.get(i).getName()); + Assertions.assertEquals(expect.get(i).getType(), actual.get(i).getType()); } } @@ -167,19 +168,19 @@ public void testGetDataType(@Mocked SlotReference slot, @Mocked TableIf slotTabl }; // test i=0 DataType dataType = MTMVPlanUtil.getDataType(slot, 0, connectContext, "pcol", Sets.newHashSet("dcol")); - Assert.assertEquals(VarcharType.MAX_VARCHAR_TYPE, dataType); + Assertions.assertEquals(VarcharType.MAX_VARCHAR_TYPE, dataType); // test isColumnFromTable and is not managed table dataType = MTMVPlanUtil.getDataType(slot, 1, connectContext, "pcol", Sets.newHashSet("dcol")); - Assert.assertEquals(StringType.INSTANCE, dataType); + Assertions.assertEquals(StringType.INSTANCE, dataType); // test is partitionCol dataType = MTMVPlanUtil.getDataType(slot, 1, connectContext, "slot_name", Sets.newHashSet("dcol")); - Assert.assertEquals(VarcharType.MAX_VARCHAR_TYPE, dataType); + Assertions.assertEquals(VarcharType.MAX_VARCHAR_TYPE, dataType); // test is partitdistribution Col dataType = MTMVPlanUtil.getDataType(slot, 1, connectContext, "pcol", Sets.newHashSet("slot_name")); - Assert.assertEquals(VarcharType.MAX_VARCHAR_TYPE, dataType); + Assertions.assertEquals(VarcharType.MAX_VARCHAR_TYPE, dataType); // test managed table new Expectations() { { @@ -194,7 +195,7 @@ public void testGetDataType(@Mocked SlotReference slot, @Mocked TableIf slotTabl }; dataType = MTMVPlanUtil.getDataType(slot, 1, connectContext, "pcol", Sets.newHashSet("slot_name")); - Assert.assertEquals(StringType.INSTANCE, dataType); + Assertions.assertEquals(StringType.INSTANCE, dataType); // test is not column table boolean originalUseMaxLengthOfVarcharInCtas = connectContext.getSessionVariable().useMaxLengthOfVarcharInCtas; @@ -211,11 +212,11 @@ public void testGetDataType(@Mocked SlotReference slot, @Mocked TableIf slotTabl }; connectContext.getSessionVariable().useMaxLengthOfVarcharInCtas = true; dataType = MTMVPlanUtil.getDataType(slot, 1, connectContext, "pcol", Sets.newHashSet("slot_name")); - Assert.assertEquals(VarcharType.MAX_VARCHAR_TYPE, dataType); + Assertions.assertEquals(VarcharType.MAX_VARCHAR_TYPE, dataType); connectContext.getSessionVariable().useMaxLengthOfVarcharInCtas = false; dataType = MTMVPlanUtil.getDataType(slot, 1, connectContext, "pcol", Sets.newHashSet("slot_name")); - Assert.assertEquals(new VarcharType(10), dataType); + Assertions.assertEquals(new VarcharType(10), dataType); connectContext.getSessionVariable().useMaxLengthOfVarcharInCtas = originalUseMaxLengthOfVarcharInCtas; @@ -228,7 +229,7 @@ public void testGetDataType(@Mocked SlotReference slot, @Mocked TableIf slotTabl } }; dataType = MTMVPlanUtil.getDataType(slot, 1, connectContext, "pcol", Sets.newHashSet("slot_name")); - Assert.assertEquals(TinyIntType.INSTANCE, dataType); + Assertions.assertEquals(TinyIntType.INSTANCE, dataType); // test decimal type new Expectations() { @@ -241,7 +242,7 @@ public void testGetDataType(@Mocked SlotReference slot, @Mocked TableIf slotTabl boolean originalEnableDecimalConversion = Config.enable_decimal_conversion; Config.enable_decimal_conversion = false; dataType = MTMVPlanUtil.getDataType(slot, 1, connectContext, "pcol", Sets.newHashSet("slot_name")); - Assert.assertEquals(DecimalV2Type.SYSTEM_DEFAULT, dataType); + Assertions.assertEquals(DecimalV2Type.SYSTEM_DEFAULT, dataType); Config.enable_decimal_conversion = originalEnableDecimalConversion; } @@ -304,8 +305,8 @@ public void testAnalyzeQuerynNonDeterministic() throws Exception { AnalysisException exception = Assertions.assertThrows( org.apache.doris.nereids.exceptions.AnalysisException.class, () -> { - MTMVPlanUtil.analyzeQuery(connectContext, Maps.newHashMap(), querySql, mtmvPartitionDefinition, - distributionDescriptor, null, Maps.newHashMap(), Lists.newArrayList(), logicalPlan); + MTMVPlanUtil.analyzeQuery(connectContext, Maps.newHashMap(), mtmvPartitionDefinition, + distributionDescriptor, null, Maps.newHashMap(), Lists.newArrayList(), logicalPlan, false); }); Assertions.assertTrue(exception.getMessage().contains("nonDeterministic")); } @@ -322,8 +323,8 @@ public void testAnalyzeQueryFromTablet() throws Exception { AnalysisException exception = Assertions.assertThrows( org.apache.doris.nereids.exceptions.AnalysisException.class, () -> { - MTMVPlanUtil.analyzeQuery(connectContext, Maps.newHashMap(), querySql, mtmvPartitionDefinition, - distributionDescriptor, null, Maps.newHashMap(), Lists.newArrayList(), logicalPlan); + MTMVPlanUtil.analyzeQuery(connectContext, Maps.newHashMap(), mtmvPartitionDefinition, + distributionDescriptor, null, Maps.newHashMap(), Lists.newArrayList(), logicalPlan, false); }); Assertions.assertTrue(exception.getMessage().contains("invalid expression")); } @@ -354,8 +355,8 @@ public void testAnalyzeQueryFromTempTable() throws Exception { AnalysisException exception = Assertions.assertThrows( org.apache.doris.nereids.exceptions.AnalysisException.class, () -> { - MTMVPlanUtil.analyzeQuery(connectContext, Maps.newHashMap(), querySql, mtmvPartitionDefinition, - distributionDescriptor, null, Maps.newHashMap(), Lists.newArrayList(), logicalPlan); + MTMVPlanUtil.analyzeQuery(connectContext, Maps.newHashMap(), mtmvPartitionDefinition, + distributionDescriptor, null, Maps.newHashMap(), Lists.newArrayList(), logicalPlan, false); }); Assertions.assertTrue(exception.getMessage().contains("temporary")); } @@ -373,8 +374,8 @@ public void testAnalyzeQueryFollowBaseTableFailed() throws Exception { AnalysisException exception = Assertions.assertThrows( org.apache.doris.nereids.exceptions.AnalysisException.class, () -> { - MTMVPlanUtil.analyzeQuery(connectContext, Maps.newHashMap(), querySql, mtmvPartitionDefinition, - distributionDescriptor, null, Maps.newHashMap(), Lists.newArrayList(), logicalPlan); + MTMVPlanUtil.analyzeQuery(connectContext, Maps.newHashMap(), mtmvPartitionDefinition, + distributionDescriptor, null, Maps.newHashMap(), Lists.newArrayList(), logicalPlan, false); }); Assertions.assertTrue(exception.getMessage().contains("suitable")); } @@ -390,8 +391,8 @@ public void testAnalyzeQueryNormal() throws Exception { StatementBase parsedStmt = new NereidsParser().parseSQL(querySql).get(0); LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan(); MTMVAnalyzeQueryInfo mtmvAnalyzeQueryInfo = MTMVPlanUtil.analyzeQuery(connectContext, Maps.newHashMap(), - querySql, mtmvPartitionDefinition, - distributionDescriptor, null, Maps.newHashMap(), Lists.newArrayList(), logicalPlan); + mtmvPartitionDefinition, + distributionDescriptor, null, Maps.newHashMap(), Lists.newArrayList(), logicalPlan, false); Assertions.assertTrue(mtmvAnalyzeQueryInfo.getRelation().getBaseTables().size() == 1); Assertions.assertTrue(mtmvAnalyzeQueryInfo.getMvPartitionInfo().getRelatedCol().equals("id")); Assertions.assertTrue(mtmvAnalyzeQueryInfo.getColumnDefinitions().size() == 2); @@ -410,6 +411,65 @@ public void testEnsureMTMVQueryUsable() throws Exception { MTMVPlanUtil.createMTMVContext(mtmv, MTMVPlanUtil.DISABLE_RULES_WHEN_GENERATE_MTMV_CACHE))); } + @Test + public void testAnalyzeQueryIvmAnalyzeModeSetSessionVariables() throws Exception { + String querySql = "select * from test.T4"; + MTMVPartitionDefinition mtmvPartitionDefinition = new MTMVPartitionDefinition(); + mtmvPartitionDefinition.setPartitionType(MTMVPartitionType.FOLLOW_BASE_TABLE); + mtmvPartitionDefinition.setPartitionCol("id"); + DistributionDescriptor distributionDescriptor = new DistributionDescriptor(false, true, 10, + Lists.newArrayList("id")); + StatementBase parsedStmt = new NereidsParser().parseSQL(querySql).get(0); + LogicalPlan logicalPlan = ((LogicalPlanAdapter) parsedStmt).getLogicalPlan(); + + // enableIvmNormalize=false: no IVM session variables set + CountingSessionVariable disabledVar = new CountingSessionVariable(); + connectContext.setSessionVariable(disabledVar); + MTMVPlanUtil.analyzeQuery(connectContext, Maps.newHashMap(), mtmvPartitionDefinition, + distributionDescriptor, null, Maps.newHashMap(), Lists.newArrayList(), logicalPlan, + false); + Assertions.assertEquals(0, disabledVar.getEnableIvmRewriteSetCount()); + + // enableIvmNormalize=true: ENABLE_IVM_NORMAL_REWRITE set + CountingSessionVariable enabledVar = new CountingSessionVariable(); + connectContext.setSessionVariable(enabledVar); + MTMVPlanUtil.analyzeQuery(connectContext, Maps.newHashMap(), mtmvPartitionDefinition, + distributionDescriptor, null, Maps.newHashMap(), Lists.newArrayList(), logicalPlan, + true); + Assertions.assertEquals(1, enabledVar.getEnableIvmRewriteSetCount()); + } + + @Test + public void testEnsureMTMVQueryUsableEnableIvmRewriteByRefreshMethod() throws Exception { + createMvByNereids("create materialized view mv_auto_refresh BUILD DEFERRED REFRESH AUTO ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 1\n" + + " PROPERTIES ('replication_num' = '1') \n" + + " as select * from test.T4;"); + createMvByNereids("create materialized view mv_incremental_refresh " + + "BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 1\n" + + " PROPERTIES ('replication_num' = '1') \n" + + " as select * from test.T4;"); + + Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test"); + MTMV autoMtmv = (MTMV) db.getTableOrAnalysisException("mv_auto_refresh"); + MTMV incrementalMtmv = (MTMV) db.getTableOrAnalysisException("mv_incremental_refresh"); + + CountingSessionVariable autoSessionVariable = new CountingSessionVariable(); + ConnectContext autoCtx = MTMVPlanUtil.createMTMVContext(autoMtmv, + MTMVPlanUtil.DISABLE_RULES_WHEN_GENERATE_MTMV_CACHE); + autoCtx.setSessionVariable(autoSessionVariable); + Assertions.assertDoesNotThrow(() -> MTMVPlanUtil.ensureMTMVQueryUsable(autoMtmv, autoCtx)); + Assertions.assertEquals(0, autoSessionVariable.getEnableIvmRewriteSetCount()); + + CountingSessionVariable incrementalSessionVariable = new CountingSessionVariable(); + ConnectContext incrementalCtx = MTMVPlanUtil.createMTMVContext(incrementalMtmv, + MTMVPlanUtil.DISABLE_RULES_WHEN_GENERATE_MTMV_CACHE); + incrementalCtx.setSessionVariable(incrementalSessionVariable); + Assertions.assertDoesNotThrow(() -> MTMVPlanUtil.ensureMTMVQueryUsable(incrementalMtmv, incrementalCtx)); + Assertions.assertEquals(1, incrementalSessionVariable.getEnableIvmRewriteSetCount()); + } + @Test public void testEnsureMTMVQueryAnalyzeFailed() throws Exception { createTable("CREATE TABLE IF NOT EXISTS analyze_faild_t_partition (\n" @@ -496,4 +556,71 @@ public void testEnsureMTMVQueryNotEqual() throws Exception { }); Assertions.assertTrue(exception.getMessage().contains("changed")); } + + @Test + public void testIncrementalMvIsUniqueKeyWithMow() throws Exception { + createMvByNereids("create materialized view mv_ivm_unique_key " + + "BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 1\n" + + " PROPERTIES ('replication_num' = '1') \n" + + " as select * from test.T4;"); + + Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test"); + MTMV mtmv = (MTMV) db.getTableOrAnalysisException("mv_ivm_unique_key"); + + Assertions.assertEquals(org.apache.doris.catalog.KeysType.UNIQUE_KEYS, mtmv.getKeysType()); + Assertions.assertTrue(mtmv.getEnableUniqueKeyMergeOnWrite()); + } + + @Test + public void testNonIncrementalMvIsDuplicateKey() throws Exception { + createMvByNereids("create materialized view mv_non_ivm_dup_key " + + "BUILD DEFERRED REFRESH COMPLETE ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 1\n" + + " PROPERTIES ('replication_num' = '1') \n" + + " as select * from test.T4;"); + + Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test"); + MTMV mtmv = (MTMV) db.getTableOrAnalysisException("mv_non_ivm_dup_key"); + + Assertions.assertEquals(org.apache.doris.catalog.KeysType.DUP_KEYS, mtmv.getKeysType()); + } + + @Test + public void testIncrementalMvWithUserSpecifiedUniqueKeyFails() { + Assertions.assertThrows(Exception.class, () -> + createMvByNereids("create materialized view mv_ivm_user_unique_key " + + "UNIQUE KEY(id) " + + "BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n" + + " DISTRIBUTED BY HASH(id) BUCKETS 1\n" + + " PROPERTIES ('replication_num' = '1') \n" + + " as select * from test.T4;")); + } + + @Test + public void testIncrementalMvWithUserSpecifiedDupKeyFails() { + Assertions.assertThrows(Exception.class, () -> + createMvByNereids("create materialized view mv_ivm_user_dup_key " + + "DUPLICATE KEY(id) " + + "BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n" + + " DISTRIBUTED BY HASH(id) BUCKETS 1\n" + + " PROPERTIES ('replication_num' = '1') \n" + + " as select * from test.T4;")); + } + + private static class CountingSessionVariable extends SessionVariable { + private int enableIvmRewriteSetCount; + + @Override + public boolean setVarOnce(String varName, String value) { + if (ENABLE_IVM_NORMAL_REWRITE.equals(varName) && "true".equals(value)) { + enableIvmRewriteSetCount++; + } + return super.setVarOnce(varName, value); + } + + public int getEnableIvmRewriteSetCount() { + return enableIvmRewriteSetCount; + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java index 8074ce99f81972..57186de7c19b37 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVTest.java @@ -36,6 +36,7 @@ import org.apache.doris.mtmv.MTMVRefreshEnum.MTMVState; import org.apache.doris.mtmv.MTMVRefreshEnum.RefreshMethod; import org.apache.doris.mtmv.MTMVRefreshEnum.RefreshTrigger; +import org.apache.doris.thrift.TStorageType; import com.google.common.collect.Lists; import com.google.common.collect.Maps; @@ -45,6 +46,7 @@ import org.junit.Test; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Set; @@ -198,4 +200,25 @@ public void testAlterStatus() { Assert.assertEquals(MTMVState.SCHEMA_CHANGE, status.getState()); Assert.assertEquals(MTMVRefreshState.SUCCESS, status.getRefreshState()); } + + @Test + public void testGetInsertedColumnNamesIncludesAllIvmHiddenColumns() { + MTMV mtmv = new MTMV(); + List schema = Lists.newArrayList( + new Column(Column.IVM_ROW_ID_COL, PrimitiveType.LARGEINT, false), + new Column(Column.IVM_HIDDEN_COLUMN_PREFIX + "SNAPSHOT_COL__", PrimitiveType.BIGINT, false), + new Column("k1", PrimitiveType.INT, true), + new Column("hidden", ScalarType.createType(PrimitiveType.INT), false, null, + false, "comment", false, Column.COLUMN_UNIQUE_ID_INIT_VALUE) + ); + mtmv.setBaseIndexId(1L); + mtmv.setIndexMeta(1L, "mv", schema, 0, 0, (short) 1, TStorageType.COLUMN, org.apache.doris.catalog.KeysType.DUP_KEYS); + + List insertedColumnNames = mtmv.getInsertedColumnNames(); + + Assert.assertEquals(Lists.newArrayList( + Column.IVM_ROW_ID_COL, + Column.IVM_HIDDEN_COLUMN_PREFIX + "SNAPSHOT_COL__", + "k1"), insertedColumnNames); + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmDeltaExecutorTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmDeltaExecutorTest.java new file mode 100644 index 00000000000000..db4208da560bb4 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmDeltaExecutorTest.java @@ -0,0 +1,206 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv.ivm; + +import org.apache.doris.catalog.MTMV; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.mtmv.BaseTableInfo; +import org.apache.doris.mtmv.MTMVPlanUtil; +import org.apache.doris.nereids.rules.RuleType; +import org.apache.doris.nereids.trees.plans.commands.Command; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.StmtExecutor; + +import mockit.Expectations; +import mockit.Mock; +import mockit.MockUp; +import mockit.Mocked; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.List; + +public class IvmDeltaExecutorTest { + + private IvmDeltaExecutor deltaExecutor; + + @BeforeEach + public void setUp() { + deltaExecutor = new IvmDeltaExecutor(); + } + + @Test + public void testExecuteEmptyBundles(@Mocked MTMV mtmv) throws AnalysisException { + IvmRefreshContext context = newContext(mtmv); + deltaExecutor.execute(context, Collections.emptyList()); + } + + @Test + public void testExecuteSingleBundleSuccess(@Mocked MTMV mtmv, + @Mocked Command command) throws Exception { + ConnectContext mockCtx = new ConnectContext(); + new MockUp() { + @Mock + public ConnectContext createMTMVContext(MTMV mv, List disableRules) { + return mockCtx; + } + }; + + List runCalled = new ArrayList<>(); + new Expectations() { + { + command.run((ConnectContext) any, (StmtExecutor) any); + result = new mockit.Delegate() { + @SuppressWarnings("unused") + void run(ConnectContext ctx, StmtExecutor executor) { + runCalled.add(true); + ctx.getState().setOk(); + } + }; + } + }; + + BaseTableInfo baseTableInfo = new BaseTableInfo(mtmv, 0L); + DeltaCommandBundle bundle = new DeltaCommandBundle(baseTableInfo, command); + + deltaExecutor.execute(newContext(mtmv), Collections.singletonList(bundle)); + Assertions.assertEquals(1, runCalled.size()); + } + + @Test + public void testExecuteCommandRunThrowsException(@Mocked MTMV mtmv, + @Mocked Command command) throws Exception { + ConnectContext mockCtx = new ConnectContext(); + new MockUp() { + @Mock + public ConnectContext createMTMVContext(MTMV mv, List disableRules) { + return mockCtx; + } + }; + + new Expectations() { + { + command.run((ConnectContext) any, (StmtExecutor) any); + result = new RuntimeException("command run failed"); + } + }; + + BaseTableInfo baseTableInfo = new BaseTableInfo(mtmv, 0L); + DeltaCommandBundle bundle = new DeltaCommandBundle(baseTableInfo, command); + + AnalysisException ex = Assertions.assertThrows(AnalysisException.class, + () -> deltaExecutor.execute(newContext(mtmv), Collections.singletonList(bundle))); + Assertions.assertTrue(ex.getMessage().contains("IVM delta execution failed")); + Assertions.assertTrue(ex.getMessage().contains("command run failed")); + } + + @Test + public void testExecuteCommandReturnsErrorState(@Mocked MTMV mtmv, + @Mocked Command command) throws Exception { + ConnectContext mockCtx = new ConnectContext(); + new MockUp() { + @Mock + public ConnectContext createMTMVContext(MTMV mv, List disableRules) { + return mockCtx; + } + }; + + new Expectations() { + { + command.run((ConnectContext) any, (StmtExecutor) any); + result = new mockit.Delegate() { + @SuppressWarnings("unused") + void run(ConnectContext ctx, StmtExecutor executor) { + ctx.getState().setError("something went wrong"); + } + }; + } + }; + + BaseTableInfo baseTableInfo = new BaseTableInfo(mtmv, 0L); + DeltaCommandBundle bundle = new DeltaCommandBundle(baseTableInfo, command); + + AnalysisException ex = Assertions.assertThrows(AnalysisException.class, + () -> deltaExecutor.execute(newContext(mtmv), Collections.singletonList(bundle))); + Assertions.assertTrue(ex.getMessage().contains("IVM delta execution failed")); + Assertions.assertTrue(ex.getMessage().contains("something went wrong")); + } + + @Test + public void testExecuteMultipleBundlesStopsOnFirstFailure(@Mocked MTMV mtmv, + @Mocked Command okCommand, @Mocked Command failCommand, + @Mocked Command thirdCommand) throws Exception { + ConnectContext mockCtx = new ConnectContext(); + new MockUp() { + @Mock + public ConnectContext createMTMVContext(MTMV mv, List disableRules) { + return mockCtx; + } + }; + + List executionOrder = new ArrayList<>(); + new Expectations() { + { + okCommand.run((ConnectContext) any, (StmtExecutor) any); + result = new mockit.Delegate() { + @SuppressWarnings("unused") + void run(ConnectContext ctx, StmtExecutor executor) { + executionOrder.add(1); + ctx.getState().setOk(); + } + }; + failCommand.run((ConnectContext) any, (StmtExecutor) any); + result = new mockit.Delegate() { + @SuppressWarnings("unused") + void run(ConnectContext ctx, StmtExecutor executor) throws Exception { + executionOrder.add(2); + throw new RuntimeException("second bundle failed"); + } + }; + thirdCommand.run((ConnectContext) any, (StmtExecutor) any); + result = new mockit.Delegate() { + @SuppressWarnings("unused") + void run(ConnectContext ctx, StmtExecutor executor) { + executionOrder.add(3); + ctx.getState().setOk(); + } + }; + minTimes = 0; + } + }; + + List bundles = new ArrayList<>(); + BaseTableInfo baseTableInfo = new BaseTableInfo(mtmv, 0L); + bundles.add(new DeltaCommandBundle(baseTableInfo, okCommand)); + bundles.add(new DeltaCommandBundle(baseTableInfo, failCommand)); + bundles.add(new DeltaCommandBundle(baseTableInfo, thirdCommand)); + + Assertions.assertThrows(AnalysisException.class, + () -> deltaExecutor.execute(newContext(mtmv), bundles)); + Assertions.assertEquals(Arrays.asList(1, 2), executionOrder); + } + + private static IvmRefreshContext newContext(MTMV mtmv) { + return new IvmRefreshContext(mtmv, new ConnectContext(), + new org.apache.doris.mtmv.MTMVRefreshContext(mtmv)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmDeltaRewriterTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmDeltaRewriterTest.java new file mode 100644 index 00000000000000..9db7a4b5552bb8 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmDeltaRewriterTest.java @@ -0,0 +1,119 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv.ivm; + +import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.properties.OrderKey; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalResultSink; +import org.apache.doris.nereids.trees.plans.logical.LogicalSort; +import org.apache.doris.nereids.util.PlanConstructor; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.collect.ImmutableList; +import mockit.Expectations; +import mockit.Mocked; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; + +class IvmDeltaRewriterTest { + + private LogicalOlapScan buildScan() { + OlapTable table = PlanConstructor.newOlapTable(0, "t1", 0); + table.setQualifiedDbName("test_db"); + return new LogicalOlapScan(PlanConstructor.getNextRelationId(), table, ImmutableList.of("test_db")); + } + + @Test + void testScanOnlyProducesInsertBundle(@Mocked MTMV mtmv) { + LogicalOlapScan scan = buildScan(); + ImmutableList exprs = ImmutableList.copyOf(scan.getOutput()); + LogicalProject project = new LogicalProject<>(exprs, scan); + LogicalResultSink> plan = new LogicalResultSink<>(exprs, project); + + new Expectations() { + { + mtmv.getQualifiedDbName(); + result = "test_db"; + mtmv.getName(); + result = "test_mv"; + } + }; + + IvmDeltaRewriteContext ctx = new IvmDeltaRewriteContext(mtmv, new ConnectContext(), null); + List bundles = new IvmDeltaRewriter().rewrite(plan, ctx); + + Assertions.assertEquals(1, bundles.size()); + Assertions.assertEquals("t1", bundles.get(0).getBaseTableInfo().getTableName()); + Assertions.assertInstanceOf(InsertIntoTableCommand.class, bundles.get(0).getCommand()); + } + + @Test + void testProjectScanProducesInsertBundle(@Mocked MTMV mtmv) { + LogicalOlapScan scan = buildScan(); + ImmutableList exprs = ImmutableList.copyOf(scan.getOutput()); + LogicalProject innerProject = new LogicalProject<>(exprs, scan); + LogicalProject> outerProject = new LogicalProject<>(exprs, innerProject); + LogicalResultSink plan = new LogicalResultSink<>(exprs, outerProject); + + new Expectations() { + { + mtmv.getQualifiedDbName(); + result = "test_db"; + mtmv.getName(); + result = "test_mv"; + } + }; + + IvmDeltaRewriteContext ctx = new IvmDeltaRewriteContext(mtmv, new ConnectContext(), null); + List bundles = new IvmDeltaRewriter().rewrite(plan, ctx); + + Assertions.assertEquals(1, bundles.size()); + Assertions.assertEquals("t1", bundles.get(0).getBaseTableInfo().getTableName()); + Assertions.assertInstanceOf(InsertIntoTableCommand.class, bundles.get(0).getCommand()); + } + + @Test + void testUnsupportedSortNodeThrows(@Mocked MTMV mtmv) { + LogicalOlapScan scan = buildScan(); + ImmutableList exprs = ImmutableList.copyOf(scan.getOutput()); + LogicalSort sort = new LogicalSort<>( + ImmutableList.of(new OrderKey(scan.getOutput().get(0), true, true)), scan); + LogicalResultSink plan = new LogicalResultSink<>(exprs, sort); + + IvmDeltaRewriteContext ctx = new IvmDeltaRewriteContext(mtmv, new ConnectContext(), null); + AnalysisException ex = Assertions.assertThrows(AnalysisException.class, + () -> new IvmDeltaRewriter().rewrite(plan, ctx)); + Assertions.assertTrue(ex.getMessage().contains("LogicalSort")); + } + + @Test + void testContextRejectsNulls(@Mocked MTMV mtmv) { + Assertions.assertThrows(NullPointerException.class, + () -> new IvmDeltaRewriteContext(null, new ConnectContext(), null)); + Assertions.assertThrows(NullPointerException.class, + () -> new IvmDeltaRewriteContext(mtmv, null, null)); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmRefreshManagerTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmRefreshManagerTest.java new file mode 100644 index 00000000000000..82e19933c6bcae --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/ivm/IvmRefreshManagerTest.java @@ -0,0 +1,272 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.mtmv.ivm; + +import org.apache.doris.catalog.MTMV; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.TableIf; +import org.apache.doris.common.AnalysisException; +import org.apache.doris.mtmv.BaseTableInfo; +import org.apache.doris.mtmv.MTMVRelation; +import org.apache.doris.mtmv.MTMVUtil; +import org.apache.doris.nereids.trees.plans.commands.Command; +import org.apache.doris.qe.ConnectContext; + +import com.google.common.collect.Sets; +import mockit.Expectations; +import mockit.Mock; +import mockit.MockUp; +import mockit.Mocked; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.Collections; +import java.util.HashMap; +import java.util.List; + +public class IvmRefreshManagerTest { + + @Test + public void testRefreshContextRejectsNulls(@Mocked MTMV mtmv) { + ConnectContext connectContext = new ConnectContext(); + org.apache.doris.mtmv.MTMVRefreshContext mtmvRefreshContext = new org.apache.doris.mtmv.MTMVRefreshContext(mtmv); + + Assertions.assertThrows(NullPointerException.class, + () -> new IvmRefreshContext(null, connectContext, mtmvRefreshContext)); + Assertions.assertThrows(NullPointerException.class, + () -> new IvmRefreshContext(mtmv, null, mtmvRefreshContext)); + Assertions.assertThrows(NullPointerException.class, + () -> new IvmRefreshContext(mtmv, connectContext, null)); + } + + @Test + public void testManagerRejectsNulls() { + Assertions.assertThrows(NullPointerException.class, + () -> new IvmRefreshManager(null)); + } + + @Test + public void testManagerReturnsNoBundlesFallback(@Mocked MTMV mtmv) { + TestDeltaExecutor executor = new TestDeltaExecutor(); + TestIvmRefreshManager manager = new TestIvmRefreshManager(executor, + newContext(mtmv), Collections.emptyList()); + + IvmRefreshResult result = manager.doRefresh(mtmv); + + Assertions.assertFalse(result.isSuccess()); + Assertions.assertEquals(FallbackReason.PLAN_PATTERN_UNSUPPORTED, result.getFallbackReason()); + Assertions.assertFalse(executor.executeCalled); + } + + @Test + public void testManagerExecutesBundles(@Mocked MTMV mtmv, @Mocked Command deltaWriteCommand) { + TestDeltaExecutor executor = new TestDeltaExecutor(); + List bundles = makeBundles(deltaWriteCommand, mtmv); + TestIvmRefreshManager manager = new TestIvmRefreshManager(executor, newContext(mtmv), bundles); + + IvmRefreshResult result = manager.doRefresh(mtmv); + + Assertions.assertTrue(result.isSuccess()); + Assertions.assertTrue(executor.executeCalled); + Assertions.assertEquals(bundles, executor.lastBundles); + } + + @Test + public void testManagerReturnsExecutionFallbackOnExecutorFailure(@Mocked MTMV mtmv, + @Mocked Command deltaWriteCommand) { + TestDeltaExecutor executor = new TestDeltaExecutor(); + executor.throwOnExecute = true; + TestIvmRefreshManager manager = new TestIvmRefreshManager(executor, + newContext(mtmv), makeBundles(deltaWriteCommand, mtmv)); + + IvmRefreshResult result = manager.doRefresh(mtmv); + + Assertions.assertFalse(result.isSuccess()); + Assertions.assertEquals(FallbackReason.INCREMENTAL_EXECUTION_FAILED, result.getFallbackReason()); + Assertions.assertTrue(executor.executeCalled); + } + + @Test + public void testManagerReturnsSnapshotFallbackWhenBuildContextFails(@Mocked MTMV mtmv) { + TestDeltaExecutor executor = new TestDeltaExecutor(); + TestIvmRefreshManager manager = new TestIvmRefreshManager(executor, null, Collections.emptyList()); + manager.throwOnBuild = true; + + IvmRefreshResult result = manager.doRefresh(mtmv); + + Assertions.assertFalse(result.isSuccess()); + Assertions.assertEquals(FallbackReason.SNAPSHOT_ALIGNMENT_UNSUPPORTED, result.getFallbackReason()); + Assertions.assertFalse(executor.executeCalled); + } + + @Test + public void testManagerReturnsBinlogBrokenBeforeNereidsFlow(@Mocked MTMV mtmv) { + IvmInfo ivmInfo = new IvmInfo(); + ivmInfo.setBinlogBroken(true); + new Expectations() { + { + mtmv.getIvmInfo(); + result = ivmInfo; + } + }; + + TestDeltaExecutor executor = new TestDeltaExecutor(); + TestIvmRefreshManager manager = new TestIvmRefreshManager(executor, + newContext(mtmv), Collections.emptyList()); + manager.useSuperPrecheck = true; + + IvmRefreshResult result = manager.doRefresh(mtmv); + + Assertions.assertFalse(result.isSuccess()); + Assertions.assertEquals(FallbackReason.BINLOG_BROKEN, result.getFallbackReason()); + Assertions.assertFalse(executor.executeCalled); + } + + @Test + public void testManagerReturnsStreamUnsupportedWithoutBinding(@Mocked MTMV mtmv, + @Mocked MTMVRelation relation, @Mocked OlapTable olapTable) { + IvmInfo ivmInfo = new IvmInfo(); + BaseTableInfo baseTableInfo = new BaseTableInfo(olapTable, 2L); + new Expectations() { + { + mtmv.getIvmInfo(); + result = ivmInfo; + minTimes = 1; + mtmv.getRelation(); + result = relation; + relation.getBaseTablesOneLevelAndFromView(); + result = Sets.newHashSet(baseTableInfo); + } + }; + + TestDeltaExecutor executor = new TestDeltaExecutor(); + TestIvmRefreshManager manager = new TestIvmRefreshManager(executor, + newContext(mtmv), Collections.emptyList()); + manager.useSuperPrecheck = true; + + IvmRefreshResult result = manager.doRefresh(mtmv); + + Assertions.assertFalse(result.isSuccess()); + Assertions.assertEquals(FallbackReason.STREAM_UNSUPPORTED, result.getFallbackReason()); + Assertions.assertFalse(executor.executeCalled); + } + + @Test + public void testManagerPassesHealthyPrecheckAndExecutes(@Mocked MTMV mtmv, + @Mocked MTMVRelation relation, @Mocked OlapTable olapTable, @Mocked Command deltaWriteCommand) { + IvmInfo ivmInfo = new IvmInfo(); + new Expectations() { + { + olapTable.getId(); + result = 1L; + olapTable.getName(); + result = "t1"; + olapTable.getDBName(); + result = "db1"; + } + }; + BaseTableInfo baseTableInfo = new BaseTableInfo(olapTable, 2L); + ivmInfo.setBaseTableStreams(new HashMap<>()); + ivmInfo.getBaseTableStreams().put(baseTableInfo, new IvmStreamRef(StreamType.OLAP, null, null)); + new MockUp() { + @Mock + public TableIf getTable(BaseTableInfo input) { + return olapTable; + } + }; + new Expectations() { + { + mtmv.getIvmInfo(); + result = ivmInfo; + minTimes = 1; + mtmv.getRelation(); + result = relation; + relation.getBaseTablesOneLevelAndFromView(); + result = Sets.newHashSet(baseTableInfo); + } + }; + + TestDeltaExecutor executor = new TestDeltaExecutor(); + List bundles = makeBundles(deltaWriteCommand, mtmv); + TestIvmRefreshManager manager = new TestIvmRefreshManager(executor, newContext(mtmv), bundles); + manager.useSuperPrecheck = true; + + IvmRefreshResult result = manager.doRefresh(mtmv); + + Assertions.assertTrue(result.isSuccess()); + Assertions.assertTrue(executor.executeCalled); + } + + private static IvmRefreshContext newContext(MTMV mtmv) { + return new IvmRefreshContext(mtmv, new ConnectContext(), new org.apache.doris.mtmv.MTMVRefreshContext(mtmv)); + } + + private static List makeBundles(Command deltaWriteCommand, MTMV mtmv) { + return Collections.singletonList(new DeltaCommandBundle(new BaseTableInfo(mtmv, 0L), deltaWriteCommand)); + } + + private static class TestDeltaExecutor extends IvmDeltaExecutor { + private boolean executeCalled; + private boolean throwOnExecute; + private List lastBundles; + + @Override + public void execute(IvmRefreshContext context, List bundles) throws AnalysisException { + executeCalled = true; + lastBundles = bundles; + if (throwOnExecute) { + throw new AnalysisException("executor failed"); + } + } + } + + private static class TestIvmRefreshManager extends IvmRefreshManager { + private final IvmRefreshContext context; + private final List bundles; + private boolean throwOnBuild; + private boolean useSuperPrecheck; + + private TestIvmRefreshManager(IvmDeltaExecutor deltaExecutor, + IvmRefreshContext context, List bundles) { + super(deltaExecutor); + this.context = context; + this.bundles = bundles; + } + + @Override + IvmRefreshResult precheck(MTMV mtmv) { + if (useSuperPrecheck) { + return super.precheck(mtmv); + } + return IvmRefreshResult.success(); + } + + @Override + IvmRefreshContext buildRefreshContext(MTMV mtmv) throws Exception { + if (throwOnBuild) { + throw new AnalysisException("build context failed"); + } + return context; + } + + @Override + List analyzeDeltaCommandBundles(IvmRefreshContext ctx) { + return bundles; + } + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java index 288c5f65a2bd34..0668b64094dd37 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslatorTest.java @@ -17,6 +17,7 @@ package org.apache.doris.nereids.glue.translator; +import org.apache.doris.analysis.ArithmeticExpr; import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.GroupingInfo; import org.apache.doris.analysis.SlotRef; @@ -24,6 +25,7 @@ import org.apache.doris.catalog.Column; import org.apache.doris.catalog.KeysType; import org.apache.doris.catalog.OlapTable; +import org.apache.doris.nereids.processor.post.PlanPostProcessors; import org.apache.doris.nereids.properties.DataTrait; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; @@ -37,6 +39,7 @@ import org.apache.doris.nereids.trees.plans.PreAggStatus; import org.apache.doris.nereids.trees.plans.physical.PhysicalFilter; import org.apache.doris.nereids.trees.plans.physical.PhysicalOlapScan; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; import org.apache.doris.nereids.trees.plans.physical.PhysicalProject; import org.apache.doris.nereids.types.IntegerType; import org.apache.doris.nereids.util.PlanChecker; @@ -70,6 +73,9 @@ protected void runBeforeAll() throws Exception { createDatabase("test_db"); createTable("create table test_db.t(a int, b int) distributed by hash(a) buckets 3 " + "properties('replication_num' = '1');"); + createTable("create table test_db.t_topn_lazy(c1 int, c2 int, c3 int) " + + "duplicate key(c1) distributed by hash(c1) buckets 1 " + + "properties('replication_num' = '1', 'light_schema_change' = 'true');"); connectContext.getSessionVariable().setDisableNereidsRules("prune_empty_partition"); } @@ -165,4 +171,83 @@ public void testRepeatInputOutputOrder() throws Exception { } ); } + + @Test + public void testRootFragmentOutputExprsUseFinalAggregateTuple() throws Exception { + Planner planner = getSQLPlanner("select count(*) from test_db.t"); + PlanFragment rootFragment = planner.getFragments().get(0); + + Assertions.assertEquals(1, rootFragment.getOutputExprs().size()); + Assertions.assertInstanceOf(AggregationNode.class, rootFragment.getPlanRoot()); + Assertions.assertInstanceOf(SlotRef.class, rootFragment.getOutputExprs().get(0)); + + AggregationNode aggregationNode = (AggregationNode) rootFragment.getPlanRoot(); + SlotRef outputExpr = (SlotRef) rootFragment.getOutputExprs().get(0); + TupleDescriptor outputTuple = planner.getDescTable().getTupleDesc(aggregationNode.getOutputTupleIds().get(0)); + + Assertions.assertEquals(outputTuple.getSlots().get(0).getId(), outputExpr.getDesc().getId()); + } + + @Test + public void testRootFragmentOutputExprsPruneTopNOrderByOnlySlots() throws Exception { + Planner planner = getSQLPlanner( + "select Status from tasks('type'='mv') order by CreateTime desc limit 1"); + PlanFragment rootFragment = planner.getFragments().get(0); + + Assertions.assertEquals(1, rootFragment.getOutputExprs().size()); + Assertions.assertInstanceOf(SlotRef.class, rootFragment.getOutputExprs().get(0)); + Assertions.assertEquals("Status", ((SlotRef) rootFragment.getOutputExprs().get(0)).getColumnName()); + } + + @Test + public void testCountDistinctNullFragmentOutputExprsAreBound() throws Exception { + Planner planner = getSQLPlanner("select count(distinct NULL) from test_db.t"); + + Assertions.assertNotNull(planner); + Assertions.assertFalse(planner.getFragments().isEmpty()); + Assertions.assertEquals(1, planner.getFragments().get(0).getOutputExprs().size()); + Assertions.assertInstanceOf(SlotRef.class, planner.getFragments().get(0).getOutputExprs().get(0)); + + for (PlanFragment fragment : planner.getFragments()) { + if (fragment.getOutputExprs() == null) { + continue; + } + for (Expr outputExpr : fragment.getOutputExprs()) { + Assertions.assertNotNull(outputExpr); + } + } + } + + @Test + public void testRootFragmentOutputExprsKeepComputedProjectionAboveDeferredTopN() throws Exception { + boolean originEnableTwoPhaseReadOpt = connectContext.getSessionVariable().enableTwoPhaseReadOpt; + long originTopnOptLimitThreshold = connectContext.getSessionVariable().topnOptLimitThreshold; + int originTopnLazyMaterializationThreshold = + connectContext.getSessionVariable().topNLazyMaterializationThreshold; + try { + connectContext.getSessionVariable().enableTwoPhaseReadOpt = true; + connectContext.getSessionVariable().topnOptLimitThreshold = 1000; + connectContext.getSessionVariable().topNLazyMaterializationThreshold = -1; + + String sql = "select c1 + 1, c2 + 1 from " + + "(select c1, c2 from test_db.t_topn_lazy order by c1 limit 10) t"; + PlanChecker checker = PlanChecker.from(connectContext) + .analyze(sql) + .rewrite() + .implement(); + PhysicalPlan plan = checker.getPhysicalPlan(); + plan = new PlanPostProcessors(checker.getCascadesContext()).process(plan); + PlanFragment rootFragment = new PhysicalPlanTranslator( + new PlanTranslatorContext(checker.getCascadesContext())).translatePlan(plan); + + Assertions.assertEquals(2, rootFragment.getOutputExprs().size()); + rootFragment.getOutputExprs().forEach(Assertions::assertNotNull); + Assertions.assertTrue(rootFragment.getOutputExprs().stream().allMatch(ArithmeticExpr.class::isInstance)); + } finally { + connectContext.getSessionVariable().enableTwoPhaseReadOpt = originEnableTwoPhaseReadOpt; + connectContext.getSessionVariable().topnOptLimitThreshold = originTopnOptLimitThreshold; + connectContext.getSessionVariable().topNLazyMaterializationThreshold = + originTopnLazyMaterializationThreshold; + } + } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/IvmNormalizeMtmvTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/IvmNormalizeMtmvTest.java new file mode 100644 index 00000000000000..81803d3c61e44b --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/IvmNormalizeMtmvTest.java @@ -0,0 +1,536 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.rules.rewrite; + +import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.KeysType; +import org.apache.doris.catalog.OlapTable; +import org.apache.doris.catalog.TableProperty; +import org.apache.doris.mtmv.ivm.IvmAggMeta; +import org.apache.doris.mtmv.ivm.IvmAggMeta.AggTarget; +import org.apache.doris.mtmv.ivm.IvmAggMeta.AggType; +import org.apache.doris.mtmv.ivm.IvmNormalizeResult; +import org.apache.doris.mtmv.ivm.IvmUtil; +import org.apache.doris.nereids.CascadesContext; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.exceptions.AnalysisException; +import org.apache.doris.nereids.jobs.JobContext; +import org.apache.doris.nereids.properties.PhysicalProperties; +import org.apache.doris.nereids.trees.expressions.Alias; +import org.apache.doris.nereids.trees.expressions.ExprId; +import org.apache.doris.nereids.trees.expressions.Expression; +import org.apache.doris.nereids.trees.expressions.NamedExpression; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.expressions.functions.agg.AnyValue; +import org.apache.doris.nereids.trees.expressions.functions.agg.Avg; +import org.apache.doris.nereids.trees.expressions.functions.agg.Count; +import org.apache.doris.nereids.trees.expressions.functions.agg.Max; +import org.apache.doris.nereids.trees.expressions.functions.agg.Min; +import org.apache.doris.nereids.trees.expressions.functions.agg.Sum; +import org.apache.doris.nereids.trees.expressions.functions.scalar.UuidNumeric; +import org.apache.doris.nereids.trees.expressions.literal.BooleanLiteral; +import org.apache.doris.nereids.trees.expressions.literal.LargeIntLiteral; +import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; +import org.apache.doris.nereids.trees.plans.Plan; +import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType; +import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; +import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink; +import org.apache.doris.nereids.trees.plans.logical.LogicalProject; +import org.apache.doris.nereids.trees.plans.logical.LogicalSort; +import org.apache.doris.nereids.types.LargeIntType; +import org.apache.doris.nereids.util.MemoTestUtils; +import org.apache.doris.nereids.util.PlanConstructor; +import org.apache.doris.qe.ConnectContext; +import org.apache.doris.qe.SessionVariable; +import org.apache.doris.thrift.TPartialUpdateNewRowPolicy; + +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableSet; +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.math.BigInteger; +import java.util.ArrayList; +import java.util.List; +import java.util.Set; +import java.util.stream.Collectors; + +class IvmNormalizeMtmvTest { + + // DUP_KEYS table — row-id = UuidNumeric(), non-deterministic + private final LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0); + + @Test + void testGateDisabledKeepsPlanUnchanged() { + Plan result = new IvmNormalizeMtmv().rewriteRoot(scan, newJobContext(false)); + Assertions.assertSame(scan, result); + } + + @Test + void testScanInjectsRowIdAtIndexZero() { + JobContext jobContext = newJobContext(true); + Plan result = new IvmNormalizeMtmv().rewriteRoot(scan, jobContext); + + // scan is wrapped in a project + Assertions.assertInstanceOf(LogicalProject.class, result); + LogicalProject project = (LogicalProject) result; + Assertions.assertSame(scan, project.child()); + + // first output is the row-id alias + List outputs = project.getOutput(); + Assertions.assertEquals(scan.getOutput().size() + 1, outputs.size()); + Slot rowIdSlot = outputs.get(0); + Assertions.assertEquals(Column.IVM_ROW_ID_COL, rowIdSlot.getName()); + + // row-id expression is UuidNumeric for DUP_KEYS + Alias rowIdAlias = (Alias) project.getProjects().get(0); + Assertions.assertInstanceOf(UuidNumeric.class, rowIdAlias.child()); + + // IvmNormalizeResult records non-deterministic for DUP_KEYS + IvmNormalizeResult normalizeResult = jobContext.getCascadesContext().getIvmNormalizeResult().get(); + Assertions.assertEquals(1, normalizeResult.getRowIdDeterminism().size()); + Assertions.assertFalse(normalizeResult.getRowIdDeterminism().values().iterator().next()); + } + + @Test + void testProjectOnScanPropagatesRowId() { + Slot slot = scan.getOutput().get(0); + LogicalProject project = new LogicalProject<>(ImmutableList.of(slot), scan); + + Plan result = new IvmNormalizeMtmv().rewriteRoot(project, newJobContext(true)); + + // outer project has row-id at index 0 + Assertions.assertInstanceOf(LogicalProject.class, result); + LogicalProject outer = (LogicalProject) result; + Assertions.assertEquals(Column.IVM_ROW_ID_COL, outer.getOutput().get(0).getName()); + // child is the scan-wrapping project + Assertions.assertInstanceOf(LogicalProject.class, outer.child()); + Assertions.assertSame(scan, ((LogicalProject) outer.child()).child()); + } + + @Test + void testProjectReplacesRowIdPlaceholderAndKeepsExprId() { + Alias placeholder = new Alias(new NullLiteral(LargeIntType.INSTANCE), Column.IVM_ROW_ID_COL); + ExprId placeholderExprId = placeholder.getExprId(); + Slot slot = scan.getOutput().get(0); + LogicalProject project = new LogicalProject<>(ImmutableList.of(placeholder, slot), scan); + + Plan result = new IvmNormalizeMtmv().rewriteRoot(project, newJobContext(true)); + + Assertions.assertInstanceOf(LogicalProject.class, result); + LogicalProject rewrittenProject = (LogicalProject) result; + Assertions.assertInstanceOf(Alias.class, rewrittenProject.getProjects().get(0)); + Alias rewrittenRowId = (Alias) rewrittenProject.getProjects().get(0); + Assertions.assertEquals(placeholderExprId, rewrittenRowId.getExprId()); + Assertions.assertEquals(Column.IVM_ROW_ID_COL, rewrittenRowId.getName()); + Assertions.assertInstanceOf(Slot.class, rewrittenRowId.child()); + Assertions.assertEquals(Column.IVM_ROW_ID_COL, ((Slot) rewrittenRowId.child()).getName()); + } + + @Test + void testSinkWithPlaceholderChildReplacesRowIdAndPreservesExprId() { + // Simulate what BindSink produces for an incremental MTMV full-refresh: + // a project child with user columns + a NULL placeholder for the IVM row-id at the end. + Slot k1Slot = scan.getOutput().get(0); + Alias rowIdPlaceholder = new Alias(new NullLiteral(LargeIntType.INSTANCE), Column.IVM_ROW_ID_COL); + ExprId placeholderExprId = rowIdPlaceholder.getExprId(); + LogicalProject projectWithPlaceholder = new LogicalProject<>( + ImmutableList.of(k1Slot, rowIdPlaceholder), scan); + LogicalOlapTableSink sink = new LogicalOlapTableSink<>( + new Database(), + scan.getTable(), + ImmutableList.of(scan.getTable().getBaseSchema().get(0)), + new ArrayList<>(), + ImmutableList.of(k1Slot, rowIdPlaceholder.toSlot()), + false, + TPartialUpdateNewRowPolicy.APPEND, + DMLCommandType.NONE, + projectWithPlaceholder); + + Plan result = new IvmNormalizeMtmv().rewriteRoot(sink, newJobContextForRoot(sink, true)); + + Assertions.assertInstanceOf(LogicalOlapTableSink.class, result); + LogicalOlapTableSink rewrittenSink = (LogicalOlapTableSink) result; + // child is a project with the placeholder replaced + Assertions.assertInstanceOf(LogicalProject.class, rewrittenSink.child()); + LogicalProject rewrittenProject = (LogicalProject) rewrittenSink.child(); + // non-IVM column unchanged at index 0 + Assertions.assertEquals(k1Slot.getName(), rewrittenProject.getProjects().get(0).getName()); + // IVM placeholder at index 1 is now an Alias wrapping the real row-id scan slot + Assertions.assertInstanceOf(Alias.class, rewrittenProject.getProjects().get(1)); + Alias rewrittenRowId = (Alias) rewrittenProject.getProjects().get(1); + Assertions.assertEquals(placeholderExprId, rewrittenRowId.getExprId()); + Assertions.assertEquals(Column.IVM_ROW_ID_COL, rewrittenRowId.getName()); + Assertions.assertInstanceOf(Slot.class, rewrittenRowId.child()); + // sink outputExprs updated via withChildAndUpdateOutput — row-id slot at index 1 + Assertions.assertEquals(Column.IVM_ROW_ID_COL, rewrittenSink.getOutputExprs().get(1).getName()); + } + + @Test + void testLogicalOlapTableSinkKeepsSinkShapeAndNormalizesChild() { + Slot slot = scan.getOutput().get(0); + LogicalOlapTableSink sink = new LogicalOlapTableSink<>( + new Database(), + scan.getTable(), + ImmutableList.of(scan.getTable().getBaseSchema().get(0)), + new ArrayList<>(), + ImmutableList.of(slot), + false, + TPartialUpdateNewRowPolicy.APPEND, + DMLCommandType.NONE, + scan); + + Plan result = new IvmNormalizeMtmv().rewriteRoot(sink, newJobContextForRoot(sink, true)); + + Assertions.assertInstanceOf(LogicalOlapTableSink.class, result); + LogicalOlapTableSink rewrittenSink = (LogicalOlapTableSink) result; + Assertions.assertEquals(ImmutableList.of(slot.getName()), + rewrittenSink.getCols().stream().map(org.apache.doris.catalog.Column::getName) + .collect(Collectors.toList())); + Assertions.assertEquals(Column.IVM_ROW_ID_COL, rewrittenSink.getOutputExprs().get(0).getName()); + Assertions.assertInstanceOf(LogicalProject.class, rewrittenSink.child()); + Assertions.assertEquals(Column.IVM_ROW_ID_COL, rewrittenSink.child().getOutput().get(0).getName()); + } + + @Test + void testMowTableRowIdIsDeterministic() { + OlapTable mowTable = PlanConstructor.newOlapTable(10, "mow", 0, KeysType.UNIQUE_KEYS); + TableProperty tableProperty = new TableProperty(new java.util.HashMap<>()); + tableProperty.setEnableUniqueKeyMergeOnWrite(true); + mowTable.setTableProperty(tableProperty); + LogicalOlapScan mowScan = new LogicalOlapScan( + PlanConstructor.getNextRelationId(), mowTable, ImmutableList.of("db")); + + JobContext jobContext = newJobContextForScan(mowScan, true); + Plan result = new IvmNormalizeMtmv().rewriteRoot(mowScan, jobContext); + + Assertions.assertInstanceOf(LogicalProject.class, result); + Assertions.assertEquals(Column.IVM_ROW_ID_COL, result.getOutput().get(0).getName()); + IvmNormalizeResult normalizeResult = jobContext.getCascadesContext().getIvmNormalizeResult().get(); + Assertions.assertTrue(normalizeResult.getRowIdDeterminism().values().iterator().next()); + } + + @Test + void testMorTableThrows() { + // UNIQUE_KEYS without MOW (MOR) is not supported + OlapTable morTable = PlanConstructor.newOlapTable(11, "mor", 0, KeysType.UNIQUE_KEYS); + LogicalOlapScan morScan = new LogicalOlapScan( + PlanConstructor.getNextRelationId(), morTable, ImmutableList.of("db")); + + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> new IvmNormalizeMtmv().rewriteRoot(morScan, newJobContextForScan(morScan, true))); + } + + @Test + void testAggKeyTableThrows() { + OlapTable aggTable = PlanConstructor.newOlapTable(12, "agg", 0, KeysType.AGG_KEYS); + LogicalOlapScan aggScan = new LogicalOlapScan( + PlanConstructor.getNextRelationId(), aggTable, ImmutableList.of("db")); + + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> new IvmNormalizeMtmv().rewriteRoot(aggScan, newJobContextForScan(aggScan, true))); + } + + @Test + void testUnsupportedPlanNodeThrows() { + LogicalSort sort = new LogicalSort<>(ImmutableList.of(), scan); + + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> new IvmNormalizeMtmv().rewriteRoot(sort, newJobContext(true))); + } + + @Test + void testUnsupportedNodeAsChildThrows() { + Slot slot = scan.getOutput().get(0); + LogicalSort sort = new LogicalSort<>(ImmutableList.of(), scan); + LogicalProject project = new LogicalProject<>(ImmutableList.of(slot), sort); + + Assertions.assertThrows(org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> new IvmNormalizeMtmv().rewriteRoot(project, newJobContext(true))); + } + + @Test + void testNormalizedPlanStoredInIvmNormalizeResult() { + JobContext jobContext = newJobContext(true); + Plan result = new IvmNormalizeMtmv().rewriteRoot(scan, jobContext); + + IvmNormalizeResult normalizeResult = jobContext.getCascadesContext().getIvmNormalizeResult().get(); + Assertions.assertNotNull(normalizeResult.getNormalizedPlan()); + Assertions.assertSame(result, normalizeResult.getNormalizedPlan()); + } + + @Test + void testIdempotencyGuardSkipsSecondRewrite() { + JobContext jobContext = newJobContext(true); + Plan firstResult = new IvmNormalizeMtmv().rewriteRoot(scan, jobContext); + // Second rewrite on the same CascadesContext should return root unchanged + Plan secondResult = new IvmNormalizeMtmv().rewriteRoot(firstResult, jobContext); + Assertions.assertSame(firstResult, secondResult); + } + + // ======================== Aggregate tests ======================== + + /** + * Builds a normalized aggregate: Aggregate(groupBy=[idSlot], output=[idSlot, Alias(Sum(nameSlot))]) + * over the DUP_KEYS scan. This is the shape NormalizeAggregate produces. + */ + private LogicalAggregate buildGroupedAgg() { + // scan has: id (INT), name (STRING) + Slot idSlot = scan.getOutput().get(0); + Slot nameSlot = scan.getOutput().get(1); + Alias sumAlias = new Alias(new Sum(nameSlot), "sum_name"); + List groupBy = ImmutableList.of(idSlot); + List outputs = ImmutableList.of(idSlot, sumAlias); + return new LogicalAggregate<>(groupBy, outputs, true, java.util.Optional.empty(), scan); + } + + /** + * Builds a scalar aggregate (no GROUP BY): Aggregate(groupBy=[], output=[Alias(Count())]) + */ + private LogicalAggregate buildScalarAgg() { + Alias countAlias = new Alias(new Count(), "cnt"); + List groupBy = ImmutableList.of(); + List outputs = ImmutableList.of(countAlias); + return new LogicalAggregate<>(groupBy, outputs, true, java.util.Optional.empty(), scan); + } + + @Test + void testGroupedAggInjectsRowIdAndHiddenColumns() { + LogicalAggregate agg = buildGroupedAgg(); + JobContext jobContext = newJobContextForRoot(agg, true); + Plan result = new IvmNormalizeMtmv().rewriteRoot(agg, jobContext); + + // Result is a Project wrapping the modified Aggregate + Assertions.assertInstanceOf(LogicalProject.class, result); + LogicalProject topProject = (LogicalProject) result; + Assertions.assertInstanceOf(LogicalAggregate.class, topProject.child()); + + // Top project outputs: [row_id, id, sum_name, __DORIS_IVM_AGG_COUNT_COL__, hidden_0_SUM, hidden_0_COUNT] + List outputNames = topProject.getOutput().stream() + .map(Slot::getName).collect(Collectors.toList()); + Assertions.assertEquals(Column.IVM_ROW_ID_COL, outputNames.get(0)); + Assertions.assertEquals("id", outputNames.get(1)); + Assertions.assertEquals("sum_name", outputNames.get(2)); + Assertions.assertEquals(Column.IVM_AGG_COUNT_COL, outputNames.get(3)); + Assertions.assertEquals(IvmUtil.ivmAggHiddenColumnName(0, "SUM"), outputNames.get(4)); + Assertions.assertEquals(IvmUtil.ivmAggHiddenColumnName(0, "COUNT"), outputNames.get(5)); + + // row-id expression is hash(id) via Cast(MurmurHash364) + Alias rowIdAlias = (Alias) topProject.getProjects().get(0); + Assertions.assertInstanceOf( + org.apache.doris.nereids.trees.expressions.Cast.class, rowIdAlias.child()); + + // IvmNormalizeResult has aggMeta + IvmNormalizeResult normalizeResult = jobContext.getCascadesContext().getIvmNormalizeResult().get(); + IvmAggMeta aggMeta = normalizeResult.getAggMeta(); + Assertions.assertNotNull(aggMeta); + Assertions.assertFalse(aggMeta.isScalarAgg()); + Assertions.assertEquals(1, aggMeta.getGroupKeySlots().size()); + Assertions.assertEquals("id", aggMeta.getGroupKeySlots().get(0).getName()); + Assertions.assertEquals(Column.IVM_AGG_COUNT_COL, aggMeta.getGroupCountSlot().getName()); + + // One agg target: SUM + Assertions.assertEquals(1, aggMeta.getAggTargets().size()); + AggTarget target = aggMeta.getAggTargets().get(0); + Assertions.assertEquals(0, target.getOrdinal()); + Assertions.assertEquals(AggType.SUM, target.getAggType()); + Assertions.assertEquals("sum_name", target.getVisibleSlot().getName()); + Assertions.assertNotNull(target.getHiddenStateSlot("SUM")); + Assertions.assertNotNull(target.getHiddenStateSlot("COUNT")); + + // Row-id determinism: grouped agg → deterministic + Assertions.assertTrue(normalizeResult.getRowIdDeterminism().values().iterator().next()); + } + + @Test + void testScalarAggRowIdIsZeroConstant() { + LogicalAggregate agg = buildScalarAgg(); + JobContext jobContext = newJobContextForRoot(agg, true); + Plan result = new IvmNormalizeMtmv().rewriteRoot(agg, jobContext); + + Assertions.assertInstanceOf(LogicalProject.class, result); + LogicalProject topProject = (LogicalProject) result; + + // row-id is LargeIntLiteral(0) for scalar agg + Alias rowIdAlias = (Alias) topProject.getProjects().get(0); + Assertions.assertInstanceOf(LargeIntLiteral.class, rowIdAlias.child()); + Assertions.assertEquals(BigInteger.ZERO, ((LargeIntLiteral) rowIdAlias.child()).getValue()); + + // IvmAggMeta: scalar, no group keys + IvmNormalizeResult normalizeResult = jobContext.getCascadesContext().getIvmNormalizeResult().get(); + IvmAggMeta aggMeta = normalizeResult.getAggMeta(); + Assertions.assertTrue(aggMeta.isScalarAgg()); + Assertions.assertTrue(aggMeta.getGroupKeySlots().isEmpty()); + Assertions.assertEquals(1, aggMeta.getAggTargets().size()); + Assertions.assertEquals(AggType.COUNT_STAR, aggMeta.getAggTargets().get(0).getAggType()); + + // Row-id determinism: scalar agg → non-deterministic + Assertions.assertFalse(normalizeResult.getRowIdDeterminism().values().iterator().next()); + } + + @Test + void testMultipleAggFunctionsProduceCorrectHiddenColumns() { + Slot idSlot = scan.getOutput().get(0); + Slot nameSlot = scan.getOutput().get(1); + // SELECT id, COUNT(*), SUM(name), AVG(name), MIN(name), MAX(name) GROUP BY id + Alias countStarAlias = new Alias(new Count(), "cnt"); + Alias sumAlias = new Alias(new Sum(nameSlot), "s"); + Alias avgAlias = new Alias(new Avg(nameSlot), "a"); + Alias minAlias = new Alias(new Min(nameSlot), "mn"); + Alias maxAlias = new Alias(new Max(nameSlot), "mx"); + + List groupBy = ImmutableList.of(idSlot); + List outputs = ImmutableList.of( + idSlot, countStarAlias, sumAlias, avgAlias, minAlias, maxAlias); + LogicalAggregate agg = new LogicalAggregate<>( + groupBy, outputs, true, java.util.Optional.empty(), scan); + + JobContext jobContext = newJobContextForRoot(agg, true); + Plan result = new IvmNormalizeMtmv().rewriteRoot(agg, jobContext); + + IvmNormalizeResult normalizeResult = jobContext.getCascadesContext().getIvmNormalizeResult().get(); + IvmAggMeta aggMeta = normalizeResult.getAggMeta(); + Assertions.assertEquals(5, aggMeta.getAggTargets().size()); + + // ordinal 0: COUNT_STAR → hidden: COUNT + AggTarget t0 = aggMeta.getAggTargets().get(0); + Assertions.assertEquals(AggType.COUNT_STAR, t0.getAggType()); + Assertions.assertEquals(1, t0.getHiddenStateSlots().size()); + Assertions.assertNotNull(t0.getHiddenStateSlot("COUNT")); + + // ordinal 1: SUM → hidden: SUM, COUNT + AggTarget t1 = aggMeta.getAggTargets().get(1); + Assertions.assertEquals(AggType.SUM, t1.getAggType()); + Assertions.assertEquals(2, t1.getHiddenStateSlots().size()); + + // ordinal 2: AVG → hidden: SUM, COUNT + AggTarget t2 = aggMeta.getAggTargets().get(2); + Assertions.assertEquals(AggType.AVG, t2.getAggType()); + Assertions.assertEquals(2, t2.getHiddenStateSlots().size()); + + // ordinal 3: MIN → hidden: MIN, COUNT + AggTarget t3 = aggMeta.getAggTargets().get(3); + Assertions.assertEquals(AggType.MIN, t3.getAggType()); + Assertions.assertNotNull(t3.getHiddenStateSlot("MIN")); + + // ordinal 4: MAX → hidden: MAX, COUNT + AggTarget t4 = aggMeta.getAggTargets().get(4); + Assertions.assertEquals(AggType.MAX, t4.getAggType()); + Assertions.assertNotNull(t4.getHiddenStateSlot("MAX")); + + // Verify hidden column naming in the project output + LogicalProject topProject = (LogicalProject) result; + Set outputNames = topProject.getOutput().stream() + .map(Slot::getName).collect(Collectors.toSet()); + // Global group count + Assertions.assertTrue(outputNames.contains(Column.IVM_AGG_COUNT_COL)); + // Per-agg hidden columns + Assertions.assertTrue(outputNames.contains(IvmUtil.ivmAggHiddenColumnName(0, "COUNT"))); + Assertions.assertTrue(outputNames.contains(IvmUtil.ivmAggHiddenColumnName(1, "SUM"))); + Assertions.assertTrue(outputNames.contains(IvmUtil.ivmAggHiddenColumnName(1, "COUNT"))); + Assertions.assertTrue(outputNames.contains(IvmUtil.ivmAggHiddenColumnName(2, "SUM"))); + Assertions.assertTrue(outputNames.contains(IvmUtil.ivmAggHiddenColumnName(2, "COUNT"))); + Assertions.assertTrue(outputNames.contains(IvmUtil.ivmAggHiddenColumnName(3, "MIN"))); + Assertions.assertTrue(outputNames.contains(IvmUtil.ivmAggHiddenColumnName(3, "COUNT"))); + Assertions.assertTrue(outputNames.contains(IvmUtil.ivmAggHiddenColumnName(4, "MAX"))); + Assertions.assertTrue(outputNames.contains(IvmUtil.ivmAggHiddenColumnName(4, "COUNT"))); + } + + @Test + void testCountExprProducesCountExprType() { + Slot nameSlot = scan.getOutput().get(1); + Alias countExprAlias = new Alias(new Count(nameSlot), "cnt_name"); + List outputs = ImmutableList.of(countExprAlias); + LogicalAggregate agg = new LogicalAggregate<>( + ImmutableList.of(), outputs, true, java.util.Optional.empty(), scan); + + JobContext jobContext = newJobContextForRoot(agg, true); + new IvmNormalizeMtmv().rewriteRoot(agg, jobContext); + + IvmAggMeta aggMeta = jobContext.getCascadesContext().getIvmNormalizeResult().get().getAggMeta(); + Assertions.assertEquals(AggType.COUNT_EXPR, aggMeta.getAggTargets().get(0).getAggType()); + } + + @Test + void testAggUnderFilterThrows() { + LogicalAggregate agg = buildGroupedAgg(); + LogicalFilter filter = new LogicalFilter<>( + ImmutableSet.of(BooleanLiteral.TRUE), agg); + + Assertions.assertThrows(AnalysisException.class, + () -> new IvmNormalizeMtmv().rewriteRoot(filter, newJobContextForRoot(filter, true))); + } + + @Test + void testDistinctAggThrows() { + Slot nameSlot = scan.getOutput().get(1); + Alias distinctCount = new Alias(new Count(true, nameSlot), "cnt_distinct"); + List outputs = ImmutableList.of(distinctCount); + LogicalAggregate agg = new LogicalAggregate<>( + ImmutableList.of(), outputs, true, java.util.Optional.empty(), scan); + + Assertions.assertThrows(AnalysisException.class, + () -> new IvmNormalizeMtmv().rewriteRoot(agg, newJobContextForRoot(agg, true))); + } + + @Test + void testUnsupportedAggFunctionThrows() { + Slot nameSlot = scan.getOutput().get(1); + Alias anyValAlias = new Alias(new AnyValue(nameSlot), "av"); + List outputs = ImmutableList.of(anyValAlias); + LogicalAggregate agg = new LogicalAggregate<>( + ImmutableList.of(), outputs, true, java.util.Optional.empty(), scan); + + Assertions.assertThrows(AnalysisException.class, + () -> new IvmNormalizeMtmv().rewriteRoot(agg, newJobContextForRoot(agg, true))); + } + + @Test + void testBareGroupByWithoutAggFunctionsThrows() { + Slot idSlot = scan.getOutput().get(0); + // GROUP BY id with no aggregate functions + List groupBy = ImmutableList.of(idSlot); + List outputs = ImmutableList.of(idSlot); + LogicalAggregate agg = new LogicalAggregate<>( + groupBy, outputs, true, java.util.Optional.empty(), scan); + + Assertions.assertThrows(AnalysisException.class, + () -> new IvmNormalizeMtmv().rewriteRoot(agg, newJobContextForRoot(agg, true))); + } + + private JobContext newJobContext(boolean enableIvmNormalRewrite) { + return newJobContextForScan(scan, enableIvmNormalRewrite); + } + + private JobContext newJobContextForScan(LogicalOlapScan rootScan, boolean enableIvmNormalRewrite) { + return newJobContextForRoot(rootScan, enableIvmNormalRewrite); + } + + private JobContext newJobContextForRoot(Plan root, boolean enableIvmNormalRewrite) { + ConnectContext connectContext = MemoTestUtils.createConnectContext(); + SessionVariable sessionVariable = new SessionVariable(); + sessionVariable.setEnableIvmNormalRewrite(enableIvmNormalRewrite); + connectContext.setSessionVariable(sessionVariable); + StatementContext statementContext = new StatementContext(connectContext, null); + CascadesContext cascadesContext = CascadesContext.initContext(statementContext, root, PhysicalProperties.ANY); + return new JobContext(cascadesContext, PhysicalProperties.ANY); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateMTMVCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateMTMVCommandTest.java new file mode 100644 index 00000000000000..96d9adf53cc675 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateMTMVCommandTest.java @@ -0,0 +1,565 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.nereids.trees.plans; + +import org.apache.doris.analysis.PartitionDesc; +import org.apache.doris.analysis.SinglePartitionDesc; +import org.apache.doris.catalog.Column; +import org.apache.doris.mtmv.MTMVRefreshEnum.RefreshMethod; +import org.apache.doris.nereids.parser.NereidsParser; +import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand; +import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand; +import org.apache.doris.nereids.trees.plans.commands.info.CreateMTMVInfo; +import org.apache.doris.nereids.trees.plans.commands.info.FixedRangePartition; +import org.apache.doris.nereids.trees.plans.commands.info.InPartition; +import org.apache.doris.nereids.trees.plans.commands.info.LessThanPartition; +import org.apache.doris.nereids.trees.plans.commands.info.PartitionDefinition; +import org.apache.doris.nereids.trees.plans.commands.info.PartitionTableInfo; +import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; +import org.apache.doris.utframe.TestWithFeService; + +import org.junit.jupiter.api.Assertions; +import org.junit.jupiter.api.Test; + +import java.util.List; +import java.util.stream.Collectors; + +public class CreateMTMVCommandTest extends TestWithFeService { + @Override + protected void runBeforeAll() throws Exception { + createDatabase("test"); + connectContext.setDatabase("test"); + } + + @Override + public void createTable(String sql) throws Exception { + LogicalPlan plan = new NereidsParser().parseSingle(sql); + Assertions.assertTrue(plan instanceof CreateTableCommand); + ((CreateTableCommand) plan).run(connectContext, null); + } + + @Test + public void testUnpartitionConvertToPartitionTableInfo() throws Exception { + String partitionTable = "CREATE TABLE aa1 (\n" + + " `user_id` LARGEINT NOT NULL COMMENT '\\\"用户id\\\"',\n" + + " `date` DATE NOT NULL COMMENT '\\\"数据灌入日期时间\\\"',\n" + + " `num` SMALLINT NOT NULL COMMENT '\\\"数量\\\"'\n" + + " ) ENGINE=OLAP\n" + + " DUPLICATE KEY(`user_id`, `date`, `num`)\n" + + " COMMENT 'OLAP'\n" + + " PARTITION BY RANGE(`date`)\n" + + " (\n" + + " PARTITION `p201701` VALUES [(\"2017-01-01\"), (\"2017-02-01\")),\n" + + " PARTITION `p201702` VALUES [(\"2017-02-01\"), (\"2017-03-01\")),\n" + + " PARTITION `p201703` VALUES [(\"2017-03-01\"), (\"2017-04-01\"))\n" + + " )\n" + + " DISTRIBUTED BY HASH(`user_id`) BUCKETS 2\n" + + " PROPERTIES ('replication_num' = '1') ;\n"; + createTable(partitionTable); + + String mv = "CREATE MATERIALIZED VIEW mtmv5\n" + + " BUILD DEFERRED REFRESH AUTO ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 2\n" + + " PROPERTIES ('replication_num' = '1')\n" + + " AS\n" + + " SELECT * FROM aa1;"; + + CreateMTMVInfo createMTMVInfo = getPartitionTableInfo(mv); + Assertions.assertEquals(PartitionTableInfo.EMPTY, createMTMVInfo.getPartitionTableInfo()); + } + + @Test + public void testRangePartitionConvertToPartitionTableInfo() throws Exception { + String fixedRangePartitionTable = "CREATE TABLE mm1 (\n" + + " `user_id` LARGEINT NOT NULL COMMENT '\\\"用户id\\\"',\n" + + " `date` DATE NOT NULL COMMENT '\\\"数据灌入日期时间\\\"',\n" + + " `num` SMALLINT NOT NULL COMMENT '\\\"数量\\\"'\n" + + " ) ENGINE=OLAP\n" + + " DUPLICATE KEY(`user_id`, `date`, `num`)\n" + + " COMMENT 'OLAP'\n" + + " PARTITION BY RANGE(`date`)\n" + + " (\n" + + " PARTITION `p201701` VALUES [(\"2017-01-01\"), (\"2017-02-01\")),\n" + + " PARTITION `p201702` VALUES [(\"2017-02-01\"), (\"2017-03-01\")),\n" + + " PARTITION `p201703` VALUES [(\"2017-03-01\"), (\"2017-04-01\"))\n" + + " )\n" + + " DISTRIBUTED BY HASH(`user_id`) BUCKETS 2\n" + + " PROPERTIES ('replication_num' = '1') ;\n"; + + String mv = "CREATE MATERIALIZED VIEW mtmv1\n" + + " BUILD DEFERRED REFRESH AUTO ON MANUAL\n" + + " partition by(`date`)\n" + + " DISTRIBUTED BY RANDOM BUCKETS 2\n" + + " PROPERTIES ('replication_num' = '1')\n" + + " AS\n" + + " SELECT * FROM mm1;"; + + check(fixedRangePartitionTable, mv); + } + + @Test + public void testLessThanPartitionConvertToPartitionTableInfo() throws Exception { + String lessThanPartitionTable = "CREATE TABLE te2 (\n" + + " `user_id` LARGEINT NOT NULL COMMENT '\\\"用户id\\\"',\n" + + " `date` DATE NOT NULL COMMENT '\\\"数据灌入日期时间\\\"',\n" + + " `num` SMALLINT NOT NULL COMMENT '\\\"数量\\\"'\n" + + " ) ENGINE=OLAP\n" + + " DUPLICATE KEY(`user_id`, `date`, `num`)\n" + + " COMMENT 'OLAP'\n" + + " PARTITION BY RANGE(`date`)\n" + + "(\n" + + " PARTITION `p201701` VALUES LESS THAN (\"2017-02-01\"),\n" + + " PARTITION `p201702` VALUES LESS THAN (\"2017-03-01\"),\n" + + " PARTITION `p201703` VALUES LESS THAN (\"2017-04-01\"),\n" + + " PARTITION `p2018` VALUES [(\"2018-01-01\"), (\"2019-01-01\")),\n" + + " PARTITION `other` VALUES LESS THAN (MAXVALUE)\n" + + ")\n" + + " DISTRIBUTED BY HASH(`user_id`) BUCKETS 2\n" + + " PROPERTIES ('replication_num' = '1') ;"; + + String mv = "CREATE MATERIALIZED VIEW mtmv2\n" + + " BUILD DEFERRED REFRESH AUTO ON MANUAL\n" + + " partition by(`date`)\n" + + " DISTRIBUTED BY RANDOM BUCKETS 2\n" + + " PROPERTIES ('replication_num' = '1')\n" + + " AS\n" + + " SELECT * FROM te2;"; + + check(lessThanPartitionTable, mv); + } + + @Test + public void testInPartitionConvertToPartitionTableInfo() throws Exception { + String inPartitionTable = "CREATE TABLE cc1 (\n" + + "`user_id` LARGEINT NOT NULL COMMENT '\\\"用户id\\\"',\n" + + "`date` DATE NOT NULL COMMENT '\\\"数据灌入日期时间\\\"',\n" + + "`num` SMALLINT NOT NULL COMMENT '\\\"数量\\\"'\n" + + ") ENGINE=OLAP\n" + + "DUPLICATE KEY(`user_id`, `date`, `num`)\n" + + "COMMENT 'OLAP'\n" + + "PARTITION BY LIST(`date`,`num`)\n" + + "(\n" + + " PARTITION p201701_1000 VALUES IN (('2017-01-01',1), ('2017-01-01',2)),\n" + + " PARTITION p201702_2000 VALUES IN (('2017-02-01',3), ('2017-02-01',4))\n" + + " )\n" + + " DISTRIBUTED BY HASH(`user_id`) BUCKETS 2\n" + + " PROPERTIES ('replication_num' = '1') ;"; + + String mv = "CREATE MATERIALIZED VIEW mtmv\n" + + "BUILD DEFERRED REFRESH AUTO ON MANUAL\n" + + "partition by(`date`)\n" + + "DISTRIBUTED BY RANDOM BUCKETS 2\n" + + "PROPERTIES ('replication_num' = '1')\n" + + "AS\n" + + "SELECT * FROM cc1;"; + + check(inPartitionTable, mv); + } + + private void check(String sql, String mv) throws Exception { + createTable(sql); + + CreateMTMVInfo createMTMVInfo = getPartitionTableInfo(mv); + PartitionTableInfo partitionTableInfo = createMTMVInfo.getPartitionTableInfo(); + PartitionDesc partitionDesc = createMTMVInfo.getPartitionDesc(); + + List partitionDefs = partitionTableInfo.getPartitionDefs(); + List singlePartitionDescs = partitionDesc.getSinglePartitionDescs(); + + assertPartitionInfo(partitionDefs, singlePartitionDescs); + } + + private void assertPartitionInfo(List partitionDefs, List singlePartitionDescs) { + Assertions.assertEquals(singlePartitionDescs.size(), partitionDefs.size()); + + for (int i = 0; i < singlePartitionDescs.size(); i++) { + PartitionDefinition partitionDefinition = partitionDefs.get(i); + SinglePartitionDesc singlePartitionDesc = singlePartitionDescs.get(i); + + if (partitionDefinition instanceof InPartition) { + InPartition inPartition = (InPartition) partitionDefinition; + + Assertions.assertEquals(singlePartitionDesc.getPartitionName(), partitionDefinition.getPartitionName()); + Assertions.assertEquals(singlePartitionDesc.getPartitionKeyDesc().getPartitionType().name(), "IN"); + Assertions.assertEquals(singlePartitionDesc.getPartitionKeyDesc().getInValues().size(), inPartition.getValues().size()); + } else if (partitionDefinition instanceof FixedRangePartition) { + FixedRangePartition fixedRangePartition = (FixedRangePartition) partitionDefinition; + + Assertions.assertEquals(singlePartitionDesc.getPartitionName(), partitionDefinition.getPartitionName()); + Assertions.assertEquals(singlePartitionDesc.getPartitionKeyDesc().getPartitionType().name(), "FIXED"); + Assertions.assertEquals(fixedRangePartition.getLowerBounds().size(), singlePartitionDesc.getPartitionKeyDesc().getLowerValues().size()); + Assertions.assertEquals(fixedRangePartition.getUpperBounds().size(), singlePartitionDesc.getPartitionKeyDesc().getUpperValues().size()); + } else if (partitionDefinition instanceof LessThanPartition) { + LessThanPartition lessThanPartition = (LessThanPartition) partitionDefinition; + + Assertions.assertEquals(singlePartitionDesc.getPartitionName(), partitionDefinition.getPartitionName()); + Assertions.assertEquals(singlePartitionDesc.getPartitionKeyDesc().getPartitionType().name(), "LESS_THAN"); + Assertions.assertEquals(lessThanPartition.getValues().size(), singlePartitionDesc.getPartitionKeyDesc().getUpperValues().size()); + } + } + } + + private CreateMTMVInfo getPartitionTableInfo(String sql) throws Exception { + NereidsParser nereidsParser = new NereidsParser(); + LogicalPlan logicalPlan = nereidsParser.parseSingle(sql); + Assertions.assertTrue(logicalPlan instanceof CreateMTMVCommand); + CreateMTMVCommand command = (CreateMTMVCommand) logicalPlan; + command.getCreateMTMVInfo().analyze(connectContext); + + return command.getCreateMTMVInfo(); + } + + @Test + public void testMTMVRejectVarbinary() throws Exception { + String mv = "CREATE MATERIALIZED VIEW mv_vb\n" + + " BUILD DEFERRED REFRESH AUTO ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 2\n" + + " PROPERTIES ('replication_num' = '1')\n" + + " AS SELECT X'AB' as vb;"; + + LogicalPlan plan = new NereidsParser().parseSingle(mv); + Assertions.assertTrue(plan instanceof CreateMTMVCommand); + CreateMTMVCommand cmd = (CreateMTMVCommand) plan; + + org.apache.doris.nereids.exceptions.AnalysisException ex = Assertions.assertThrows( + org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> cmd.getCreateMTMVInfo().analyze(connectContext)); + System.out.println(ex.getMessage()); + Assertions.assertTrue(ex.getMessage().contains("MTMV do not support varbinary type")); + Assertions.assertTrue(ex.getMessage().contains("vb")); + } + + @Test + public void testCreateMTMVWithIncrementRefreshMethod() throws Exception { + String mv = "CREATE MATERIALIZED VIEW mtmv_increment\n" + + " BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 2\n" + + " PROPERTIES ('replication_num' = '1')\n" + + " AS SELECT 1 AS k1;"; + + LogicalPlan plan = new NereidsParser().parseSingle(mv); + Assertions.assertTrue(plan instanceof CreateMTMVCommand); + CreateMTMVCommand cmd = (CreateMTMVCommand) plan; + + Assertions.assertEquals(RefreshMethod.INCREMENTAL, + cmd.getCreateMTMVInfo().getRefreshInfo().getRefreshMethod()); + } + + @Test + public void testCreateMTMVRewriteQuerySqlWithDefinedColumnsForScanPlan() throws Exception { + createTable("create table test.mtmv_scan_base (id int, score int)\n" + + "duplicate key(id)\n" + + "distributed by hash(id) buckets 1\n" + + "properties('replication_num' = '1');"); + + CreateMTMVInfo createMTMVInfo = getPartitionTableInfo("CREATE MATERIALIZED VIEW mtmv_scan_alias" + + " (mv_id, mv_score)\n" + + " BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 2\n" + + " PROPERTIES ('replication_num' = '1')\n" + + " AS\n" + + " SELECT * FROM mtmv_scan_base;"); + + Assertions.assertEquals(Column.IVM_ROW_ID_COL, createMTMVInfo.getColumns().get(0).getName()); + Assertions.assertFalse(createMTMVInfo.getColumns().get(0).isVisible()); + Assertions.assertEquals("mv_id", createMTMVInfo.getColumns().get(1).getName()); + Assertions.assertEquals("mv_score", createMTMVInfo.getColumns().get(2).getName()); + Assertions.assertTrue(createMTMVInfo.getQuerySql().contains("AS `mv_id`")); + Assertions.assertTrue(createMTMVInfo.getQuerySql().contains("AS `mv_score`")); + } + + @Test + public void testCreateMTMVRewriteQuerySqlWithDefinedColumnsForProjectScanPlan() throws Exception { + createTable("create table test.mtmv_project_scan_base (id int, score int)\n" + + "duplicate key(id)\n" + + "distributed by hash(id) buckets 1\n" + + "properties('replication_num' = '1');"); + + CreateMTMVInfo createMTMVInfo = getPartitionTableInfo("CREATE MATERIALIZED VIEW mtmv_project_scan_alias" + + " (mv_inc_id, mv_score)\n" + + " BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 2\n" + + " PROPERTIES ('replication_num' = '1')\n" + + " AS\n" + + " SELECT id + 1, score FROM mtmv_project_scan_base;"); + + Assertions.assertEquals(Column.IVM_ROW_ID_COL, createMTMVInfo.getColumns().get(0).getName()); + Assertions.assertFalse(createMTMVInfo.getColumns().get(0).isVisible()); + Assertions.assertEquals("mv_inc_id", createMTMVInfo.getColumns().get(1).getName()); + Assertions.assertEquals("mv_score", createMTMVInfo.getColumns().get(2).getName()); + Assertions.assertTrue(createMTMVInfo.getQuerySql().contains("AS `mv_inc_id`")); + Assertions.assertTrue(createMTMVInfo.getQuerySql().contains("AS `mv_score`")); + } + + @Test + public void testCreateMTMVWithoutDefinedColumnsInjectsRowId() throws Exception { + createTable("create table test.mtmv_no_cols_base (id int, score int)\n" + + "duplicate key(id)\n" + + "distributed by hash(id) buckets 1\n" + + "properties('replication_num' = '1');"); + + CreateMTMVInfo createMTMVInfo = getPartitionTableInfo("CREATE MATERIALIZED VIEW mtmv_no_cols" + + " BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 2\n" + + " PROPERTIES ('replication_num' = '1')\n" + + " AS\n" + + " SELECT id, score FROM mtmv_no_cols_base;"); + + Assertions.assertEquals(Column.IVM_ROW_ID_COL, createMTMVInfo.getColumns().get(0).getName()); + Assertions.assertFalse(createMTMVInfo.getColumns().get(0).isVisible()); + Assertions.assertEquals("id", createMTMVInfo.getColumns().get(1).getName()); + Assertions.assertEquals("score", createMTMVInfo.getColumns().get(2).getName()); + } + + @Test + public void testCreateMTMVRewriteQuerySqlContainsAliases() throws Exception { + createTable("create table test.mtmv_alias_base (id int, score int)\n" + + "duplicate key(id)\n" + + "distributed by hash(id) buckets 1\n" + + "properties('replication_num' = '1');"); + + CreateMTMVInfo createMTMVInfo = getPartitionTableInfo("CREATE MATERIALIZED VIEW mtmv_alias" + + " (mv_id, mv_score)\n" + + " BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 2\n" + + " PROPERTIES ('replication_num' = '1')\n" + + " AS\n" + + " SELECT id, score FROM mtmv_alias_base;"); + + String querySql = createMTMVInfo.getQuerySql(); + Assertions.assertTrue(querySql.contains("AS `mv_id`"), "querySql should contain AS `mv_id`: " + querySql); + Assertions.assertTrue(querySql.contains("AS `mv_score`"), "querySql should contain AS `mv_score`: " + querySql); + Assertions.assertFalse(querySql.contains("AS `mv_" + Column.IVM_ROW_ID_COL + "`"), + "querySql should not alias the row-id column: " + querySql); + } + + @Test + public void testCreateIvmMVColumnCountMismatchFails() throws Exception { + createTable("create table test.mtmv_col_mismatch_base (id int, score int)\n" + + "duplicate key(id)\n" + + "distributed by hash(id) buckets 1\n" + + "properties('replication_num' = '1');"); + + // user specifies 2 column names but query only selects 1 column — should fail + org.apache.doris.nereids.exceptions.AnalysisException ex = Assertions.assertThrows( + org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> getPartitionTableInfo("CREATE MATERIALIZED VIEW mtmv_col_mismatch" + + " (mv_id, mv_score)\n" + + " BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 2\n" + + " PROPERTIES ('replication_num' = '1')\n" + + " AS\n" + + " SELECT id FROM mtmv_col_mismatch_base;")); + Assertions.assertTrue(ex.getMessage().contains("simpleColumnDefinitions size is not equal"), + "unexpected message: " + ex.getMessage()); + } + + @Test + public void testVarBinaryModifyColumnRejected() throws Exception { + createTable("create table test.vb_alt (k1 int, v1 int)\n" + + "duplicate key(k1)\n" + + "distributed by hash(k1) buckets 1\n" + + "properties('replication_num' = '1');"); + + org.apache.doris.nereids.trees.plans.logical.LogicalPlan plan = + new org.apache.doris.nereids.parser.NereidsParser() + .parseSingle("alter table test.vb_alt modify column v1 VARBINARY"); + Assertions.assertTrue( + plan instanceof org.apache.doris.nereids.trees.plans.commands.AlterTableCommand); + org.apache.doris.nereids.trees.plans.commands.AlterTableCommand cmd2 = + (org.apache.doris.nereids.trees.plans.commands.AlterTableCommand) plan; + Assertions.assertThrows(Throwable.class, () -> cmd2.run(connectContext, null)); + } + + // ====== Aggregate IVM test cases ====== + + @Test + public void testCreateAggImmvWithMultipleAggFunctions() throws Exception { + createTable("create table test.agg_multi_base (k1 int, v1 int)\n" + + "duplicate key(k1)\n" + + "distributed by hash(k1) buckets 1\n" + + "properties('replication_num' = '1');"); + + CreateMTMVInfo info = getPartitionTableInfo( + "CREATE MATERIALIZED VIEW agg_multi_mv\n" + + " BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 2\n" + + " PROPERTIES ('replication_num' = '1')\n" + + " AS\n" + + " SELECT k1, COUNT(*), SUM(v1) FROM agg_multi_base GROUP BY k1;"); + + List cols = info.getColumns(); + + // row_id at index 0, hidden + Assertions.assertEquals(Column.IVM_ROW_ID_COL, cols.get(0).getName()); + Assertions.assertFalse(cols.get(0).isVisible()); + + // 3 visible user columns: k1, count(*), sum(v1) + List visibleCols = cols.stream() + .filter(Column::isVisible).collect(Collectors.toList()); + Assertions.assertEquals(3, visibleCols.size()); + + // hidden trailing agg state columns: + // __DORIS_IVM_AGG_COUNT_COL__ (group count) + // __DORIS_IVM_AGG_0_COUNT_COL__ (COUNT(*) hidden) + // __DORIS_IVM_AGG_1_SUM_COL__ (SUM hidden) + // __DORIS_IVM_AGG_1_COUNT_COL__ (SUM count hidden) + List hiddenCols = cols.stream() + .filter(c -> !c.isVisible()).collect(Collectors.toList()); + Assertions.assertEquals(5, hiddenCols.size()); // row_id + 4 trailing + List hiddenNames = hiddenCols.stream() + .map(Column::getName).collect(Collectors.toList()); + Assertions.assertTrue(hiddenNames.contains(Column.IVM_AGG_COUNT_COL)); + Assertions.assertTrue(hiddenNames.contains("__DORIS_IVM_AGG_0_COUNT_COL__")); + Assertions.assertTrue(hiddenNames.contains("__DORIS_IVM_AGG_1_SUM_COL__")); + Assertions.assertTrue(hiddenNames.contains("__DORIS_IVM_AGG_1_COUNT_COL__")); + } + + @Test + public void testCreateAggImmvWithHavingThrows() throws Exception { + createTable("create table test.agg_having_base (k1 int, v1 int)\n" + + "duplicate key(k1)\n" + + "distributed by hash(k1) buckets 1\n" + + "properties('replication_num' = '1');"); + + // HAVING produces a Filter above Aggregate, which is rejected by IvmNormalizeMtmv + org.apache.doris.nereids.exceptions.AnalysisException ex = Assertions.assertThrows( + org.apache.doris.nereids.exceptions.AnalysisException.class, + () -> getPartitionTableInfo( + "CREATE MATERIALIZED VIEW agg_having_mv\n" + + " BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 2\n" + + " PROPERTIES ('replication_num' = '1')\n" + + " AS\n" + + " SELECT k1, SUM(v1) FROM agg_having_base GROUP BY k1" + + " HAVING SUM(v1) > 10;")); + Assertions.assertTrue( + ex.getMessage().contains("IVM aggregate must be the top-level operator"), + "unexpected message: " + ex.getMessage()); + } + + @Test + public void testCreateScalarAggImmv() throws Exception { + createTable("create table test.scalar_agg_base (k1 int, v1 int)\n" + + "duplicate key(k1)\n" + + "distributed by hash(k1) buckets 1\n" + + "properties('replication_num' = '1');"); + + CreateMTMVInfo info = getPartitionTableInfo( + "CREATE MATERIALIZED VIEW scalar_agg_mv\n" + + " BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 2\n" + + " PROPERTIES ('replication_num' = '1')\n" + + " AS\n" + + " SELECT COUNT(*), SUM(v1) FROM scalar_agg_base;"); + + List cols = info.getColumns(); + + // row_id at index 0, hidden + Assertions.assertEquals(Column.IVM_ROW_ID_COL, cols.get(0).getName()); + Assertions.assertFalse(cols.get(0).isVisible()); + + // 2 visible: count(*), sum(v1) — no group keys for scalar agg + List visibleCols = cols.stream() + .filter(Column::isVisible).collect(Collectors.toList()); + Assertions.assertEquals(2, visibleCols.size()); + + // hidden: row_id + IVM_AGG_COUNT_COL + AGG_0_COUNT + AGG_1_SUM + AGG_1_COUNT = 5 + List hiddenCols = cols.stream() + .filter(c -> !c.isVisible()).collect(Collectors.toList()); + Assertions.assertEquals(5, hiddenCols.size()); + List hiddenNames = hiddenCols.stream() + .map(Column::getName).collect(Collectors.toList()); + Assertions.assertTrue(hiddenNames.contains(Column.IVM_AGG_COUNT_COL)); + } + + @Test + public void testCreateAggImmvWithAvg() throws Exception { + createTable("create table test.agg_avg_base (k1 int, v1 decimal(10, 2))\n" + + "duplicate key(k1)\n" + + "distributed by hash(k1) buckets 1\n" + + "properties('replication_num' = '1');"); + + CreateMTMVInfo info = getPartitionTableInfo( + "CREATE MATERIALIZED VIEW agg_avg_mv\n" + + " BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 2\n" + + " PROPERTIES ('replication_num' = '1')\n" + + " AS\n" + + " SELECT k1, AVG(v1) FROM agg_avg_base GROUP BY k1;"); + + List cols = info.getColumns(); + + // row_id hidden at index 0 + Assertions.assertEquals(Column.IVM_ROW_ID_COL, cols.get(0).getName()); + Assertions.assertFalse(cols.get(0).isVisible()); + + // 2 visible: k1, avg(v1) + List visibleCols = cols.stream() + .filter(Column::isVisible).collect(Collectors.toList()); + Assertions.assertEquals(2, visibleCols.size()); + + // AVG decomposes to SUM + COUNT hidden states + // hidden: row_id + IVM_AGG_COUNT_COL + AGG_0_SUM + AGG_0_COUNT = 4 + List hiddenNames = cols.stream() + .filter(c -> !c.isVisible()) + .map(Column::getName).collect(Collectors.toList()); + Assertions.assertTrue(hiddenNames.contains(Column.IVM_AGG_COUNT_COL)); + Assertions.assertTrue(hiddenNames.contains("__DORIS_IVM_AGG_0_SUM_COL__")); + Assertions.assertTrue(hiddenNames.contains("__DORIS_IVM_AGG_0_COUNT_COL__")); + } + + @Test + public void testCreateAggImmvWithMinMax() throws Exception { + createTable("create table test.agg_minmax_base (k1 int, v1 int, v2 bigint)\n" + + "duplicate key(k1)\n" + + "distributed by hash(k1) buckets 1\n" + + "properties('replication_num' = '1');"); + + CreateMTMVInfo info = getPartitionTableInfo( + "CREATE MATERIALIZED VIEW agg_minmax_mv\n" + + " BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL\n" + + " DISTRIBUTED BY RANDOM BUCKETS 2\n" + + " PROPERTIES ('replication_num' = '1')\n" + + " AS\n" + + " SELECT k1, MIN(v1), MAX(v2) FROM agg_minmax_base GROUP BY k1;"); + + List cols = info.getColumns(); + + // row_id hidden at index 0 + Assertions.assertEquals(Column.IVM_ROW_ID_COL, cols.get(0).getName()); + Assertions.assertFalse(cols.get(0).isVisible()); + + // 3 visible: k1, min(v1), max(v2) + List visibleCols = cols.stream() + .filter(Column::isVisible).collect(Collectors.toList()); + Assertions.assertEquals(3, visibleCols.size()); + + // MIN(ordinal=0): __DORIS_IVM_AGG_0_MIN_COL__, __DORIS_IVM_AGG_0_COUNT_COL__ + // MAX(ordinal=1): __DORIS_IVM_AGG_1_MAX_COL__, __DORIS_IVM_AGG_1_COUNT_COL__ + // plus group count: __DORIS_IVM_AGG_COUNT_COL__ + List hiddenNames = cols.stream() + .filter(c -> !c.isVisible()) + .map(Column::getName).collect(Collectors.toList()); + Assertions.assertTrue(hiddenNames.contains(Column.IVM_AGG_COUNT_COL)); + Assertions.assertTrue(hiddenNames.contains("__DORIS_IVM_AGG_0_MIN_COL__")); + Assertions.assertTrue(hiddenNames.contains("__DORIS_IVM_AGG_0_COUNT_COL__")); + Assertions.assertTrue(hiddenNames.contains("__DORIS_IVM_AGG_1_MAX_COL__")); + Assertions.assertTrue(hiddenNames.contains("__DORIS_IVM_AGG_1_COUNT_COL__")); + } +} diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateTableCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateTableCommandTest.java index 4cae5e946ab448..0ba5a7aff1a2b2 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateTableCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/CreateTableCommandTest.java @@ -20,7 +20,6 @@ import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.FunctionCallExpr; import org.apache.doris.analysis.PartitionDesc; -import org.apache.doris.analysis.SinglePartitionDesc; import org.apache.doris.analysis.SlotRef; import org.apache.doris.analysis.StringLiteral; import org.apache.doris.catalog.AggregateType; @@ -37,15 +36,8 @@ import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.exceptions.ParseException; import org.apache.doris.nereids.parser.NereidsParser; -import org.apache.doris.nereids.trees.plans.commands.CreateMTMVCommand; import org.apache.doris.nereids.trees.plans.commands.CreateTableCommand; -import org.apache.doris.nereids.trees.plans.commands.info.CreateMTMVInfo; import org.apache.doris.nereids.trees.plans.commands.info.CreateTableInfo; -import org.apache.doris.nereids.trees.plans.commands.info.FixedRangePartition; -import org.apache.doris.nereids.trees.plans.commands.info.InPartition; -import org.apache.doris.nereids.trees.plans.commands.info.LessThanPartition; -import org.apache.doris.nereids.trees.plans.commands.info.PartitionDefinition; -import org.apache.doris.nereids.trees.plans.commands.info.PartitionTableInfo; import org.apache.doris.nereids.trees.plans.logical.LogicalPlan; import org.apache.doris.utframe.TestWithFeService; @@ -905,181 +897,6 @@ public void testPartitionCheckForHive() { } } - @Test - public void testConvertToPartitionTableInfo() throws Exception { - testUnpartitionConvertToPartitionTableInfo(); - testRangePartitionConvertToPartitionTableInfo(); - testInPartitionConvertToPartitionTableInfo(); - testLessThanPartitionConvertToPartitionTableInfo(); - } - - private void testUnpartitionConvertToPartitionTableInfo() throws Exception { - String partitionTable = "CREATE TABLE aa1 (\n" - + " `user_id` LARGEINT NOT NULL COMMENT '\\\"用户id\\\"',\n" - + " `date` DATE NOT NULL COMMENT '\\\"数据灌入日期时间\\\"',\n" - + " `num` SMALLINT NOT NULL COMMENT '\\\"数量\\\"'\n" - + " ) ENGINE=OLAP\n" - + " DUPLICATE KEY(`user_id`, `date`, `num`)\n" - + " COMMENT 'OLAP'\n" - + " PARTITION BY RANGE(`date`)\n" - + " (\n" - + " PARTITION `p201701` VALUES [(\"2017-01-01\"), (\"2017-02-01\")),\n" - + " PARTITION `p201702` VALUES [(\"2017-02-01\"), (\"2017-03-01\")),\n" - + " PARTITION `p201703` VALUES [(\"2017-03-01\"), (\"2017-04-01\"))\n" - + " )\n" - + " DISTRIBUTED BY HASH(`user_id`) BUCKETS 2\n" - + " PROPERTIES ('replication_num' = '1') ;\n"; - createTable(partitionTable); - - String mv = "CREATE MATERIALIZED VIEW mtmv5\n" - + " BUILD DEFERRED REFRESH AUTO ON MANUAL\n" - + " DISTRIBUTED BY RANDOM BUCKETS 2\n" - + " PROPERTIES ('replication_num' = '1')\n" - + " AS\n" - + " SELECT * FROM aa1;"; - - CreateMTMVInfo createMTMVInfo = getPartitionTableInfo(mv); - Assertions.assertEquals(PartitionTableInfo.EMPTY, createMTMVInfo.getPartitionTableInfo()); - } - - private void testRangePartitionConvertToPartitionTableInfo() throws Exception { - String fixedRangePartitionTable = "CREATE TABLE mm1 (\n" - + " `user_id` LARGEINT NOT NULL COMMENT '\\\"用户id\\\"',\n" - + " `date` DATE NOT NULL COMMENT '\\\"数据灌入日期时间\\\"',\n" - + " `num` SMALLINT NOT NULL COMMENT '\\\"数量\\\"'\n" - + " ) ENGINE=OLAP\n" - + " DUPLICATE KEY(`user_id`, `date`, `num`)\n" - + " COMMENT 'OLAP'\n" - + " PARTITION BY RANGE(`date`)\n" - + " (\n" - + " PARTITION `p201701` VALUES [(\"2017-01-01\"), (\"2017-02-01\")),\n" - + " PARTITION `p201702` VALUES [(\"2017-02-01\"), (\"2017-03-01\")),\n" - + " PARTITION `p201703` VALUES [(\"2017-03-01\"), (\"2017-04-01\"))\n" - + " )\n" - + " DISTRIBUTED BY HASH(`user_id`) BUCKETS 2\n" - + " PROPERTIES ('replication_num' = '1') ;\n"; - - String mv = "CREATE MATERIALIZED VIEW mtmv1\n" - + " BUILD DEFERRED REFRESH AUTO ON MANUAL\n" - + " partition by(`date`)\n" - + " DISTRIBUTED BY RANDOM BUCKETS 2\n" - + " PROPERTIES ('replication_num' = '1')\n" - + " AS\n" - + " SELECT * FROM mm1;"; - - check(fixedRangePartitionTable, mv); - } - - private void testLessThanPartitionConvertToPartitionTableInfo() throws Exception { - String lessThanPartitionTable = "CREATE TABLE te2 (\n" - + " `user_id` LARGEINT NOT NULL COMMENT '\\\"用户id\\\"',\n" - + " `date` DATE NOT NULL COMMENT '\\\"数据灌入日期时间\\\"',\n" - + " `num` SMALLINT NOT NULL COMMENT '\\\"数量\\\"'\n" - + " ) ENGINE=OLAP\n" - + " DUPLICATE KEY(`user_id`, `date`, `num`)\n" - + " COMMENT 'OLAP'\n" - + " PARTITION BY RANGE(`date`)\n" - + "(\n" - + " PARTITION `p201701` VALUES LESS THAN (\"2017-02-01\"),\n" - + " PARTITION `p201702` VALUES LESS THAN (\"2017-03-01\"),\n" - + " PARTITION `p201703` VALUES LESS THAN (\"2017-04-01\"),\n" - + " PARTITION `p2018` VALUES [(\"2018-01-01\"), (\"2019-01-01\")),\n" - + " PARTITION `other` VALUES LESS THAN (MAXVALUE)\n" - + ")\n" - + " DISTRIBUTED BY HASH(`user_id`) BUCKETS 2\n" - + " PROPERTIES ('replication_num' = '1') ;"; - - String mv = "CREATE MATERIALIZED VIEW mtmv2\n" - + " BUILD DEFERRED REFRESH AUTO ON MANUAL\n" - + " partition by(`date`)\n" - + " DISTRIBUTED BY RANDOM BUCKETS 2\n" - + " PROPERTIES ('replication_num' = '1')\n" - + " AS\n" - + " SELECT * FROM te2;"; - - check(lessThanPartitionTable, mv); - } - - private void testInPartitionConvertToPartitionTableInfo() throws Exception { - String inPartitionTable = "CREATE TABLE cc1 (\n" - + "`user_id` LARGEINT NOT NULL COMMENT '\\\"用户id\\\"',\n" - + "`date` DATE NOT NULL COMMENT '\\\"数据灌入日期时间\\\"',\n" - + "`num` SMALLINT NOT NULL COMMENT '\\\"数量\\\"'\n" - + ") ENGINE=OLAP\n" - + "DUPLICATE KEY(`user_id`, `date`, `num`)\n" - + "COMMENT 'OLAP'\n" - + "PARTITION BY LIST(`date`,`num`)\n" - + "(\n" - + " PARTITION p201701_1000 VALUES IN (('2017-01-01',1), ('2017-01-01',2)),\n" - + " PARTITION p201702_2000 VALUES IN (('2017-02-01',3), ('2017-02-01',4))\n" - + " )\n" - + " DISTRIBUTED BY HASH(`user_id`) BUCKETS 2\n" - + " PROPERTIES ('replication_num' = '1') ;"; - - String mv = "CREATE MATERIALIZED VIEW mtmv\n" - + "BUILD DEFERRED REFRESH AUTO ON MANUAL\n" - + "partition by(`date`)\n" - + "DISTRIBUTED BY RANDOM BUCKETS 2\n" - + "PROPERTIES ('replication_num' = '1')\n" - + "AS\n" - + "SELECT * FROM cc1;"; - - check(inPartitionTable, mv); - } - - private void check(String sql, String mv) throws Exception { - createTable(sql); - - CreateMTMVInfo createMTMVInfo = getPartitionTableInfo(mv); - PartitionTableInfo partitionTableInfo = createMTMVInfo.getPartitionTableInfo(); - PartitionDesc partitionDesc = createMTMVInfo.getPartitionDesc(); - - List partitionDefs = partitionTableInfo.getPartitionDefs(); - List singlePartitionDescs = partitionDesc.getSinglePartitionDescs(); - - assertPartitionInfo(partitionDefs, singlePartitionDescs); - } - - private void assertPartitionInfo(List partitionDefs, List singlePartitionDescs) { - Assertions.assertEquals(singlePartitionDescs.size(), partitionDefs.size()); - - for (int i = 0; i < singlePartitionDescs.size(); i++) { - PartitionDefinition partitionDefinition = partitionDefs.get(i); - SinglePartitionDesc singlePartitionDesc = singlePartitionDescs.get(i); - - if (partitionDefinition instanceof InPartition) { - InPartition inPartition = (InPartition) partitionDefinition; - - Assertions.assertEquals(singlePartitionDesc.getPartitionName(), partitionDefinition.getPartitionName()); - Assertions.assertEquals(singlePartitionDesc.getPartitionKeyDesc().getPartitionType().name(), "IN"); - Assertions.assertEquals(singlePartitionDesc.getPartitionKeyDesc().getInValues().size(), inPartition.getValues().size()); - } else if (partitionDefinition instanceof FixedRangePartition) { - FixedRangePartition fixedRangePartition = (FixedRangePartition) partitionDefinition; - - Assertions.assertEquals(singlePartitionDesc.getPartitionName(), partitionDefinition.getPartitionName()); - Assertions.assertEquals(singlePartitionDesc.getPartitionKeyDesc().getPartitionType().name(), "FIXED"); - Assertions.assertEquals(fixedRangePartition.getLowerBounds().size(), singlePartitionDesc.getPartitionKeyDesc().getLowerValues().size()); - Assertions.assertEquals(fixedRangePartition.getUpperBounds().size(), singlePartitionDesc.getPartitionKeyDesc().getUpperValues().size()); - } else if (partitionDefinition instanceof LessThanPartition) { - LessThanPartition lessThanPartition = (LessThanPartition) partitionDefinition; - - Assertions.assertEquals(singlePartitionDesc.getPartitionName(), partitionDefinition.getPartitionName()); - Assertions.assertEquals(singlePartitionDesc.getPartitionKeyDesc().getPartitionType().name(), "LESS_THAN"); - Assertions.assertEquals(lessThanPartition.getValues().size(), singlePartitionDesc.getPartitionKeyDesc().getUpperValues().size()); - } - } - } - - private CreateMTMVInfo getPartitionTableInfo(String sql) throws Exception { - NereidsParser nereidsParser = new NereidsParser(); - LogicalPlan logicalPlan = nereidsParser.parseSingle(sql); - Assertions.assertTrue(logicalPlan instanceof CreateMTMVCommand); - CreateMTMVCommand command = (CreateMTMVCommand) logicalPlan; - command.getCreateMTMVInfo().analyze(connectContext); - - return command.getCreateMTMVInfo(); - } - @Test public void testVariantFieldPatternDictCompressionValidation() { String invalidSql = "create table test.tbl_variant_dict_invalid\n" @@ -1112,41 +929,4 @@ public void testVariantFieldPatternDictCompressionValidation() { Assertions.assertDoesNotThrow(() -> createTable(validSql)); } - - @Test - public void testMTMVRejectVarbinary() throws Exception { - String mv = "CREATE MATERIALIZED VIEW mv_vb\n" - + " BUILD DEFERRED REFRESH AUTO ON MANUAL\n" - + " DISTRIBUTED BY RANDOM BUCKETS 2\n" - + " PROPERTIES ('replication_num' = '1')\n" - + " AS SELECT X'AB' as vb;"; - - LogicalPlan plan = new NereidsParser().parseSingle(mv); - Assertions.assertTrue(plan instanceof CreateMTMVCommand); - CreateMTMVCommand cmd = (CreateMTMVCommand) plan; - - org.apache.doris.nereids.exceptions.AnalysisException ex = Assertions.assertThrows( - org.apache.doris.nereids.exceptions.AnalysisException.class, - () -> cmd.getCreateMTMVInfo().analyze(connectContext)); - System.out.println(ex.getMessage()); - Assertions.assertTrue(ex.getMessage().contains("MTMV do not support varbinary type")); - Assertions.assertTrue(ex.getMessage().contains("vb")); - } - - @Test - public void testVarBinaryModifyColumnRejected() throws Exception { - createTable("create table test.vb_alt (k1 int, v1 int)\n" - + "duplicate key(k1)\n" - + "distributed by hash(k1) buckets 1\n" - + "properties('replication_num' = '1');"); - - org.apache.doris.nereids.trees.plans.logical.LogicalPlan plan = - new org.apache.doris.nereids.parser.NereidsParser() - .parseSingle("alter table test.vb_alt modify column v1 VARBINARY"); - Assertions.assertTrue( - plan instanceof org.apache.doris.nereids.trees.plans.commands.AlterTableCommand); - org.apache.doris.nereids.trees.plans.commands.AlterTableCommand cmd2 = - (org.apache.doris.nereids.trees.plans.commands.AlterTableCommand) plan; - Assertions.assertThrows(Throwable.class, () -> cmd2.run(connectContext, null)); - } } diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java index 82c8122a18d72f..a441d977edbf61 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/PlanVisitorTest.java @@ -109,7 +109,7 @@ public void testTimeFunction() { nereidsPlanner -> { // Check nondeterministic collect List nondeterministicFunctionSet = - MaterializedViewUtils.extractNondeterministicFunction( + MaterializedViewUtils.extractMvNondeterministicFunction( nereidsPlanner.getAnalyzedPlan()); Assertions.assertEquals(3, nondeterministicFunctionSet.size()); Assertions.assertTrue(nondeterministicFunctionSet.get(0) instanceof Now); @@ -127,7 +127,7 @@ public void testCurrentDateFunction() { nereidsPlanner -> { // Check nondeterministic collect List nondeterministicFunctionSet = - MaterializedViewUtils.extractNondeterministicFunction( + MaterializedViewUtils.extractMvNondeterministicFunction( nereidsPlanner.getAnalyzedPlan()); Assertions.assertEquals(1, nondeterministicFunctionSet.size()); Assertions.assertTrue(nondeterministicFunctionSet.get(0) instanceof CurrentDate); @@ -143,7 +143,7 @@ public void testContainsNondeterministic() { nereidsPlanner -> { // Check nondeterministic collect List nondeterministicFunctionSet = - MaterializedViewUtils.extractNondeterministicFunction( + MaterializedViewUtils.extractMvNondeterministicFunction( nereidsPlanner.getAnalyzedPlan()); Assertions.assertEquals(1, nondeterministicFunctionSet.size()); Assertions.assertTrue(nondeterministicFunctionSet.get(0) instanceof CurrentDate); @@ -159,7 +159,7 @@ public void testUnixTimestampWithArgsFunction() { nereidsPlanner -> { // Check nondeterministic collect List nondeterministicFunctionSet = - MaterializedViewUtils.extractNondeterministicFunction( + MaterializedViewUtils.extractMvNondeterministicFunction( nereidsPlanner.getAnalyzedPlan()); Assertions.assertEquals(0, nondeterministicFunctionSet.size()); }); @@ -173,7 +173,7 @@ public void testUnixTimestampWithoutArgsFunction() { nereidsPlanner -> { // Check nondeterministic collect List nondeterministicFunctionSet = - MaterializedViewUtils.extractNondeterministicFunction( + MaterializedViewUtils.extractMvNondeterministicFunction( nereidsPlanner.getAnalyzedPlan()); Assertions.assertEquals(1, nondeterministicFunctionSet.size()); Assertions.assertTrue(nondeterministicFunctionSet.get(0) instanceof UnixTimestamp); diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java index 26a1b7cabd3eae..867a6b699a3849 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/nereids/trees/plans/commands/UpdateMvByPartitionCommandTest.java @@ -17,25 +17,65 @@ package org.apache.doris.nereids.trees.plans.commands; +import org.apache.doris.analysis.Expr; import org.apache.doris.analysis.PartitionValue; import org.apache.doris.catalog.Column; +import org.apache.doris.catalog.Database; +import org.apache.doris.catalog.Env; import org.apache.doris.catalog.ListPartitionItem; +import org.apache.doris.catalog.MTMV; import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.PrimitiveType; import org.apache.doris.catalog.RangePartitionItem; import org.apache.doris.common.AnalysisException; +import org.apache.doris.mtmv.MTMVPlanUtil; +import org.apache.doris.nereids.NereidsPlanner; +import org.apache.doris.nereids.StatementContext; +import org.apache.doris.nereids.analyzer.UnboundTableSink; +import org.apache.doris.nereids.glue.translator.PhysicalPlanTranslator; +import org.apache.doris.nereids.glue.translator.PlanTranslatorContext; +import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.expressions.IsNull; +import org.apache.doris.nereids.trees.expressions.Slot; +import org.apache.doris.nereids.trees.plans.logical.LogicalOlapTableSink; +import org.apache.doris.nereids.trees.plans.physical.PhysicalPlan; +import org.apache.doris.nereids.util.PlanChecker; +import org.apache.doris.planner.ExchangeNode; +import org.apache.doris.planner.OlapTableSink; +import org.apache.doris.planner.PlanFragment; +import org.apache.doris.thrift.TPartitionType; +import org.apache.doris.utframe.TestWithFeService; import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; import com.google.common.collect.Range; import com.google.common.collect.Sets; import org.junit.jupiter.api.Assertions; import org.junit.jupiter.api.Test; +import java.util.ArrayList; +import java.util.List; import java.util.Set; -class UpdateMvByPartitionCommandTest { +class UpdateMvByPartitionCommandTest extends TestWithFeService { + @Override + protected void runBeforeAll() throws Exception { + createDatabase("test"); + useDatabase("test"); + createTable("create table test.ivm_base (\n" + + " id int,\n" + + " value int\n" + + ") duplicate key(id)\n" + + "distributed by hash(id) buckets 1\n" + + "properties('replication_num' = '1');"); + createMvByNereids("create materialized view test.ivm_mv\n" + + "build deferred refresh incremental on manual\n" + + "distributed by random buckets 1\n" + + "properties('replication_num' = '1')\n" + + "as select id, value from test.ivm_base;"); + } + @Test void testFirstPartWithoutLowerBound() throws AnalysisException { Column column = new Column("a", PrimitiveType.INT); @@ -84,4 +124,191 @@ void testNull() throws AnalysisException { .next(); Assertions.assertEquals("OR[s IS NULL,s IN (1)]", expr.toSql()); } + + @Test + void testFromUsesInsertedColumnNamesForIncrementalMtmv() throws Exception { + MTMV mtmv = getMtmv("ivm_mv"); + + UpdateMvByPartitionCommand command = newRefreshCommand(mtmv); + + Assertions.assertInstanceOf(UnboundTableSink.class, command.getLogicalQuery()); + UnboundTableSink sink = (UnboundTableSink) command.getLogicalQuery(); + Assertions.assertEquals(mtmv.getInsertedColumnNames(), sink.getColNames()); + } + + @Test + void testAnalyzeRefreshCommandBindsSinkAfterRowIdNormalization() throws Exception { + MTMV mtmv = getMtmv("ivm_mv"); + connectContext.getSessionVariable().setEnableIvmNormalRewrite(true); + + UpdateMvByPartitionCommand command = newRefreshCommand(mtmv); + LogicalOlapTableSink sink = (LogicalOlapTableSink) PlanChecker.from(connectContext, + command.getLogicalQuery()).analyze(command.getLogicalQuery()).getPlan(); + + Assertions.assertEquals(mtmv.getInsertedColumnNames(), getColumnNames(sink.getCols())); + Assertions.assertEquals(mtmv.getInsertedColumnNames(), getNamedExpressionNames(sink.getOutputExprs())); + Assertions.assertEquals(mtmv.getInsertedColumnNames(), getSlotNames(sink.getTargetTableSlots())); + Assertions.assertEquals(Column.IVM_ROW_ID_COL, sink.child().getOutput().get(0).getName()); + } + + @Test + void testPlannerKeepsIvmRowIdAsLargeIntInSinkTupleAndOutputExprs() throws Exception { + MTMV mtmv = getMtmv("ivm_mv"); + UpdateMvByPartitionCommand command = newRefreshCommand(mtmv); + StatementContext statementContext = createStatementCtx("refresh materialized view test.ivm_mv"); + connectContext.getSessionVariable().setEnableIvmNormalRewrite(true); + + TestNereidsPlanner planner = new TestNereidsPlanner(statementContext); + PhysicalPlan physicalPlan = planner.planWithLock(command.getLogicalQuery(), PhysicalProperties.ANY); + PlanTranslatorContext translatorContext = new PlanTranslatorContext(planner.getCascadesContext()); + new PhysicalPlanTranslator(translatorContext).translatePlan(physicalPlan); + List fragments = translatorContext.getPlanFragments(); + + Assertions.assertNotNull(fragments); + Assertions.assertFalse(fragments.isEmpty()); + PlanFragment sinkFragment = findSinkFragment(fragments); + OlapTableSink sink = (OlapTableSink) sinkFragment.getSink(); + PlanFragment tabletSinkExprFragment = findTabletSinkExprFragment(fragments); + List sinkOutputExprs = tabletSinkExprFragment == null + ? sinkFragment.getOutputExprs() + : tabletSinkExprFragment.getOutputExprs(); + + Assertions.assertEquals(Column.IVM_ROW_ID_COL, + sink.getTupleDescriptor().getSlots().get(0).getColumn().getName()); + Assertions.assertEquals(PrimitiveType.LARGEINT, + sink.getTupleDescriptor().getSlots().get(0).getType().getPrimitiveType()); + Assertions.assertFalse(sinkOutputExprs.isEmpty()); + Assertions.assertEquals(PrimitiveType.LARGEINT, + sinkOutputExprs.get(0).getType().getPrimitiveType()); + } + + @Test + void testRunRefreshCommandExecutesIncrementalMtmv() throws Exception { + MTMV mtmv = getMtmv("ivm_mv"); + StatementContext statementContext = createStatementCtx("refresh materialized view test.ivm_mv"); + UpdateMvByPartitionCommand command = newRefreshCommand(mtmv); + org.apache.doris.qe.StmtExecutor executor = MTMVPlanUtil.executeCommand( + mtmv, command, statementContext, "refresh materialized view test.ivm_mv", true); + + Assertions.assertNotNull(executor); + Assertions.assertFalse(executor.getContext().getSessionVariable().isEnableMaterializedViewRewrite()); + Assertions.assertFalse(executor.getContext().getSessionVariable().isEnableDmlMaterializedViewRewrite()); + Assertions.assertTrue(executor.getContext().getSessionVariable().isEnableIvmNormalRewrite()); + Assertions.assertSame(executor.getContext(), statementContext.getConnectContext()); + Assertions.assertSame(statementContext, executor.getContext().getStatementContext()); + } + + @Test + void testExecuteCommandRebindsTaskStatementContextToExecutionContext() throws Exception { + MTMV mtmv = getMtmv("ivm_mv"); + StatementContext statementContext = new StatementContext(); + UpdateMvByPartitionCommand command = UpdateMvByPartitionCommand.from( + mtmv, Sets.newHashSet(), ImmutableMap.of(), statementContext); + org.apache.doris.qe.StmtExecutor executor = MTMVPlanUtil.executeCommand( + mtmv, command, statementContext, "refresh materialized view test.ivm_mv", true); + + Assertions.assertSame(executor.getContext(), statementContext.getConnectContext()); + Assertions.assertSame(statementContext, executor.getContext().getStatementContext()); + } + + @Test + void testRefreshCompleteMtmvWithOneRowRelation() throws Exception { + createTable("create table test.one_row_orders (\n" + + " o_orderkey int not null,\n" + + " o_custkey int not null,\n" + + " o_orderstatus char(1) not null,\n" + + " o_totalprice decimalv3(15,2) not null,\n" + + " o_orderdate date not null,\n" + + " o_orderpriority char(15) not null,\n" + + " o_clerk char(15) not null,\n" + + " o_shippriority int not null,\n" + + " o_comment varchar(79) not null\n" + + ") duplicate key(o_orderkey, o_custkey)\n" + + "partition by range(o_orderdate) (\n" + + " partition day_2 values less than ('2023-12-9'),\n" + + " partition day_3 values less than ('2023-12-11'),\n" + + " partition day_4 values less than ('2023-12-30')\n" + + ")\n" + + "distributed by hash(o_orderkey) buckets 1\n" + + "properties('replication_num' = '1');"); + executeSql("insert into test.one_row_orders values\n" + + "(1, 1, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'),\n" + + "(3, 1, 'o', 12.5, '2023-12-10', 'a', 'b', 1, 'yy');"); + createMvByNereids("create materialized view test.one_row_mv\n" + + "build deferred refresh complete on manual\n" + + "distributed by random buckets 1\n" + + "properties('replication_num' = '1')\n" + + "as select * from\n" + + "(select 1 as l_orderkey, '2023-12-10' as l_shipdate) as c_lineitem\n" + + "left join test.one_row_orders orders on c_lineitem.l_orderkey = orders.o_orderkey\n" + + " and c_lineitem.l_shipdate = o_orderdate;"); + + MTMV mtmv = getMtmv("one_row_mv"); + StatementContext statementContext = createStatementCtx("refresh materialized view test.one_row_mv"); + UpdateMvByPartitionCommand command = UpdateMvByPartitionCommand.from( + mtmv, Sets.newHashSet(), ImmutableMap.of(), statementContext); + MTMVPlanUtil.executeCommand(mtmv, command, statementContext, + "refresh materialized view test.one_row_mv", false); + } + + private UpdateMvByPartitionCommand newRefreshCommand(MTMV mtmv) throws Exception { + StatementContext statementContext = createStatementCtx("refresh materialized view test.ivm_mv"); + return UpdateMvByPartitionCommand.from(mtmv, Sets.newHashSet(), ImmutableMap.of(), statementContext); + } + + private MTMV getMtmv(String mvName) throws Exception { + Database db = Env.getCurrentEnv().getInternalCatalog().getDbOrAnalysisException("test"); + return (MTMV) db.getTableOrAnalysisException(mvName); + } + + private List getColumnNames(List columns) { + List names = new ArrayList<>(columns.size()); + for (Column column : columns) { + names.add(column.getName()); + } + return names; + } + + private List getNamedExpressionNames( + List expressions) { + List names = new ArrayList<>(expressions.size()); + for (org.apache.doris.nereids.trees.expressions.NamedExpression expression : expressions) { + names.add(expression.getName()); + } + return names; + } + + private List getSlotNames(List slots) { + List names = new ArrayList<>(slots.size()); + for (Slot slot : slots) { + names.add(slot.getName()); + } + return names; + } + + private PlanFragment findSinkFragment(List fragments) { + for (PlanFragment planFragment : fragments) { + if (planFragment.getSink() instanceof OlapTableSink) { + return planFragment; + } + } + throw new AssertionError("no sink fragment for olap table sink"); + } + + private PlanFragment findTabletSinkExprFragment(List fragments) { + for (PlanFragment planFragment : fragments) { + if (planFragment.getPlanRoot() instanceof ExchangeNode + && planFragment.getDataPartition().getType() + == TPartitionType.OLAP_TABLE_SINK_HASH_PARTITIONED) { + return planFragment; + } + } + return null; + } + + private static class TestNereidsPlanner extends NereidsPlanner { + TestNereidsPlanner(StatementContext statementContext) { + super(statementContext); + } + } } diff --git a/regression-test/data/mtmv_p0/test_ivm_basic_mtmv.out b/regression-test/data/mtmv_p0/test_ivm_basic_mtmv.out new file mode 100644 index 00000000000000..1c3e034ac6aaf4 --- /dev/null +++ b/regression-test/data/mtmv_p0/test_ivm_basic_mtmv.out @@ -0,0 +1,13 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !after_first_refresh -- +1 10 aaa +2 20 bbb +3 30 ccc + +-- !after_second_refresh -- +1 10 aaa +2 20 bbb +3 30 ccc +4 40 ddd +5 50 eee + diff --git a/regression-test/suites/mtmv_p0/test_ivm_basic_mtmv.groovy b/regression-test/suites/mtmv_p0/test_ivm_basic_mtmv.groovy new file mode 100644 index 00000000000000..16a570f5498fd0 --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_ivm_basic_mtmv.groovy @@ -0,0 +1,109 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_ivm_basic_mtmv") { + def dbName = context.dbName + def waitForMtmvTask = { String mvName -> + String status = "NULL" + long timeoutTimestamp = System.currentTimeMillis() + 5 * 60 * 1000 + while (timeoutTimestamp > System.currentTimeMillis() + && (status == "PENDING" || status == "RUNNING" || status == "NULL")) { + def tasks = sql """select * from tasks('type'='mv') + where MvDatabaseName = '${dbName}' and MvName = '${mvName}'""" + if (tasks.isEmpty()) { + Thread.sleep(1000) + continue + } + def latestTask = tasks.max { row -> row[9]?.toString() ?: "" } + status = latestTask[7].toString() + logger.info("current mv task status: " + status + ", task row: " + latestTask) + if (status == "SUCCESS") { + return + } + Thread.sleep(1000) + } + assertEquals("SUCCESS", status) + } + + sql """drop materialized view if exists mv_ivm_basic;""" + sql """drop table if exists t_ivm_basic_base;""" + + // 1. Create base table (DUP_KEYS) + sql """ + CREATE TABLE t_ivm_basic_base ( + k1 INT, + v1 INT, + v2 VARCHAR(50) + ) + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 2 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + // 2. Insert initial rows + sql """ + INSERT INTO t_ivm_basic_base VALUES + (1, 10, 'aaa'), + (2, 20, 'bbb'), + (3, 30, 'ccc'); + """ + + // 3. Create IVM materialized view (BUILD DEFERRED, REFRESH INCREMENTAL, ON MANUAL) + sql """ + CREATE MATERIALIZED VIEW mv_ivm_basic + BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 2 + PROPERTIES ( + 'replication_num' = '1' + ) + AS SELECT * FROM t_ivm_basic_base; + """ + + // 4. Verify MV metadata — state should be INIT (not yet refreshed) + def mvInfos = sql """select State from mv_infos('database'='${dbName}') where Name = 'mv_ivm_basic'""" + logger.info("mv_infos after create: " + mvInfos.toString()) + assertTrue(mvInfos.toString().contains("INIT")) + + // 5. Verify MV is UNIQUE_KEYS with MOW (enable_unique_key_merge_on_write) + def showCreate = sql """show create materialized view mv_ivm_basic""" + logger.info("show create mv: " + showCreate.toString()) + assertTrue(showCreate.toString().contains("UNIQUE KEY")) + assertTrue(showCreate.toString().contains("enable_unique_key_merge_on_write")) + + // 6. First refresh (full refresh since BUILD DEFERRED) + sql """REFRESH MATERIALIZED VIEW mv_ivm_basic AUTO""" + waitForMtmvTask("mv_ivm_basic") + + // 7. Verify data after first refresh (exclude __IVM_ROW_ID__ column) + order_qt_after_first_refresh """SELECT k1, v1, v2 FROM mv_ivm_basic""" + + // 8. Insert more rows into base table + sql """ + INSERT INTO t_ivm_basic_base VALUES + (4, 40, 'ddd'), + (5, 50, 'eee'); + """ + + // 9. Second refresh + verify updated data + sql """REFRESH MATERIALIZED VIEW mv_ivm_basic AUTO""" + waitForMtmvTask("mv_ivm_basic") + + order_qt_after_second_refresh """SELECT k1, v1, v2 FROM mv_ivm_basic""" + +} diff --git a/regression-test/suites/mtmv_p0/test_ivm_complete_refresh_rowid.groovy b/regression-test/suites/mtmv_p0/test_ivm_complete_refresh_rowid.groovy new file mode 100644 index 00000000000000..0e9216d881a1ff --- /dev/null +++ b/regression-test/suites/mtmv_p0/test_ivm_complete_refresh_rowid.groovy @@ -0,0 +1,91 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_ivm_complete_refresh_rowid", "mtmv") { + String tableName = "test_ivm_complete_refresh_rowid_base" + String mvName = "test_ivm_complete_refresh_rowid_mv" + + sql """SET show_hidden_columns = false;""" + sql """drop materialized view if exists ${mvName};""" + sql """drop table if exists ${tableName};""" + + sql """ + CREATE TABLE ${tableName} ( + k1 INT, + v1 INT, + v2 VARCHAR(50) + ) + DUPLICATE KEY(k1) + DISTRIBUTED BY HASH(k1) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + INSERT INTO ${tableName} VALUES + (1, 10, 'aaa'), + (2, 20, 'bbb'), + (3, 30, 'ccc'); + """ + + sql """ + CREATE MATERIALIZED VIEW ${mvName} + BUILD DEFERRED REFRESH INCREMENTAL ON MANUAL + DISTRIBUTED BY RANDOM BUCKETS 1 + PROPERTIES ( + 'replication_num' = '1' + ) + AS SELECT * FROM ${tableName}; + """ + + def queryMvRowsWithHiddenRowId = { + sql """SET show_hidden_columns = true;""" + def result = sql """ + SELECT __DORIS_IVM_ROW_ID_COL__, k1, v1, v2 + FROM ${mvName} + ORDER BY k1, v1, v2, __DORIS_IVM_ROW_ID_COL__; + """ + sql """SET show_hidden_columns = false;""" + return result + } + + sql """REFRESH MATERIALIZED VIEW ${mvName} COMPLETE;""" + waitingMTMVTaskFinishedByMvName(mvName) + def firstRefreshResult = queryMvRowsWithHiddenRowId() + + sql """REFRESH MATERIALIZED VIEW ${mvName} COMPLETE;""" + waitingMTMVTaskFinishedByMvName(mvName) + def secondRefreshResult = queryMvRowsWithHiddenRowId() + + logger.info("firstRefreshResult=${firstRefreshResult}") + logger.info("secondRefreshResult=${secondRefreshResult}") + + assertEquals(3, firstRefreshResult.size()) + assertEquals(firstRefreshResult.size(), secondRefreshResult.size()) + + for (int i = 0; i < firstRefreshResult.size(); i++) { + def firstRow = firstRefreshResult[i] + def secondRow = secondRefreshResult[i] + assertEquals(firstRow[1], secondRow[1]) + assertEquals(firstRow[2], secondRow[2]) + assertEquals(firstRow[3], secondRow[3]) + assertTrue(firstRow[0].toString() != secondRow[0].toString(), + "row id should change after complete refresh for row: ${firstRow}") + } + +}