From 4011faf3fd6de065baa214dd25350a114a115fa1 Mon Sep 17 00:00:00 2001 From: deardeng Date: Mon, 1 Jun 2026 10:38:23 +0800 Subject: [PATCH] [fix] (cloud) Fix local/remote tablet size semantics in schema views (#60887) In storage-compute separation, data size should be represented consistently as remote size. Previously, show tablets and information_schema.partitions could diverge from information_schema.backend_tablets, which made local/remote semantics confusing for users and operators. This change aligns cloud-mode output mapping for local/remote size columns and adds a regression test to guard the behavior. (cherry picked from commit 17617be150e43fb749cf77cd55adc0b08a2f3afb) --- .../doris/catalog/DataSizeDisplayUtil.java | 78 +++++++++++ .../doris/common/proc/TabletsProcDir.java | 9 +- .../tablefunction/MetadataGenerator.java | 8 +- .../catalog/DataSizeDisplayUtilTest.java | 124 +++++++++++++++++ .../tablets/test_tablet_size_semantics.groovy | 131 ++++++++++++++++++ 5 files changed, 346 insertions(+), 4 deletions(-) create mode 100644 fe/fe-core/src/main/java/org/apache/doris/catalog/DataSizeDisplayUtil.java create mode 100644 fe/fe-core/src/test/java/org/apache/doris/catalog/DataSizeDisplayUtilTest.java create mode 100644 regression-test/suites/cloud_p0/tablets/test_tablet_size_semantics.groovy diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/DataSizeDisplayUtil.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataSizeDisplayUtil.java new file mode 100644 index 00000000000000..03532203409ca5 --- /dev/null +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/DataSizeDisplayUtil.java @@ -0,0 +1,78 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.catalog.MaterializedIndex.IndexExtState; +import org.apache.doris.common.Config; +import org.apache.doris.common.Pair; + +public class DataSizeDisplayUtil { + private DataSizeDisplayUtil() { + } + + public static Pair getDisplayDataSize(Replica replica) { + return getDisplayDataSize(replica.getDataSize(), replica.getRemoteDataSize(), + getLocalIndexAndSegmentSize(replica)); + } + + public static Pair getDisplayDataSize(Partition partition) { + long localDataSize = partition.getDataSize(false); + long remoteDataSize = partition.getRemoteDataSize(); + if (!needCloudSizeMapping(remoteDataSize)) { + return Pair.of(localDataSize, remoteDataSize); + } + return getPartitionDisplayDataSize(partition); + } + + private static Pair getDisplayDataSize(long localDataSize, long remoteDataSize, + long localIndexAndSegmentSize) { + if (!needCloudSizeMapping(remoteDataSize)) { + return Pair.of(localDataSize, remoteDataSize); + } + if (localDataSize > 0) { + remoteDataSize = localDataSize; + localDataSize = 0; + } else if (localIndexAndSegmentSize > 0) { + remoteDataSize = localIndexAndSegmentSize; + } + return Pair.of(localDataSize, remoteDataSize); + } + + private static boolean needCloudSizeMapping(long remoteDataSize) { + return Config.isCloudMode() && remoteDataSize == 0; + } + + private static Pair getPartitionDisplayDataSize(Partition partition) { + long localDataSize = 0L; + long remoteDataSize = 0L; + for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) { + for (Tablet tablet : index.getTablets()) { + for (Replica replica : tablet.getReplicas()) { + Pair displayDataSize = getDisplayDataSize(replica); + localDataSize += displayDataSize.first; + remoteDataSize += displayDataSize.second; + } + } + } + return Pair.of(localDataSize, remoteDataSize); + } + + private static long getLocalIndexAndSegmentSize(Replica replica) { + return replica.getLocalInvertedIndexSize() + replica.getLocalSegmentSize(); + } +} diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java index cfce3db5c2b940..2e47a8f707afd0 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java @@ -18,6 +18,7 @@ package org.apache.doris.common.proc; import org.apache.doris.catalog.CloudTabletStatMgr; +import org.apache.doris.catalog.DataSizeDisplayUtil; import org.apache.doris.catalog.DiskInfo; import org.apache.doris.catalog.Env; import org.apache.doris.catalog.MaterializedIndex; @@ -29,6 +30,7 @@ import org.apache.doris.common.AnalysisException; import org.apache.doris.common.Config; import org.apache.doris.common.FeConstants; +import org.apache.doris.common.Pair; import org.apache.doris.common.util.ListComparator; import org.apache.doris.common.util.NetUtils; import org.apache.doris.common.util.TimeUtils; @@ -192,8 +194,11 @@ public List> fetchComparableResult(long version, long backendId tabletInfo.add(displayLastSuccessVersion); tabletInfo.add(replica.getLastFailedVersion()); tabletInfo.add(TimeUtils.longToTimeString(replica.getLastFailedTimestamp())); - tabletInfo.add(replica.getDataSize()); - tabletInfo.add(replica.getRemoteDataSize()); + Pair displayDataSize = DataSizeDisplayUtil.getDisplayDataSize(replica); + long localDataSize = displayDataSize.first; + long remoteDataSize = displayDataSize.second; + tabletInfo.add(localDataSize); + tabletInfo.add(remoteDataSize); tabletInfo.add(replica.getRowCount()); tabletInfo.add(replica.getState()); diff --git a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java index a3054d81781173..e6f6a7ffb40393 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java @@ -23,6 +23,7 @@ import org.apache.doris.blockrule.SqlBlockRule; import org.apache.doris.catalog.Column; import org.apache.doris.catalog.DataProperty; +import org.apache.doris.catalog.DataSizeDisplayUtil; import org.apache.doris.catalog.Database; import org.apache.doris.catalog.DatabaseIf; import org.apache.doris.catalog.DistributionInfo; @@ -1853,11 +1854,14 @@ private static void partitionsForInternalCatalog(UserIdentity currentUserIdentit trow.addToColumnValue(new TCell().setStringVal("")); // NODEGROUP (not available) trow.addToColumnValue(new TCell().setStringVal("")); // TABLESPACE_NAME (not available) - Pair sizePair = DebugUtil.getByteUint(partition.getDataSize(false)); + Pair displayDataSize = DataSizeDisplayUtil.getDisplayDataSize(partition); + long localDataSize = displayDataSize.first; + long remoteDataSize = displayDataSize.second; + Pair sizePair = DebugUtil.getByteUint(localDataSize); String readableDateSize = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(sizePair.first) + " " + sizePair.second; trow.addToColumnValue(new TCell().setStringVal(readableDateSize)); // LOCAL_DATA_SIZE - sizePair = DebugUtil.getByteUint(partition.getRemoteDataSize()); + sizePair = DebugUtil.getByteUint(remoteDataSize); readableDateSize = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(sizePair.first) + " " + sizePair.second; trow.addToColumnValue(new TCell().setStringVal(readableDateSize)); // REMOTE_DATA_SIZE diff --git a/fe/fe-core/src/test/java/org/apache/doris/catalog/DataSizeDisplayUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/catalog/DataSizeDisplayUtilTest.java new file mode 100644 index 00000000000000..d88397e977a5c0 --- /dev/null +++ b/fe/fe-core/src/test/java/org/apache/doris/catalog/DataSizeDisplayUtilTest.java @@ -0,0 +1,124 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package org.apache.doris.catalog; + +import org.apache.doris.catalog.MaterializedIndex.IndexState; +import org.apache.doris.cloud.catalog.CloudReplica; +import org.apache.doris.cloud.catalog.CloudTablet; +import org.apache.doris.common.Config; +import org.apache.doris.common.Pair; + +import org.junit.After; +import org.junit.Assert; +import org.junit.Before; +import org.junit.Test; + +public class DataSizeDisplayUtilTest { + private String originDeployMode; + private String originCloudUniqueId; + + @Before + public void setUp() { + originDeployMode = Config.deploy_mode; + originCloudUniqueId = Config.cloud_unique_id; + Config.deploy_mode = "cloud"; + Config.cloud_unique_id = ""; + } + + @After + public void tearDown() { + Config.deploy_mode = originDeployMode; + Config.cloud_unique_id = originCloudUniqueId; + } + + @Test + public void testPartitionDisplaySizeFallbackToReplicaIndexAndSegmentSize() { + MaterializedIndex baseIndex = new MaterializedIndex(10L, IndexState.NORMAL); + CloudTablet tablet = new CloudTablet(20L); + CloudReplica replica = new CloudReplica(30L, 1L, Replica.ReplicaState.NORMAL, 2L, 0, + 100L, 200L, 300L, 10L, 0L); + replica.setDataSize(0L); + replica.setLocalInvertedIndexSize(111L); + replica.setLocalSegmentSize(222L); + tablet.addReplica(replica, true); + baseIndex.addTablet(tablet, null, true); + + Partition partition = new Partition(300L, "p1", baseIndex, new RandomDistributionInfo(1)); + + Pair displayDataSize = DataSizeDisplayUtil.getDisplayDataSize(partition); + Assert.assertEquals(0L, displayDataSize.first.longValue()); + Assert.assertEquals(333L, displayDataSize.second.longValue()); + } + + @Test + public void testPartitionDisplaySizeMapsCloudDataSizeToRemoteSize() { + MaterializedIndex baseIndex = new MaterializedIndex(10L, IndexState.NORMAL); + CloudTablet tablet = new CloudTablet(20L); + CloudReplica replica = new CloudReplica(30L, 1L, Replica.ReplicaState.NORMAL, 2L, 0, + 100L, 200L, 300L, 10L, 0L); + replica.setDataSize(123L); + tablet.addReplica(replica, true); + baseIndex.addTablet(tablet, null, true); + + Partition partition = new Partition(300L, "p1", baseIndex, new RandomDistributionInfo(1)); + + Pair displayDataSize = DataSizeDisplayUtil.getDisplayDataSize(partition); + Assert.assertEquals(0L, displayDataSize.first.longValue()); + Assert.assertEquals(123L, displayDataSize.second.longValue()); + } + + @Test + public void testPartitionDisplaySizeAggregatesMixedReplicaDisplaySize() { + MaterializedIndex baseIndex = new MaterializedIndex(10L, IndexState.NORMAL); + + CloudTablet tabletWithDataSize = new CloudTablet(20L); + CloudReplica replicaWithDataSize = new CloudReplica(30L, 1L, Replica.ReplicaState.NORMAL, 2L, 0, + 100L, 200L, 300L, 10L, 0L); + replicaWithDataSize.setDataSize(123L); + tabletWithDataSize.addReplica(replicaWithDataSize, true); + baseIndex.addTablet(tabletWithDataSize, null, true); + + CloudTablet tabletWithFallbackSize = new CloudTablet(21L); + CloudReplica replicaWithFallbackSize = new CloudReplica(31L, 1L, Replica.ReplicaState.NORMAL, 2L, 0, + 100L, 200L, 300L, 10L, 1L); + replicaWithFallbackSize.setDataSize(0L); + replicaWithFallbackSize.setLocalInvertedIndexSize(111L); + replicaWithFallbackSize.setLocalSegmentSize(222L); + tabletWithFallbackSize.addReplica(replicaWithFallbackSize, true); + baseIndex.addTablet(tabletWithFallbackSize, null, true); + + Partition partition = new Partition(300L, "p1", baseIndex, new RandomDistributionInfo(2)); + + Pair displayDataSize = DataSizeDisplayUtil.getDisplayDataSize(partition); + Assert.assertEquals(0L, displayDataSize.first.longValue()); + Assert.assertEquals(456L, displayDataSize.second.longValue()); + } + + @Test + public void testReplicaDisplaySizeFallbackToReplicaIndexAndSegmentSize() { + CloudReplica replica = new CloudReplica(30L, 1L, Replica.ReplicaState.NORMAL, 2L, 0, + 100L, 200L, 300L, 10L, 0L); + replica.setDataSize(0L); + replica.setLocalInvertedIndexSize(111L); + replica.setLocalSegmentSize(222L); + + Pair displayDataSize = DataSizeDisplayUtil.getDisplayDataSize(replica); + Assert.assertEquals(0L, displayDataSize.first.longValue()); + Assert.assertEquals(333L, displayDataSize.second.longValue()); + } +} diff --git a/regression-test/suites/cloud_p0/tablets/test_tablet_size_semantics.groovy b/regression-test/suites/cloud_p0/tablets/test_tablet_size_semantics.groovy new file mode 100644 index 00000000000000..673800f97f631d --- /dev/null +++ b/regression-test/suites/cloud_p0/tablets/test_tablet_size_semantics.groovy @@ -0,0 +1,131 @@ +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("test_tablet_size_semantics") { + if (!isCloudMode()) { + return + } + + def dbName = "test_tablet_size_semantics_db" + def tableName = "test_tablet_size_semantics_tbl" + def parseDataSizeToBytes = { Object dataSize -> + String normalized = dataSize == null ? "" : dataSize.toString().replaceAll("\\s+", "") + if (normalized.isEmpty()) { + return 0L + } + if (!(normalized ==~ /.*[A-Za-z]$/)) { + normalized = normalized + "B" + } + def parsedRows = sql_return_maparray("SELECT parse_data_size('${normalized}') AS bytes") + assertTrue(parsedRows.size() == 1) + return parsedRows[0]["bytes"].toString().toLong() + } + + sql "drop database if exists ${dbName}" + sql "create database ${dbName}" + sql "use ${dbName}" + + sql """ + CREATE TABLE IF NOT EXISTS `${tableName}` ( + `k1` INT NOT NULL, + `v1` STRING NULL + ) + DUPLICATE KEY(`k1`) + DISTRIBUTED BY HASH(`k1`) BUCKETS 1 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + INSERT INTO `${tableName}` VALUES + (1, 'a'), + (2, 'b'), + (3, 'c'), + (4, 'd'), + (5, 'e'); + """ + sql "sync" + + def showTabletRows = sql_return_maparray("show tablets from ${tableName}") + assertTrue(showTabletRows.size() == 1) + def showTabletRow = showTabletRows[0] + long tabletId = showTabletRow["TabletId"].toString().toLong() + long showLocalSize = -1L + long showRemoteSize = -1L + long backendLocalSize = -1L + long backendRemoteSize = -1L + boolean sizeAligned = false + + // SHOW TABLETS size stats may lag behind backend_tablets for tens of seconds. + for (int i = 0; i < 120; i++) { + def latestShowTabletRows = sql_return_maparray("show tablets from ${tableName}") + assertTrue(latestShowTabletRows.size() == 1) + def latestShowTabletRow = latestShowTabletRows[0] + showLocalSize = latestShowTabletRow["LocalDataSize"].toString().toLong() + showRemoteSize = latestShowTabletRow["RemoteDataSize"].toString().toLong() + long backendIdForCheck = latestShowTabletRow["BackendId"].toString().toLong() + + def backendTabletRows = sql_return_maparray(""" + SELECT + tablet_local_size AS be_local_size, + tablet_remote_size AS be_remote_size + FROM information_schema.backend_tablets + WHERE tablet_id = ${tabletId} + AND be_id = ${backendIdForCheck} + ORDER BY update_time DESC + LIMIT 1 + """) + if (backendTabletRows.size() == 1) { + def backendTabletRow = backendTabletRows[0] + backendLocalSize = backendTabletRow["be_local_size"].toString().toLong() + backendRemoteSize = backendTabletRow["be_remote_size"].toString().toLong() + if (showLocalSize == backendLocalSize + && showRemoteSize == backendRemoteSize + && backendRemoteSize > 0) { + sizeAligned = true + break + } + } + + logger.info("wait tablet size aligned, show local/remote: {}/{}, backend local/remote: {}/{}", + showLocalSize, showRemoteSize, backendLocalSize, backendRemoteSize) + sleep(1000) + } + assertTrue(sizeAligned, + "tablet size not aligned within 120s, show local/remote=${showLocalSize}/${showRemoteSize}, " + + "backend local/remote=${backendLocalSize}/${backendRemoteSize}") + logger.info("final tablet size aligned, show local/remote: {}/{}, backend local/remote: {}/{}", + showLocalSize, showRemoteSize, backendLocalSize, backendRemoteSize) + def partitionRows = sql_return_maparray(""" + SELECT + local_data_size AS part_local_size, + remote_data_size AS part_remote_size + FROM information_schema.partitions + WHERE table_schema = '${dbName}' + AND table_name = '${tableName}' + ORDER BY partition_name + LIMIT 1 + """) + assertTrue(partitionRows.size() == 1) + def partitionRow = partitionRows[0] + long partitionLocalSize = parseDataSizeToBytes(partitionRow["part_local_size"]) + long partitionRemoteSize = parseDataSizeToBytes(partitionRow["part_remote_size"]) + + assertEquals(0L, partitionLocalSize) + assertTrue(partitionRemoteSize > 0) +}