[fix] (cloud) Fix local/remote tablet size semantics in schema views(#60887)

fe/fe-core/src/main/java/org/apache/doris/catalog/DataSizeDisplayUtil.java

@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog;
+
+import org.apache.doris.catalog.MaterializedIndex.IndexExtState;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.Pair;
+
+public class DataSizeDisplayUtil {
+    private DataSizeDisplayUtil() {
+    }
+
+    public static Pair<Long, Long> getDisplayDataSize(Replica replica) {
+        return getDisplayDataSize(replica.getDataSize(), replica.getRemoteDataSize(),
+                getLocalIndexAndSegmentSize(replica));
+    }
+
+    public static Pair<Long, Long> getDisplayDataSize(Partition partition) {
+        long localDataSize = partition.getDataSize(false);
+        long remoteDataSize = partition.getRemoteDataSize();
+        if (!needCloudSizeMapping(remoteDataSize)) {
+            return Pair.of(localDataSize, remoteDataSize);
+        }
+        return getPartitionDisplayDataSize(partition);
+    }
+
+    private static Pair<Long, Long> getDisplayDataSize(long localDataSize, long remoteDataSize,
+            long localIndexAndSegmentSize) {
+        if (!needCloudSizeMapping(remoteDataSize)) {
+            return Pair.of(localDataSize, remoteDataSize);
+        }
+        if (localDataSize > 0) {
+            remoteDataSize = localDataSize;
+            localDataSize = 0;
+        } else if (localIndexAndSegmentSize > 0) {
+            remoteDataSize = localIndexAndSegmentSize;
+        }
+        return Pair.of(localDataSize, remoteDataSize);
+    }
+
+    private static boolean needCloudSizeMapping(long remoteDataSize) {
+        return Config.isCloudMode() && remoteDataSize == 0;
+    }
+
+    private static Pair<Long, Long> getPartitionDisplayDataSize(Partition partition) {
+        long localDataSize = 0L;
+        long remoteDataSize = 0L;
+        for (MaterializedIndex index : partition.getMaterializedIndices(IndexExtState.VISIBLE)) {
+            for (Tablet tablet : index.getTablets()) {
+                for (Replica replica : tablet.getReplicas()) {
+                    Pair<Long, Long> displayDataSize = getDisplayDataSize(replica);
+                    localDataSize += displayDataSize.first;
+                    remoteDataSize += displayDataSize.second;
+                }
+            }
+        }
+        return Pair.of(localDataSize, remoteDataSize);
+    }
+
+    private static long getLocalIndexAndSegmentSize(Replica replica) {
+        return replica.getLocalInvertedIndexSize() + replica.getLocalSegmentSize();
+    }
+}

fe/fe-core/src/main/java/org/apache/doris/common/proc/TabletsProcDir.java

@@ -18,6 +18,7 @@
 package org.apache.doris.common.proc;
 
 import org.apache.doris.catalog.CloudTabletStatMgr;
+import org.apache.doris.catalog.DataSizeDisplayUtil;
 import org.apache.doris.catalog.DiskInfo;
 import org.apache.doris.catalog.Env;
 import org.apache.doris.catalog.MaterializedIndex;
@@ -29,6 +30,7 @@
 import org.apache.doris.common.AnalysisException;
 import org.apache.doris.common.Config;
 import org.apache.doris.common.FeConstants;
+import org.apache.doris.common.Pair;
 import org.apache.doris.common.util.ListComparator;
 import org.apache.doris.common.util.NetUtils;
 import org.apache.doris.common.util.TimeUtils;
@@ -192,8 +194,11 @@ public List<List<Comparable>> fetchComparableResult(long version, long backendId
                         tabletInfo.add(displayLastSuccessVersion);
                         tabletInfo.add(replica.getLastFailedVersion());
                         tabletInfo.add(TimeUtils.longToTimeString(replica.getLastFailedTimestamp()));
-                        tabletInfo.add(replica.getDataSize());
-                        tabletInfo.add(replica.getRemoteDataSize());
+                        Pair<Long, Long> displayDataSize = DataSizeDisplayUtil.getDisplayDataSize(replica);
+                        long localDataSize = displayDataSize.first;
+                        long remoteDataSize = displayDataSize.second;
+                        tabletInfo.add(localDataSize);
+                        tabletInfo.add(remoteDataSize);
                         tabletInfo.add(replica.getRowCount());
                         tabletInfo.add(replica.getState());
 

fe/fe-core/src/main/java/org/apache/doris/tablefunction/MetadataGenerator.java

@@ -23,6 +23,7 @@
 import org.apache.doris.blockrule.SqlBlockRule;
 import org.apache.doris.catalog.Column;
 import org.apache.doris.catalog.DataProperty;
+import org.apache.doris.catalog.DataSizeDisplayUtil;
 import org.apache.doris.catalog.Database;
 import org.apache.doris.catalog.DatabaseIf;
 import org.apache.doris.catalog.DistributionInfo;
@@ -1853,11 +1854,14 @@ private static void partitionsForInternalCatalog(UserIdentity currentUserIdentit
                     trow.addToColumnValue(new TCell().setStringVal("")); // NODEGROUP (not available)
                     trow.addToColumnValue(new TCell().setStringVal("")); // TABLESPACE_NAME (not available)
 
-                    Pair<Double, String> sizePair = DebugUtil.getByteUint(partition.getDataSize(false));
+                    Pair<Long, Long> displayDataSize = DataSizeDisplayUtil.getDisplayDataSize(partition);
+                    long localDataSize = displayDataSize.first;
+                    long remoteDataSize = displayDataSize.second;
+                    Pair<Double, String> sizePair = DebugUtil.getByteUint(localDataSize);
                     String readableDateSize = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(sizePair.first) + " "
                             + sizePair.second;
                     trow.addToColumnValue(new TCell().setStringVal(readableDateSize));  // LOCAL_DATA_SIZE
-                    sizePair = DebugUtil.getByteUint(partition.getRemoteDataSize());
+                    sizePair = DebugUtil.getByteUint(remoteDataSize);
                     readableDateSize = DebugUtil.DECIMAL_FORMAT_SCALE_3.format(sizePair.first) + " "
                             + sizePair.second;
                     trow.addToColumnValue(new TCell().setStringVal(readableDateSize)); // REMOTE_DATA_SIZE

fe/fe-core/src/test/java/org/apache/doris/catalog/DataSizeDisplayUtilTest.java

@@ -0,0 +1,124 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.catalog;
+
+import org.apache.doris.catalog.MaterializedIndex.IndexState;
+import org.apache.doris.cloud.catalog.CloudReplica;
+import org.apache.doris.cloud.catalog.CloudTablet;
+import org.apache.doris.common.Config;
+import org.apache.doris.common.Pair;
+
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+public class DataSizeDisplayUtilTest {
+    private String originDeployMode;
+    private String originCloudUniqueId;
+
+    @Before
+    public void setUp() {
+        originDeployMode = Config.deploy_mode;
+        originCloudUniqueId = Config.cloud_unique_id;
+        Config.deploy_mode = "cloud";
+        Config.cloud_unique_id = "";
+    }
+
+    @After
+    public void tearDown() {
+        Config.deploy_mode = originDeployMode;
+        Config.cloud_unique_id = originCloudUniqueId;
+    }
+
+    @Test
+    public void testPartitionDisplaySizeFallbackToReplicaIndexAndSegmentSize() {
+        MaterializedIndex baseIndex = new MaterializedIndex(10L, IndexState.NORMAL);
+        CloudTablet tablet = new CloudTablet(20L);
+        CloudReplica replica = new CloudReplica(30L, 1L, Replica.ReplicaState.NORMAL, 2L, 0,
+                100L, 200L, 300L, 10L, 0L);
+        replica.setDataSize(0L);
+        replica.setLocalInvertedIndexSize(111L);
+        replica.setLocalSegmentSize(222L);
+        tablet.addReplica(replica, true);
+        baseIndex.addTablet(tablet, null, true);
+
+        Partition partition = new Partition(300L, "p1", baseIndex, new RandomDistributionInfo(1));
+
+        Pair<Long, Long> displayDataSize = DataSizeDisplayUtil.getDisplayDataSize(partition);
+        Assert.assertEquals(0L, displayDataSize.first.longValue());
+        Assert.assertEquals(333L, displayDataSize.second.longValue());
+    }
+
+    @Test
+    public void testPartitionDisplaySizeMapsCloudDataSizeToRemoteSize() {
+        MaterializedIndex baseIndex = new MaterializedIndex(10L, IndexState.NORMAL);
+        CloudTablet tablet = new CloudTablet(20L);
+        CloudReplica replica = new CloudReplica(30L, 1L, Replica.ReplicaState.NORMAL, 2L, 0,
+                100L, 200L, 300L, 10L, 0L);
+        replica.setDataSize(123L);
+        tablet.addReplica(replica, true);
+        baseIndex.addTablet(tablet, null, true);
+
+        Partition partition = new Partition(300L, "p1", baseIndex, new RandomDistributionInfo(1));
+
+        Pair<Long, Long> displayDataSize = DataSizeDisplayUtil.getDisplayDataSize(partition);
+        Assert.assertEquals(0L, displayDataSize.first.longValue());
+        Assert.assertEquals(123L, displayDataSize.second.longValue());
+    }
+
+    @Test
+    public void testPartitionDisplaySizeAggregatesMixedReplicaDisplaySize() {
+        MaterializedIndex baseIndex = new MaterializedIndex(10L, IndexState.NORMAL);
+
+        CloudTablet tabletWithDataSize = new CloudTablet(20L);
+        CloudReplica replicaWithDataSize = new CloudReplica(30L, 1L, Replica.ReplicaState.NORMAL, 2L, 0,
+                100L, 200L, 300L, 10L, 0L);
+        replicaWithDataSize.setDataSize(123L);
+        tabletWithDataSize.addReplica(replicaWithDataSize, true);
+        baseIndex.addTablet(tabletWithDataSize, null, true);
+
+        CloudTablet tabletWithFallbackSize = new CloudTablet(21L);
+        CloudReplica replicaWithFallbackSize = new CloudReplica(31L, 1L, Replica.ReplicaState.NORMAL, 2L, 0,
+                100L, 200L, 300L, 10L, 1L);
+        replicaWithFallbackSize.setDataSize(0L);
+        replicaWithFallbackSize.setLocalInvertedIndexSize(111L);
+        replicaWithFallbackSize.setLocalSegmentSize(222L);
+        tabletWithFallbackSize.addReplica(replicaWithFallbackSize, true);
+        baseIndex.addTablet(tabletWithFallbackSize, null, true);
+
+        Partition partition = new Partition(300L, "p1", baseIndex, new RandomDistributionInfo(2));
+
+        Pair<Long, Long> displayDataSize = DataSizeDisplayUtil.getDisplayDataSize(partition);
+        Assert.assertEquals(0L, displayDataSize.first.longValue());
+        Assert.assertEquals(456L, displayDataSize.second.longValue());
+    }
+
+    @Test
+    public void testReplicaDisplaySizeFallbackToReplicaIndexAndSegmentSize() {
+        CloudReplica replica = new CloudReplica(30L, 1L, Replica.ReplicaState.NORMAL, 2L, 0,
+                100L, 200L, 300L, 10L, 0L);
+        replica.setDataSize(0L);
+        replica.setLocalInvertedIndexSize(111L);
+        replica.setLocalSegmentSize(222L);
+
+        Pair<Long, Long> displayDataSize = DataSizeDisplayUtil.getDisplayDataSize(replica);
+        Assert.assertEquals(0L, displayDataSize.first.longValue());
+        Assert.assertEquals(333L, displayDataSize.second.longValue());
+    }
+}

regression-test/suites/cloud_p0/tablets/test_tablet_size_semantics.groovy

@@ -0,0 +1,131 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+suite("test_tablet_size_semantics") {
+    if (!isCloudMode()) {
+        return
+    }
+
+    def dbName = "test_tablet_size_semantics_db"
+    def tableName = "test_tablet_size_semantics_tbl"
+    def parseDataSizeToBytes = { Object dataSize ->
+        String normalized = dataSize == null ? "" : dataSize.toString().replaceAll("\\s+", "")
+        if (normalized.isEmpty()) {
+            return 0L
+        }
+        if (!(normalized ==~ /.*[A-Za-z]$/)) {
+            normalized = normalized + "B"
+        }
+        def parsedRows = sql_return_maparray("SELECT parse_data_size('${normalized}') AS bytes")
+        assertTrue(parsedRows.size() == 1)
+        return parsedRows[0]["bytes"].toString().toLong()
+    }
+
+    sql "drop database if exists ${dbName}"
+    sql "create database ${dbName}"
+    sql "use ${dbName}"
+
+    sql """
+        CREATE TABLE IF NOT EXISTS `${tableName}` (
+            `k1` INT NOT NULL,
+            `v1` STRING NULL
+        )
+        DUPLICATE KEY(`k1`)
+        DISTRIBUTED BY HASH(`k1`) BUCKETS 1
+        PROPERTIES (
+            "replication_num" = "1"
+        );
+    """
+
+    sql """
+        INSERT INTO `${tableName}` VALUES
+        (1, 'a'),
+        (2, 'b'),
+        (3, 'c'),
+        (4, 'd'),
+        (5, 'e');
+    """
+    sql "sync"
+
+    def showTabletRows = sql_return_maparray("show tablets from ${tableName}")
+    assertTrue(showTabletRows.size() == 1)
+    def showTabletRow = showTabletRows[0]
+    long tabletId = showTabletRow["TabletId"].toString().toLong()
+    long showLocalSize = -1L
+    long showRemoteSize = -1L
+    long backendLocalSize = -1L
+    long backendRemoteSize = -1L
+    boolean sizeAligned = false
+
+    // SHOW TABLETS size stats may lag behind backend_tablets for tens of seconds.
+    for (int i = 0; i < 120; i++) {
+        def latestShowTabletRows = sql_return_maparray("show tablets from ${tableName}")
+        assertTrue(latestShowTabletRows.size() == 1)
+        def latestShowTabletRow = latestShowTabletRows[0]
+        showLocalSize = latestShowTabletRow["LocalDataSize"].toString().toLong()
+        showRemoteSize = latestShowTabletRow["RemoteDataSize"].toString().toLong()
+        long backendIdForCheck = latestShowTabletRow["BackendId"].toString().toLong()
+
+        def backendTabletRows = sql_return_maparray("""
+            SELECT
+                tablet_local_size AS be_local_size,
+                tablet_remote_size AS be_remote_size
+            FROM information_schema.backend_tablets
+            WHERE tablet_id = ${tabletId}
+              AND be_id = ${backendIdForCheck}
+            ORDER BY update_time DESC
+            LIMIT 1
+        """)
+        if (backendTabletRows.size() == 1) {
+            def backendTabletRow = backendTabletRows[0]
+            backendLocalSize = backendTabletRow["be_local_size"].toString().toLong()
+            backendRemoteSize = backendTabletRow["be_remote_size"].toString().toLong()
+            if (showLocalSize == backendLocalSize
+                    && showRemoteSize == backendRemoteSize
+                    && backendRemoteSize > 0) {
+                sizeAligned = true
+                break
+            }
+        }
+
+        logger.info("wait tablet size aligned, show local/remote: {}/{}, backend local/remote: {}/{}",
+                showLocalSize, showRemoteSize, backendLocalSize, backendRemoteSize)
+        sleep(1000)
+    }
+    assertTrue(sizeAligned,
+            "tablet size not aligned within 120s, show local/remote=${showLocalSize}/${showRemoteSize}, " +
+                    "backend local/remote=${backendLocalSize}/${backendRemoteSize}")
+    logger.info("final tablet size aligned, show local/remote: {}/{}, backend local/remote: {}/{}",
+                showLocalSize, showRemoteSize, backendLocalSize, backendRemoteSize)
+    def partitionRows = sql_return_maparray("""
+        SELECT
+            local_data_size AS part_local_size,
+            remote_data_size AS part_remote_size
+        FROM information_schema.partitions
+        WHERE table_schema = '${dbName}'
+          AND table_name = '${tableName}'
+        ORDER BY partition_name
+        LIMIT 1
+    """)
+    assertTrue(partitionRows.size() == 1)
+    def partitionRow = partitionRows[0]
+    long partitionLocalSize = parseDataSizeToBytes(partitionRow["part_local_size"])
+    long partitionRemoteSize = parseDataSizeToBytes(partitionRow["part_remote_size"])
+
+    assertEquals(0L, partitionLocalSize)
+    assertTrue(partitionRemoteSize > 0)
+}