Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public boolean test(
long rowCount,
InternalRow minValues,
InternalRow maxValues,
Long[] nullCounts,
int[] nullCounts,
List<Predicate> children) {
for (Predicate child : children) {
if (!child.test(rowCount, minValues, maxValues, nullCounts)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -58,7 +58,7 @@ public boolean test(InternalRow row) {

@Override
public boolean test(
long rowCount, InternalRow minValues, InternalRow maxValues, Long[] nullCounts) {
long rowCount, InternalRow minValues, InternalRow maxValues, int[] nullCounts) {
return function.test(rowCount, minValues, maxValues, nullCounts, children);
}

Expand Down Expand Up @@ -102,7 +102,7 @@ public abstract boolean test(
long rowCount,
InternalRow minValues,
InternalRow maxValues,
Long[] nullCounts,
int[] nullCounts,
List<Predicate> children);

public abstract Optional<Predicate> negate(List<Predicate> children);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ public boolean test(
long rowCount,
Object min,
Object max,
Long nullCount,
long nullCount,
Object patternLiteral) {
return true;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public boolean test(
long rowCount,
Object min,
Object max,
Long nullCount,
long nullCount,
Object patternLiteral) {
return true;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public boolean test(DataType type, Object field, Object literal) {

@Override
public boolean test(
DataType type, long rowCount, Object min, Object max, Long nullCount, Object literal) {
DataType type, long rowCount, Object min, Object max, long nullCount, Object literal) {
return compareLiteral(type, literal, min) >= 0 && compareLiteral(type, literal, max) <= 0;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public boolean test(DataType type, Object field, Object literal) {

@Override
public boolean test(
DataType type, long rowCount, Object min, Object max, Long nullCount, Object literal) {
DataType type, long rowCount, Object min, Object max, long nullCount, Object literal) {
return compareLiteral(type, literal, max) <= 0;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public boolean test(DataType type, Object field, Object literal) {

@Override
public boolean test(
DataType type, long rowCount, Object min, Object max, Long nullCount, Object literal) {
DataType type, long rowCount, Object min, Object max, long nullCount, Object literal) {
return compareLiteral(type, literal, max) < 0;
}

Expand Down
4 changes: 2 additions & 2 deletions fluss-common/src/main/java/org/apache/fluss/predicate/In.java
Original file line number Diff line number Diff line change
Expand Up @@ -56,9 +56,9 @@ public boolean test(
long rowCount,
Object min,
Object max,
Long nullCount,
long nullCount,
List<Object> literals) {
if (nullCount != null && rowCount == nullCount) {
if (nullCount >= 0 && rowCount == nullCount) {
return false;
}
for (Object literal : literals) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ public boolean test(DataType type, Object field) {
}

@Override
public boolean test(DataType type, long rowCount, Object min, Object max, Long nullCount) {
return nullCount == null || nullCount < rowCount;
public boolean test(DataType type, long rowCount, Object min, Object max, long nullCount) {
return nullCount < 0 || nullCount < rowCount;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -41,8 +41,8 @@ public boolean test(DataType type, Object field) {
}

@Override
public boolean test(DataType type, long rowCount, Object min, Object max, Long nullCount) {
return nullCount == null || nullCount > 0;
public boolean test(DataType type, long rowCount, Object min, Object max, long nullCount) {
return nullCount < 0 || nullCount > 0;
}

@Override
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ public abstract class LeafFunction implements Serializable {
* @param rowCount the total number of rows
* @param min the minimum value of the field in the rows
* @param max the maximum value of the field in the rows
* @param nullCount the number of null values in the field, or null if unknown
* @param nullCount the number of null values in the field, or -1 if unknown
* @param literals the literals to test against the field
* @return true if there is any row satisfies the condition, false otherwise
*/
Expand All @@ -58,7 +58,7 @@ public abstract boolean test(
long rowCount,
Object min,
Object max,
Long nullCount,
long nullCount,
List<Object> literals);

public abstract Optional<LeafFunction> negate();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

package org.apache.fluss.predicate;

import org.apache.fluss.record.LogRecordBatchStatistics;
import org.apache.fluss.row.InternalRow;
import org.apache.fluss.types.DataType;
import org.apache.fluss.types.DecimalType;
Expand Down Expand Up @@ -87,11 +88,14 @@ public boolean test(InternalRow row) {

@Override
public boolean test(
long rowCount, InternalRow minValues, InternalRow maxValues, Long[] nullCounts) {
long rowCount, InternalRow minValues, InternalRow maxValues, int[] nullCounts) {
Object min = get(minValues, fieldIndex, type);
Object max = get(maxValues, fieldIndex, type);
Long nullCount = nullCounts != null ? nullCounts[fieldIndex] : null;
if (nullCount == null || rowCount != nullCount) {
long nullCount =
nullCounts != null
? nullCounts[fieldIndex]
: LogRecordBatchStatistics.NULL_COUNT_UNAVAILABLE;
if (nullCount < 0 || rowCount != nullCount) {
// not all null
// min or max is null
// unknown stats
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public abstract class LeafUnaryFunction extends LeafFunction {
public abstract boolean test(DataType type, Object value);

public abstract boolean test(
DataType type, long rowCount, Object min, Object max, Long nullCount);
DataType type, long rowCount, Object min, Object max, long nullCount);

@Override
public boolean test(DataType type, Object value, List<Object> literals) {
Expand All @@ -46,7 +46,7 @@ public boolean test(
long rowCount,
Object min,
Object max,
Long nullCount,
long nullCount,
List<Object> literals) {
return test(type, rowCount, min, max, nullCount);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public boolean test(DataType type, Object field, Object literal) {

@Override
public boolean test(
DataType type, long rowCount, Object min, Object max, Long nullCount, Object literal) {
DataType type, long rowCount, Object min, Object max, long nullCount, Object literal) {
return compareLiteral(type, literal, min) >= 0;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public boolean test(DataType type, Object field, Object literal) {

@Override
public boolean test(
DataType type, long rowCount, Object min, Object max, Long nullCount, Object literal) {
DataType type, long rowCount, Object min, Object max, long nullCount, Object literal) {
return compareLiteral(type, literal, min) > 0;
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -44,7 +44,7 @@ public boolean test(DataType type, Object field, Object literal) {

@Override
public boolean test(
DataType type, long rowCount, Object min, Object max, Long nullCount, Object literal) {
DataType type, long rowCount, Object min, Object max, long nullCount, Object literal) {
// ony when max == min == literal, the result is false,
// otherwise, the row set MAY contain the literal.
return compareLiteral(type, literal, min) != 0 || compareLiteral(type, literal, max) != 0;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,9 +55,9 @@ public boolean test(
long rowCount,
Object min,
Object max,
Long nullCount,
long nullCount,
List<Object> literals) {
if (nullCount != null && rowCount == nullCount) {
if (nullCount >= 0 && rowCount == nullCount) {
return false;
}
for (Object literal : literals) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public abstract class NullFalseLeafBinaryFunction extends LeafFunction {
public abstract boolean test(DataType type, Object field, Object literal);

public abstract boolean test(
DataType type, long rowCount, Object min, Object max, Long nullCount, Object literal);
DataType type, long rowCount, Object min, Object max, long nullCount, Object literal);

@Override
public boolean test(DataType type, Object field, List<Object> literals) {
Expand All @@ -49,9 +49,9 @@ public boolean test(
long rowCount,
Object min,
Object max,
Long nullCount,
long nullCount,
List<Object> literals) {
if (nullCount != null) {
if (nullCount >= 0) {
if (rowCount == nullCount || literals.get(0) == null) {
return false;
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public boolean test(
long rowCount,
InternalRow minValues,
InternalRow maxValues,
Long[] nullCounts,
int[] nullCounts,
List<Predicate> children) {
for (Predicate child : children) {
if (child.test(rowCount, minValues, maxValues, nullCounts)) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@ public interface Predicate extends Serializable {
* @return return true is likely to hit (there may also be false positives), return false is
* absolutely not possible to hit.
*/
boolean test(long rowCount, InternalRow minValues, InternalRow maxValues, Long[] nullCounts);
boolean test(long rowCount, InternalRow minValues, InternalRow maxValues, int[] nullCounts);

/**
* @return the negation predicate of this predicate if possible.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ public boolean test(
long rowCount,
Object min,
Object max,
Long nullCount,
long nullCount,
Object patternLiteral) {
String minStr = min.toString();
String maxStr = max.toString();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ public class DefaultLogRecordBatchStatistics implements LogRecordBatchStatistics

private final int[] statsIndexMapping;

private final Long[] statsNullCounts;
private final int[] statsNullCounts;

// Offsets for min/max values in the byte array
private final int minValuesOffset;
Expand All @@ -80,7 +80,7 @@ public class DefaultLogRecordBatchStatistics implements LogRecordBatchStatistics

private InternalRow cachedMinRow;
private InternalRow cachedMaxRow;
private Long[] cachedNullCounts;
private int[] cachedNullCounts;

private final int[] reversedStatsIndexMapping;

Expand All @@ -91,7 +91,7 @@ public DefaultLogRecordBatchStatistics(
int size,
RowType rowType,
int schemaId,
Long[] nullCounts,
int[] nullCounts,
int minValuesOffset,
int maxValuesOffset,
int minValuesSize,
Expand Down Expand Up @@ -146,16 +146,15 @@ public InternalRow getMaxValues() {
}

@Override
public Long[] getNullCounts() {
public int[] getNullCounts() {
if (cachedNullCounts != null) {
return cachedNullCounts;
}
cachedNullCounts = new Long[rowType.getFieldCount()];
cachedNullCounts = new int[rowType.getFieldCount()];
Arrays.fill(cachedNullCounts, LogRecordBatchStatistics.NULL_COUNT_UNAVAILABLE);
for (int i = 0; i < rowType.getFieldCount(); i++) {
if (this.reversedStatsIndexMapping[i] >= 0) {
cachedNullCounts[i] = statsNullCounts[reversedStatsIndexMapping[i]];
} else {
cachedNullCounts[i] = -1L;
}
}
return cachedNullCounts;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,9 @@
/** Statistics information of {@link LogRecordBatch LogRecordBatch}. */
public interface LogRecordBatchStatistics {

/** Sentinel value indicating that null count statistics are not available for a field. */
int NULL_COUNT_UNAVAILABLE = -1;

/**
* Get the minimum values as an InternalRow.
*
Expand All @@ -37,11 +40,12 @@ public interface LogRecordBatchStatistics {
InternalRow getMaxValues();

/**
* Get the null counts for each field.
* Get the null counts for each field. Uses {@code -1} to indicate that null count statistics
* are not available for a particular field.
*
* @return Array of null counts
* @return Array of null counts, where -1 means not available
*/
Long[] getNullCounts();
int[] getNullCounts();

/**
* Whether the statistics information for a specific field is available.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ public class LogRecordBatchStatisticsCollector {
// Statistics arrays (only for columns that need statistics)
private final Object[] minValues;
private final Object[] maxValues;
private final Long[] nullCounts;
private final int[] nullCounts;

private final LogRecordBatchStatisticsWriter statisticsWriter;

Expand All @@ -58,11 +58,11 @@ public LogRecordBatchStatisticsCollector(RowType rowType, int[] statsIndexMappin
// Initialize statistics arrays
this.minValues = new Object[statsIndexMapping.length];
this.maxValues = new Object[statsIndexMapping.length];
this.nullCounts = new Long[statsIndexMapping.length];
this.nullCounts = new int[statsIndexMapping.length];

this.statisticsWriter = new LogRecordBatchStatisticsWriter(rowType, statsIndexMapping);

Arrays.fill(nullCounts, 0L);
Arrays.fill(nullCounts, 0);
}

/**
Expand Down Expand Up @@ -116,7 +116,7 @@ public int estimatedSizeInBytes() {

/** Reset the collector to collect new statistics. */
public void reset() {
Arrays.fill(nullCounts, 0L);
Arrays.fill(nullCounts, 0);
Arrays.fill(minValues, null);
Arrays.fill(maxValues, null);
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -113,9 +113,9 @@ public static DefaultLogRecordBatchStatistics parseStatistics(

// Read null counts at fixed offset
int nullCountsStart = position + nullCountsOffset(statisticsColumnCount);
Long[] nullCounts = new Long[statisticsColumnCount];
int[] nullCounts = new int[statisticsColumnCount];
for (int i = 0; i < statisticsColumnCount; i++) {
nullCounts[i] = (long) segment.getInt(nullCountsStart + 4 * i);
nullCounts[i] = segment.getInt(nullCountsStart + 4 * i);
}

// Read min values size at fixed offset
Expand Down
Loading