Fix memory accounting of DictionaryRowGroup
@@ -36,6 +36,7 @@ import com.facebook.presto.orc.stream.StreamDataOutput;
import com.google.common.collect.ImmutableList;
import com.google.common.collect.ImmutableMap;
import io.airlift.slice.Slice;
+import org.openjdk.jol.info.ClassLayout;
import java.io.IOException;
import java.util.ArrayList;
@@ -55,6 +56,7 @@ import static com.facebook.presto.orc.metadata.Stream.StreamKind.DATA;
import static com.google.common.base.Preconditions.checkArgument;
import static com.google.common.base.Preconditions.checkState;
import static io.airlift.slice.SizeOf.sizeOf;
+import static io.airlift.slice.SizeOf.sizeOfObjectArray;
import static java.lang.Math.toIntExact;
import static java.util.Objects.requireNonNull;
import static java.util.stream.Collectors.toList;
@@ -73,7 +75,7 @@ public abstract class DictionaryColumnWriter
private final PresentOutputStream presentStream;
private final CompressedMetadataWriter compressedMetadataWriter;
private final List<DictionaryRowGroup> rowGroups = new ArrayList<>();
- private long columnStatisticsRetainedSizeInBytes;
+ private long rowGroupRetainedSizeInBytes;
private int[] rowGroupIndexes;
private int rowGroupValueCount;
@@ -284,7 +286,7 @@ public abstract class DictionaryColumnWriter
ColumnStatistics statistics = createColumnStatistics();
DictionaryRowGroup rowGroup = new DictionaryRowGroup(rowGroupIndexes, rowGroupValueCount, statistics);
rowGroups.add(rowGroup);
- columnStatisticsRetainedSizeInBytes += rowGroup.getColumnStatistics().getRetainedSizeInBytes();
+ rowGroupRetainedSizeInBytes += rowGroup.getRetainedSizeInBytes();
rowGroupValueCount = 0;
return ImmutableMap.of(column, statistics);
}
@@ -414,7 +416,8 @@ public abstract class DictionaryColumnWriter
dataStream.getRetainedBytes() +
presentStream.getRetainedBytes() +
getRetainedDictionaryBytes() +
- columnStatisticsRetainedSizeInBytes;
+ rowGroupRetainedSizeInBytes +
+ sizeOfObjectArray(rowGroups.size());
}
@Override
@@ -425,7 +428,7 @@ public abstract class DictionaryColumnWriter
dataStream.reset();
presentStream.reset();
rowGroups.clear();
- columnStatisticsRetainedSizeInBytes = 0;
+ rowGroupRetainedSizeInBytes = 0;
rowGroupValueCount = 0;
resetDictionary();
@@ -441,6 +444,8 @@ public abstract class DictionaryColumnWriter
private static class DictionaryRowGroup
{
+ private static final int INSTANCE_SIZE = ClassLayout.parseClass(DictionaryRowGroup.class).instanceSize();
+
private final int[] dictionaryIndexes;
private final ColumnStatistics columnStatistics;
@@ -468,6 +473,13 @@ public abstract class DictionaryColumnWriter
{
return columnStatistics;
}
+
+ public long getRetainedSizeInBytes()
+ {
+ return INSTANCE_SIZE +
+ sizeOf(dictionaryIndexes) +
+ columnStatistics.getRetainedSizeInBytes();
+ }
}
static class BlockStatistics
