Minor fix in DictionaryCompressionOptimizer
If an dictionary column failed to convert to direct encoded due to
stripe size limitation, it is no longer considered as a candidate
to direct conversion. However, we should still update the its
growth info and use the stats when select the dictionary column to
convert.
@@ -116,7 +116,9 @@ public class DictionaryCompressionOptimizer
.sum();
// update the dictionary growth history
- directConversionCandidates.forEach(column -> column.updateHistory(stripeRowCount));
+ allWriters.stream()
+ .filter(writer -> !writer.isDirectEncoded())
+ .forEach(column -> column.updateHistory(stripeRowCount));
if (dictionaryMemoryBytes <= dictionaryMemoryMaxBytesLow) {
return;
@@ -246,7 +248,8 @@ public class DictionaryCompressionOptimizer
long totalDictionaryBytesPerNewRow = 0;
long totalDictionaryIndexBytesPerRow = 0;
- for (DictionaryColumnManager column : directConversionCandidates) {
+ for (DictionaryColumnManager column : allWriters) {
+ if (!column.isDirectEncoded()) {
totalDictionaryRawBytes += column.getRawBytes();
totalDictionaryBytes += column.getDictionaryBytes();
totalDictionaryIndexBytes += column.getIndexBytes();
@@ -255,6 +258,7 @@ public class DictionaryCompressionOptimizer
totalDictionaryBytesPerNewRow += column.getDictionaryBytesPerFutureRow();
totalDictionaryIndexBytesPerRow += column.getIndexBytesPerRow();
}
+ }
long totalUncompressedBytesPerRow = totalNonDictionaryBytesPerRow + totalDictionaryRawBytesPerRow;
