Add more log in spilling stage
Adding logging during spilling and unspilling to help understand how much data has been spilled/unspilled during
query execution.
@@ -370,6 +370,9 @@ public class HashBuilderOperator
checkSpillSucceeded(spillInProgress);
spillInProgress = getSpiller().spill(page);
+ long retainedSizeOfPage = page.getRetainedSizeInBytes();
+ long sizeOfPage = page.getSizeInBytes();
+ log.debug("Spilling for operator %s, sizeOfPage %s, retinedSizeOfPage %s", operatorContext, sizeOfPage, retainedSizeOfPage);
// check that spilled data can still fit into memory limit as otherwise
// it fails later during unspilling when all spilled pages need to be loaded into memory
long maxUserMemoryBytes = getQueryMaxMemoryPerNode(operatorContext.getSession()).toBytes();
@@ -395,6 +398,8 @@ public class HashBuilderOperator
finishMemoryRevoke = Optional.of(() -> {
index.clear();
+ long estimatedIndexSize = index.getEstimatedSize().toBytes();
+ log.debug("Revoking Memory for operator %s, estimatedIndexSize: %s, spilling bytes: %s", operatorContext, estimatedIndexSize, localRevocableMemoryContext.getBytes());
localUserMemoryContext.setBytes(index.getEstimatedSize().toBytes(), enforceBroadcastMemoryLimit);
localRevocableMemoryContext.setBytes(0);
lookupSourceFactory.setPartitionSpilledLookupSourceHandle(partitionIndex, spilledLookupSourceHandle);
@@ -407,6 +412,8 @@ public class HashBuilderOperator
lookupSourceFactory.setPartitionSpilledLookupSourceHandle(partitionIndex, spilledLookupSourceHandle);
lookupSourceNotNeeded = Optional.empty();
index.clear();
+ long estimatedIndexSize = index.getEstimatedSize().toBytes();
+ log.debug("Revoking Memory for operator %s, estimatedIndexSize: %s, spilling bytes: %s", operatorContext, estimatedIndexSize, localRevocableMemoryContext.getBytes());
localUserMemoryContext.setBytes(index.getEstimatedSize().toBytes(), enforceBroadcastMemoryLimit);
localRevocableMemoryContext.setBytes(0);
lookupSourceChecksum = OptionalLong.of(lookupSourceSupplier.checksum());
@@ -555,7 +562,10 @@ public class HashBuilderOperator
verify(spiller.isPresent());
verify(!unspillInProgress.isPresent());
- localUserMemoryContext.setBytes(getSpiller().getSpilledPagesInMemorySize() + index.getEstimatedSize().toBytes(), enforceBroadcastMemoryLimit);
+ long memorySizeOfSpillPages = getSpiller().getSpilledPagesInMemorySize();
+ long estimatedSizeOfIndex = index.getEstimatedSize().toBytes();
+ log.debug("Unspilling lookup source for operator %s: memorySizeOfSpillPages: %s estimatedSizeOfIndex: %s", operatorContext, memorySizeOfSpillPages, estimatedSizeOfIndex);
+ localUserMemoryContext.setBytes(memorySizeOfSpillPages + estimatedSizeOfIndex, enforceBroadcastMemoryLimit);
unspillInProgress = Optional.of(getSpiller().getAllSpilledPages());
state = State.INPUT_UNSPILLING;
