Fix int overflow error when spilling large page
Page serialization requires page size to fin in an integer, so larger
pages could hit an int overflow error.
@@ -19,7 +19,6 @@ import com.facebook.presto.operator.SpillContext;
import com.facebook.presto.spi.PrestoException;
import com.facebook.presto.spi.page.PagesSerde;
import com.facebook.presto.spi.page.PagesSerdeUtil;
-import com.facebook.presto.spi.page.SerializedPage;
import com.facebook.presto.spi.spiller.SpillCipher;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.collect.AbstractIterator;
@@ -44,6 +43,8 @@ import java.util.Iterator;
import java.util.List;
import java.util.Optional;
+import static com.facebook.presto.common.block.PageBuilderStatus.DEFAULT_MAX_PAGE_SIZE_IN_BYTES;
+import static com.facebook.presto.execution.buffer.PageSplitterUtil.splitPage;
import static com.facebook.presto.spi.StandardErrorCode.GENERIC_INTERNAL_ERROR;
import static com.facebook.presto.spi.page.PagesSerdeUtil.writeSerializedPage;
import static com.facebook.presto.spiller.FileSingleStreamSpillerFactory.SPILL_FILE_PREFIX;
@@ -144,11 +145,15 @@ public class FileSingleStreamSpiller
while (pageIterator.hasNext()) {
Page page = pageIterator.next();
spilledPagesInMemorySize += page.getSizeInBytes();
- SerializedPage serializedPage = serde.serialize(page);
+ // page serialization requires page.getSizeInBytes() + Integer.BYTES to fit in an integer
+ splitPage(page, DEFAULT_MAX_PAGE_SIZE_IN_BYTES).stream()
+ .map(serde::serialize)
+ .forEach(serializedPage -> {
long pageSize = serializedPage.getSizeInBytes();
localSpillContext.updateBytes(pageSize);
spillerStats.addToTotalSpilledBytes(pageSize);
writeSerializedPage(output, serializedPage);
+ });
}
}
catch (UncheckedIOException | IOException e) {
