Limit the size of ORC reader block to 1 GB
@@ -26,6 +26,7 @@ import com.facebook.presto.spi.block.SliceArrayBlock;
import com.facebook.presto.spi.type.Type;
import io.airlift.slice.Slice;
import io.airlift.slice.Slices;
+import io.airlift.units.DataSize;
import javax.annotation.Nonnull;
import javax.annotation.Nullable;
@@ -43,12 +44,15 @@ import static com.facebook.presto.spi.type.Chars.trimSpacesAndTruncateToLength;
import static com.facebook.presto.spi.type.Varchars.isVarcharType;
import static com.facebook.presto.spi.type.Varchars.truncateToLength;
import static com.google.common.base.MoreObjects.toStringHelper;
+import static io.airlift.units.DataSize.Unit.GIGABYTE;
+import static java.lang.Math.toIntExact;
import static java.util.Objects.requireNonNull;
public class SliceDirectStreamReader
implements StreamReader
{
private static final byte[] EMPTY_BYTE_ARRAY = new byte[0];
+ private static final int ONE_GIGABYTE = toIntExact(new DataSize(1, GIGABYTE).toBytes());
private final StreamDescriptor streamDescriptor;
@@ -137,19 +141,22 @@ public class SliceDirectStreamReader
}
}
- int totalLength = 0;
+ long totalLength = 0;
for (int i = 0; i < nextBatchSize; i++) {
if (!isNullVector[i]) {
totalLength += lengthVector[i];
}
}
+ if (totalLength > ONE_GIGABYTE) {
+ throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Column values too large to process in Presto. %s column values larger than 1GB", nextBatchSize);
+ }
byte[] data = EMPTY_BYTE_ARRAY;
if (totalLength > 0) {
if (dataStream == null) {
throw new OrcCorruptionException(streamDescriptor.getOrcDataSourceId(), "Value is not null but data stream is not present");
}
- data = dataStream.next(totalLength);
+ data = dataStream.next(toIntExact(totalLength));
}
Slice[] sliceVector = new Slice[nextBatchSize];
