Extract method for creating Hive split from InputSplit
@@ -306,28 +306,10 @@ public class BackgroundHiveSplitLoader
FileInputFormat.setInputPaths(targetJob, targetPath);
InputSplit[] targetSplits = targetInputFormat.getSplits(targetJob, 0);
- for (InputSplit inputSplit : targetSplits) {
- FileSplit split = (FileSplit) inputSplit;
- FileSystem targetFilesystem = hdfsEnvironment.getFileSystem(session.getUser(), split.getPath());
- FileStatus file = targetFilesystem.getFileStatus(split.getPath());
- hiveSplitSource.addToQueue(createHiveSplitIterator(
- partitionName,
- file.getPath().toString(),
- targetFilesystem.getFileBlockLocations(file, split.getStart(), split.getLength()),
- split.getStart(),
- split.getLength(),
- schema,
- partitionKeys,
- false,
- session,
- OptionalInt.empty(),
- effectivePredicate,
- partition.getColumnCoercions()));
- if (stopped) {
+ if (addSplitsToSource(targetSplits, partitionName, partitionKeys, schema, effectivePredicate, partition.getColumnCoercions())) {
return;
}
}
- }
return;
}
@@ -395,6 +377,39 @@ public class BackgroundHiveSplitLoader
fileIterators.addLast(iterator);
}
+ private boolean addSplitsToSource(
+ InputSplit[] targetSplits,
+ String partitionName,
+ List<HivePartitionKey> partitionKeys,
+ Properties schema,
+ TupleDomain<HiveColumnHandle> effectivePredicate,
+ Map<Integer, HiveType> columnCoercions)
+ throws IOException
+ {
+ for (InputSplit inputSplit : targetSplits) {
+ FileSplit split = (FileSplit) inputSplit;
+ FileSystem targetFilesystem = hdfsEnvironment.getFileSystem(session.getUser(), split.getPath());
+ FileStatus file = targetFilesystem.getFileStatus(split.getPath());
+ hiveSplitSource.addToQueue(createHiveSplitIterator(
+ partitionName,
+ file.getPath().toString(),
+ targetFilesystem.getFileBlockLocations(file, split.getStart(), split.getLength()),
+ split.getStart(),
+ split.getLength(),
+ schema,
+ partitionKeys,
+ false,
+ session,
+ OptionalInt.empty(),
+ effectivePredicate,
+ columnCoercions));
+ if (stopped) {
+ return true;
+ }
+ }
+ return false;
+ }
+
private void addToHiveSplitSourceRoundRobin(List<Iterator<HiveSplit>> iteratorList)
{
while (true) {
