Herd/Spark - Defect - loadDataFrame loads partitions only from the latest format version
Formatted for readability purposes
@@ -363,18 +363,25 @@ class DefaultSource(apiClientFactory: (String, Option[String], Option[String]) =
.getBusinessObjectDataPartitions(params.namespace, params.businessObjectName, formatUsage, formatFileType, null, fmt.getPartitionKey,
partitionList, null)
val versionPattern = new Regex("/data-v([0-9]+)/")
- val allData = Seq.empty ++ partitionsFromDDL.getPartitions.asScala.map { partition =>
+ val allData = Seq.empty ++ partitionsFromDDL.getPartitions.asScala.map {
+ partition =>
(
- new Integer(formatVersion), if (partition.getPartitionColumns.get(0).getPartitionColumnValue == null) {
- "none"
- }
- else {
+ new Integer(formatVersion),
+
+ if (partition.getPartitionColumns.get(0).getPartitionColumnValue == null) {
+ "none" } else {
partition.getPartitionColumns.get(0).getPartitionColumnValue
- }, partition.getPartitionColumns.asScala.drop(1)
- .map(_.getPartitionColumnValue), versionPattern.findFirstMatchIn(partition.getPartitionLocation) match {
+ },
+
+ partition.getPartitionColumns.asScala.drop(1).map(_.getPartitionColumnValue),
+
+ versionPattern.findFirstMatchIn(partition.getPartitionLocation)
+ match {
case Some(i) => new Integer(i.group(1).toInt)
case None => new Integer(0)
- }, partition.getPartitionLocation
+ },
+
+ partition.getPartitionLocation
)
}
