Add decimal dataType without precision - Modify saveDataType code
@@ -44,6 +44,7 @@ import scala.util.matching.Regex
import org.finra.herd.sdk.invoker.{ApiClient, ApiException}
import org.finra.herd.sdk.model._
+
/** A custom data source that integrates with Herd for metadata management
*
* It delegates to the built-in Spark file formats (PARQUET, CSV, ORC) for the actual reading and writing of data.
@@ -736,9 +737,7 @@ class DefaultSource(apiClientFactory: (String, Option[String], Option[String]) =
case DateType => col.setType("DATE")
case d: DecimalType =>
col.setType("DECIMAL")
- println("\n d.precision" + d.precision + "\n")
- println("\n d.scale" + d.scale + "\n")
- col.setSize( (if (d.precision.toString != null) d.precision.toString else "10") + "," + (if (d.scale.toString != null) d.scale.toString else "0"))
+ col.setSize(d.precision + "," + d.scale)
case TimestampType => col.setType("TIMESTAMP")
case BooleanType => col.setType("BOOLEAN")
case _ => col.setType(toComplexHerdType(column))
@@ -771,17 +770,12 @@ class DefaultSource(apiClientFactory: (String, Option[String], Option[String]) =
case "DOUBLE" => DoubleType
case "DATE" => DateType
case "DECIMAL" =>
- if (col.getSize() != null) {
- println("\ndecimal size:" + col.getSize() + "\n")
- val size = col.getSize()
+ var size = "10,0"
+ if(col.getSize != null) {
+ size = col.getSize()
+ }
val Array(precision, scale) = (if (size.indexOf(",") == -1) (size + ",0") else size).split(",").map(_.toInt)
- println("\nprecision: " + precision + "\n")
- println("\nscale: " + scale + "\n")
DecimalType(precision, scale)
- }
- else {
- DecimalType(10, 0)
- }
case "TIMESTAMP" => TimestampType
case "BOOLEAN" => BooleanType
case _ => toComplexSparkType(col)
