xml attributePrefix option for spark1 vs spark2
@@ -59,6 +59,10 @@ public class XMLFileSchemaParser extends AbstractSparkFileSchemaParser implement
@PolicyProperty(name = "Row Tag", required = true, hint = "Specify root tag to extract from", value = ",",additionalProperties = {@PropertyLabelValue(label = "spark.option",value = "rowTag")})
private String rowTag = "";
+ @PolicyProperty(name = "Attribute Prefix", required = true, hint = "The prefix for attributes so that we can differentiating attributes and elements. This will be the prefix for field names", value = "_",additionalProperties = {@PropertyLabelValue(label = "spark.option",value = "attributePrefix")})
+ private String attributePrefix = "_";
+
+
@Override
public Schema parse(InputStream is, Charset charset, TableSchemaType target) throws IOException {
File tempFile = null;
@@ -83,7 +87,7 @@ public class XMLFileSchemaParser extends AbstractSparkFileSchemaParser implement
// Parse using Spark
try (InputStream fis = new FileInputStream(tempFile)) {
- schema = (HiveTableSchema) getSparkParserService().doParse(fis, SparkFileType.XML, target, new XMLCommandBuilder(hiveParse.getStartTag()));
+ schema = (HiveTableSchema) getSparkParserService().doParse(fis, SparkFileType.XML, target, new XMLCommandBuilder(hiveParse.getStartTag(),attributePrefix));
}
schema.setStructured(true);
@@ -137,7 +141,7 @@ public class XMLFileSchemaParser extends AbstractSparkFileSchemaParser implement
@Override
public SparkCommandBuilder getSparkCommandBuilder() {
- XMLCommandBuilder xmlCommandBuilder = new XMLCommandBuilder(getRowTag());
+ XMLCommandBuilder xmlCommandBuilder = new XMLCommandBuilder(getRowTag(), getAttributePrefix());
xmlCommandBuilder.setDataframeVariable(dataFrameVariable);
xmlCommandBuilder.setLimit(limit);
return xmlCommandBuilder;
@@ -170,8 +174,11 @@ public class XMLFileSchemaParser extends AbstractSparkFileSchemaParser implement
String xmlRowTag;
- XMLCommandBuilder(String rowTag) {
+ String attributePrefix;
+
+ XMLCommandBuilder(String rowTag, String attributePrefix) {
this.xmlRowTag = rowTag;
+ this.attributePrefix = attributePrefix;
}
@Override
@@ -180,7 +187,7 @@ public class XMLFileSchemaParser extends AbstractSparkFileSchemaParser implement
sb.append("\nimport com.databricks.spark.xml._;\n");
sb.append((dataframeVariable != null ? "var " + dataframeVariable + " = " : ""));
- sb.append(String.format("sqlContext.read.format(\"com.databricks.spark.xml\").option(\"rowTag\",\"%s\").load(\"%s\")", xmlRowTag, pathToFile));
+ sb.append(String.format("sqlContext.read.format(\"com.databricks.spark.xml\").option(\"rowTag\",\"%s\").option(\"attributePrefix\",\"%s\").load(\"%s\")", xmlRowTag, attributePrefix,pathToFile));
return sb.toString();
}
@@ -189,7 +196,7 @@ public class XMLFileSchemaParser extends AbstractSparkFileSchemaParser implement
StringBuilder sb = new StringBuilder();
sb.append("\nimport com.databricks.spark.xml._;\n");
- sb.append(unionDataFrames(paths,"sqlContext.read.format(\"com.databricks.spark.xml\").option(\"rowTag\",\"%s\").load(\"%s\")\n",xmlRowTag));
+ sb.append(unionDataFrames(paths,"sqlContext.read.format(\"com.databricks.spark.xml\").option(\"rowTag\",\"%s\").option(\"attributePrefix\",\"%s\").load(\"%s\")\n",xmlRowTag,attributePrefix));
return sb.toString();
}
@@ -205,6 +212,14 @@ public class XMLFileSchemaParser extends AbstractSparkFileSchemaParser implement
this.rowTag = rowTag;
}
+ public String getAttributePrefix() {
+ return attributePrefix;
+ }
+
+ public void setAttributePrefix(String attributePrefix) {
+ this.attributePrefix = attributePrefix;
+ }
+
static class HiveXMLSchemaHandler extends DefaultHandler {
/**
