Build Spark data catalog lib in Herd code base
-Add support to spark environment variables user/password
@@ -147,7 +147,17 @@ class DataCatalog(val spark: SparkSession, host: String) extends Serializable {
logger.info("credSDLC=" + credSDLC)
logger.info("credComponent=" + credComponent)
+ val user = spark.conf.getOption("spark.herd.username").orNull
+ if (user != null) {
+ this.username = user
+ } else {
+ this.username = credName
+ }
+ val pwd = spark.conf.getOption("spark.herd.password").orNull
+ if (pwd != null) {
+ this.password = pwd
+ } else {
var context = new util.HashMap[String, String] {
put("AGS", credAGS)
put("SDLC", credSDLC)
@@ -156,8 +166,6 @@ class DataCatalog(val spark: SparkSession, host: String) extends Serializable {
}
}
- this.username = credName
-
val clientConf = new ClientConfiguration
// TODO: Add proxy configuration from environment: "CRED_PROXY" and "CRED_PORT"
val provider = new DefaultAWSCredentialsProviderChain
@@ -165,6 +173,7 @@ class DataCatalog(val spark: SparkSession, host: String) extends Serializable {
val kms: AWSKMSClient = new AWSKMSClient(provider, clientConf).withRegion(Regions.US_EAST_1)
this.password = new JCredStash("credential-store", ddb, kms, new CredStashBouncyCastleCrypto).getSecret(credName, context)
}
+ }
/**
* Auxiliary constuctor using credstash
