Fixes to Spark S3A implementation PoC
@@ -77,7 +77,7 @@ function setup_object_storage_access {
fi
spark_config_file="$CP_CAP_SPARK_INSTALL_DIR/conf/spark-defaults.conf"
- if grep -q "spark.hadoop.fs.s3a" "$spark_config_file"; then
+ if grep -q "spark.hadoop.fs.s3a" "$spark_config_file" 2>/dev/null; then
pipe_log_warn "Spark config file at $spark_config_file already contains S3A configuration, if WILL NOT be updated" "$SPARK_MASTER_SETUP_TASK"
else
cat >> "$spark_config_file" << EOF
@@ -89,6 +89,16 @@ spark.driver.extraJavaOptions -Dcom.amazonaws.services
EOF
fi
+ spark_env_file="$CP_CAP_SPARK_INSTALL_DIR/conf/spark-env.sh"
+ if grep -q "AWS_" "$spark_env_file" 2>/dev/null; then
+ pipe_log_warn "Spark environment file at $spark_env_file already contains S3A configuration, if WILL NOT be updated" "$SPARK_MASTER_SETUP_TASK"
+ else
+ cat >> "$spark_env_file" << EOF
+AWS_ACCESS_KEY_ID="${!key_id_var_name}"
+AWS_SECRET_ACCESS_KEY="${!access_key_var_name}"
+EOF
+ fi
+
pipe_log_info "Object Storage access configuration is done" "$SPARK_MASTER_SETUP_TASK"
return 0
}
