Spark scheduler: fix worker count calculation and add env vars to the cp_env.sh
@@ -70,6 +70,10 @@ if [ $? -ne 0 ]; then
exit 1
fi
+if [ -f "$CP_ENV_FILE_TO_SOURCE" ]; then
+ echo "export PATH=\$PATH:$CP_CAP_SPARK_INSTALL_DIR/bin" >> "$CP_ENV_FILE_TO_SOURCE"
+fi
+
pipe_log_info "Spark installed into $CP_CAP_SPARK_INSTALL_DIR" "$SPARK_MASTER_SETUP_TASK"
# Check jq is installed (used to parse the Spark API responses)
@@ -105,6 +109,10 @@ if ! java -version > /dev/null 2>&1; then
exit 1
fi
+ if [ -f "$CP_ENV_FILE_TO_SOURCE" ]; then
+ echo "export JAVA_HOME=$JAVA_HOME" >> "$CP_ENV_FILE_TO_SOURCE"
+ fi
+
pipe_log_info "JAVA installed into $JAVA_HOME" "$SPARK_MASTER_SETUP_TASK"
fi
@@ -161,11 +169,11 @@ else
_MASTER_API_ENDPOINT="http://$CP_CAP_SPARK_HOST:$CP_CAP_SPARK_UI_PORT/json/"
_MASTER_EXEC_WAIT_ATTEMPTS=${_MASTER_EXEC_WAIT_ATTEMPTS:-60}
_MASTER_EXEC_WAIT_SEC=${_MASTER_EXEC_WAIT_SEC:-10}
- _CURRENT_EXEC_HOSTS_COUNT=$(curl -s $_MASTER_API_ENDPOINT | jq -r '.aliveworkers')
- while [ "$node_count" -gt "$_CURRENT_EXEC_HOSTS_COUNT" ]; do
- pipe_log_info "Waiting for workers to connect. $_CURRENT_EXEC_HOSTS_COUNT out of $node_count are ready" "$SPARK_MASTER_SETUP_TASK_WORKERS"
+ _CURRENT_WORKERS_COUNT=$(( $(curl -s $_MASTER_API_ENDPOINT | jq -r '.aliveworkers') - 1 ))
+ while [ "$node_count" -gt "$_CURRENT_WORKERS_COUNT" ]; do
+ pipe_log_info "Waiting for workers to connect. $_CURRENT_WORKERS_COUNT out of $node_count are ready" "$SPARK_MASTER_SETUP_TASK_WORKERS"
sleep $_MASTER_EXEC_WAIT_SEC
- _CURRENT_EXEC_HOSTS_COUNT=$(curl -s $_MASTER_API_ENDPOINT | jq -r '.aliveworkers')
+ _CURRENT_WORKERS_COUNT=$(( $(curl -s $_MASTER_API_ENDPOINT | jq -r '.aliveworkers') - 1 ))
_MASTER_EXEC_WAIT_ATTEMPTS=$(( _MASTER_EXEC_WAIT_ATTEMPTS-1 ))
if (( $_MASTER_EXEC_WAIT_ATTEMPTS <= 0 )); then
