Build Documentation

---
name: Build Documentation

on:
  workflow_call:
    inputs:
      salt-version:
        type: string
        required: true
        description: The Salt version to set prior to building packages.
      cache-seed:
        required: true
        type: string
        description: Seed used to invalidate caches

env:
  COLUMNS: 190
  AWS_MAX_ATTEMPTS: "10"
  AWS_RETRY_MODE: "adaptive"
  PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/
  PIP_EXTRA_INDEX_URL: https://pypi.org/simple
  PIP_DISABLE_PIP_VERSION_CHECK: "1"

jobs:
  build:
    name: Build
    runs-on:
      - ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        docs-output:
          - linkcheck
          - spellcheck
          - html
          - epub
          # - pdf

    steps:
      - uses: actions/checkout@v4

      - name: Download Release Patch
        if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
        uses: actions/download-artifact@v4
        with:
          name: salt-${{ inputs.salt-version }}.patch

      - name: Get Python Version
        id: get-python-version
        uses: ./.github/actions/get-python-version
        with:
          python-binary: python3

      - name: Setup Python Tools Scripts
        id: python-tools-scripts
        uses: ./.github/actions/setup-python-tools-scripts
        with:
          cache-prefix: ${{ inputs.cache-seed }}

      - name: Configure Git
        if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
        run: |
          tools pkg configure-git

      - name: Apply release patch
        if: ${{ startsWith(github.event.ref, 'refs/tags') == false }}
        run: |
          tools pkg apply-release-patch salt-${{ inputs.salt-version }}.patch --delete

      - name: Cache Python Tools Docs Virtualenv
        id: tools-venvs-dependencies-cache
        uses: ./.github/actions/cache
        with:
          path: .tools-venvs/docs
          key: ${{ inputs.cache-seed }}|${{ github.workflow }}|${{ github.job }}|tools-venvs|${{ steps.python-tools-scripts.outputs.version }}|docs|${{ steps.get-python-version.outputs.version }}|${{ hashFiles('requirements/**/docs.txt') }}

      - name: Prepare Docs Build
        run: |
          git clone https://gitlab.com/saltstack/open/docs/builddocs-fonts.git .builddocs-fonts
          sudo mkdir -p /usr/share/fonts/truetype /usr/share/fonts/opentype
          sudo cp -rfv .builddocs-fonts/truetype/*.ttf /usr/share/fonts/truetype/
          sudo cp -rfv .builddocs-fonts/opentype/*.otf /usr/share/fonts/opentype/
          sudo fc-cache -f -v

      - name: Build Documentation (${{ matrix.docs-output }})
        id: build-docs
        shell: bash
        continue-on-error: ${{ matrix.docs-output == 'linkcheck' || matrix.docs-output == 'spellcheck' }}
        env:
          LATEST_RELEASE: "${{ inputs.salt-version }}"
          SALT_ON_SALTSTACK: "1"
          ARCHIVE_FILENAME: "${{ format('salt-{0}-docs-{1}.tar.xz', inputs.salt-version, matrix.docs-output) }}"
        run: |
          tools docs ${{ matrix.docs-output }}

      - name: Upload Built Documentation Artifact(${{ matrix.docs-output }})
        if: ${{ steps.build-docs.outputs.has-artifacts == 'true' }}
        uses: actions/upload-artifact@v4
        with:
          name: ${{ steps.build-docs.outputs.artifact-name }}
          path: ${{ steps.build-docs.outputs.artifact-path }}
          retention-days: 7
          if-no-files-found: error


Test Artifact

name: Test Artifact

on:
  workflow_call:
    inputs:
      distro-slug:
        required: true
        type: string
        description: The OS slug to run tests against
      runner:
        required: true
        type: string
        description: The GitHub runner name
      platform:
        required: true
        type: string
        description: The platform being tested
      arch:
        required: true
        type: string
        description: The platform arch being tested
      pkg-type:
        required: true
        type: string
        description: The platform arch being tested
      salt-version:
        type: string
        required: true
        description: The Salt version of the packages to install and test
      cache-prefix:
        required: true
        type: string
        description: Seed used to invalidate caches
      testing-releases:
        required: true
        type: string
        description: A JSON list of releases to test upgrades against
      nox-version:
        required: true
        type: string
        description: The nox version to install
      python-version:
        required: false
        type: string
        description: The python version to run tests with
        default: "3.10"
      package-name:
        required: false
        type: string
        description: The onedir package name to use
        default: salt
      nox-session:
        required: false
        type: string
        description: The nox session to run
        default: ci-test-onedir
      skip-code-coverage:
        required: false
        type: boolean
        description: Skip code coverage
        default: false

env:
  COLUMNS: 190
  PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/
  PIP_EXTRA_INDEX_URL: https://pypi.org/simple
  PIP_DISABLE_PIP_VERSION_CHECK: "1"
  RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"

jobs:

  generate-matrix:
    name: Generate Matrix
    runs-on:
      # We need to run on our self-hosted runners because we need proper credentials
      # for boto3 to scan through our repositories.
      - self-hosted
      - linux
      - x86_64
    outputs:
      pkg-matrix-include: ${{ steps.generate-pkg-matrix.outputs.matrix }}
      build-reports: ${{ steps.generate-pkg-matrix.outputs.build-reports }}
    steps:

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t"

      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: Setup Python Tools Scripts
        uses: ./.github/actions/setup-python-tools-scripts
        with:
          cache-prefix: ${{ inputs.cache-prefix }}

      - name: Generate Package Test Matrix
        id: generate-pkg-matrix
        run: |
          tools ci pkg-matrix ${{ inputs.distro-slug }} ${{ inputs.pkg-type }}  --testing-releases ${{ join(fromJSON(inputs.testing-releases), ' ') }}

  test:
    name: Test
    runs-on: ${{ inputs.runner }}
    timeout-minutes: 120  # 2 Hours - More than this and something is wrong
    needs:
      - generate-matrix
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.generate-matrix.outputs.pkg-matrix-include) }}

    steps:

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t"

      - name: "Set `TIMESTAMP` environment variable"
        shell: bash
        run: |
          echo "TIMESTAMP=$(date +%s)" | tee -a "$GITHUB_ENV"

      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: Download Packages
        uses: actions/download-artifact@v4
        with:
          name: salt-${{ inputs.salt-version }}-${{ inputs.arch }}-${{ inputs.pkg-type }}
          path: artifacts/pkg/

      - name: Install System Dependencies
        run: |
          brew install tree

      - name: List Packages
        run: |
          tree artifacts/pkg/

      - name: Download Onedir Tarball as an Artifact
        uses: actions/download-artifact@v4
        with:
          name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
          path: artifacts/

      - name: Decompress Onedir Tarball
        shell: bash
        run: |
          python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
          cd artifacts
          tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz

      - name: Set up Python ${{ inputs.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: "${{ inputs.python-version }}"

      - name: Install Nox
        run: |
          python3 -m pip install 'nox==${{ inputs.nox-version }}'

      - name: Download nox.macos.${{ inputs.arch }}.tar.* artifact for session ${{ inputs.nox-session }}
        uses: actions/download-artifact@v4
        with:
          name: nox-macos-${{ inputs.arch }}-${{ inputs.nox-session }}

      - name: Decompress .nox Directory
        run: |
          nox --force-color -e decompress-dependencies -- macos ${{ inputs.arch }}

      - name: Show System Info
        env:
          SKIP_REQUIREMENTS_INSTALL: "1"
          PRINT_SYSTEM_INFO_ONLY: "1"
        run: |
          sudo -E nox --force-color -e ${{ inputs.nox-session }}-pkgs -- ${{ matrix.tests-chunk }}

      - name: Run Package Tests
        env:
          SKIP_REQUIREMENTS_INSTALL: "1"
          PRINT_TEST_SELECTION: "0"
          PRINT_TEST_PLAN_ONLY: "0"
          PRINT_SYSTEM_INFO: "0"
          RERUN_FAILURES: "1"
          GITHUB_ACTIONS_PIPELINE: "1"
          SKIP_INITIAL_GH_ACTIONS_FAILURES: "1"
          COVERAGE_CONTEXT: ${{ inputs.distro-slug }}
        run: |
          sudo -E nox --force-color -e ${{ inputs.nox-session }}-pkgs -- ${{ matrix.tests-chunk }} \
          ${{ matrix.version && format('--prev-version={0}', matrix.version) || ''}}

      - name: Fix file ownership
        run: |
          sudo chown -R "$(id -un)" .

      - name: Prepare Test Run Artifacts
        id: download-artifacts-from-vm
        if: always()
        run: |
          # Delete the salt onedir, we won't need it anymore and it will prevent
          # from it showing in the tree command below
          rm -rf artifacts/salt*
          tree -a artifacts

      - name: Upload Test Run Artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}-${{ inputs.arch }}-${{ matrix.tests-chunk }}-${{ matrix.version || 'no-version'}}-${{ env.TIMESTAMP }}
          path: |
            artifacts
            !artifacts/pkg/*
            !artifacts/salt/*
            !artifacts/salt-*.tar.*

  report:
    name: Report
    runs-on: ubuntu-latest
    if: always() && fromJSON(needs.generate-matrix.outputs.build-reports) && needs.test.result != 'cancelled' && needs.test.result != 'skipped'
    needs:
      - generate-matrix
      - test

    steps:
      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t"

      - name: Merge Test Run Artifacts
        continue-on-error: true
        uses: actions/upload-artifact/merge@v4
        with:
          name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}
          pattern: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}-*
          separate-directories: true
          delete-merged: true

      - name: Download Test Run Artifacts
        id: download-test-run-artifacts
        uses: actions/download-artifact@v4
        with:
          name: pkg-testrun-artifacts-${{ inputs.distro-slug }}-${{ inputs.pkg-type }}
          path: artifacts

      - name: Show Test Run Artifacts
        if: always() && steps.download-test-run-artifacts.outcome == 'success'
        run: |
          tree -a artifacts

      - name: Set up Python ${{ inputs.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: "${{ inputs.python-version }}"

      - name: Install Nox
        run: |
          python3 -m pip install 'nox==${{ inputs.nox-version }}'


Backport PR

name: Backport PR
run-name: "Backport PR #${{ github.event.number }}"

on:
  pull_request_target:
    types:
      - "labeled"
      - "closed"

permissions:
  contents: write
  pull-requests: write

jobs:
  backport:
    name: Backport PR
    runs-on:
      - ubuntu-latest
    if: |
      github.event.pull_request.merged == true
      && (
        contains(github.event.pull_request.labels.*.name, 'backport:master') ||
        contains(github.event.pull_request.labels.*.name, 'backport:3006.x') ||
        contains(github.event.pull_request.labels.*.name, 'backport:3005.x')
      )
      && (
        (github.event.action == 'labeled' && (
          contains(github.event.pull_request.labels.*.name, 'backport:master') ||
          contains(github.event.pull_request.labels.*.name, 'backport:3006.x') ||
          contains(github.event.pull_request.labels.*.name, 'backport:3005.x')
        ))
        || (github.event.action == 'closed')
      )
    steps:
      - name: Backport Action
        uses: sorenlouv/backport-github-action@v8.9.7
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          auto_backport_label_prefix: "backport:"
          add_original_reviewers: true

      - name: Info log
        if: ${{ success() }}
        run: jq -C '.' ~/.backport/backport.info.log

      - name: Debug log
        if: ${{ failure() }}
        run: jq -C '.' ~/.backport/backport.debug.log


Update Winrepo

---
name: Update Winrepo

on:
  release:
    types: [released]
  workflow_dispatch:
    inputs:
      salt-version:
        type: string
        required: true
        description: >
          The Salt version to set prior to building packages and staging the release.
          (DO NOT prefix the version with a v, ie, 3006.0 NOT v3006.0).

permissions:
  contents: read

jobs:
  update-winrepo:
    name: Update Winrepo
    runs-on: ubuntu-latest
    steps:

    - name: Checkout Salt
      uses: actions/checkout@v4
      with:
        path: salt

    - name: Checkout WinRepo
      uses: actions/checkout@v4
      with:
        path: winrepo
        repository: saltstack/salt-winrepo-ng

    - name: Set Up Python 3.10
      uses: actions/setup-python@v5
      with:
        python-version: "3.10"

    - name: Add Version to Minion Definition File
      working-directory: salt
      run: |
        pwd
        ls -al ../winrepo/salt-minion.sls
        python .github/workflows/scripts/update_winrepo.py \
          --file ../winrepo/salt-minion.sls \
          --version ${{ inputs.salt-version || github.ref_name }}
        grep ${{ inputs.salt-version || github.ref_name }} ../winrepo/salt-minion.sls

    - name: Commit Changes
      working-directory: winrepo
      run: |
        git status
        git add .
        git config user.name 'saltbot-open'
        git config user.email 'saltbot-open@users.noreply.github.com'
        git commit -m "Adding salt minion ${{ inputs.salt-version && format('v{0}', inputs.salt-version) || github.ref_name }}"

    - name: Create Pull Request
      id: cpr
      uses: peter-evans/create-pull-request@v5
      with:
        path: winrepo
        push-to-fork: saltbot-open/salt-winrepo-ng
        token: ${{ secrets.SALTBOT_OPEN_SALT_WINREPO_NG }}

#    - name: Enable Pull Request Automerge
#      if: steps.cpr.outputs.pull-request-operation == 'created'
#      uses: peter-evans/enable-pull-request-automerge@v2
#      with:
#        token: ${{ secrets.SALTBOT_OPEN_SALT_WINREPO_NG }}
#        pull-request-number: ${{ steps.cpr.outputs.pull-request-number }}
#        merge-method: squash


Upload VirusTotal

---
name: Upload VirusTotal

on:
  release:
    types: [released]
  workflow_dispatch:
    inputs:
      salt-version:
        type: string
        required: true
        description: >
          The Salt version to get from staging to upload to VirusTotal.
          (DO NOT prefix the version with a v, ie, 3006.0 NOT v3006.0).

permissions:
  contents: read

env:
  COLUMNS: 190
  AWS_MAX_ATTEMPTS: "10"
  AWS_RETRY_MODE: "adaptive"
  PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/
  PIP_EXTRA_INDEX_URL: https://pypi.org/simple

jobs:
  upload-virustotal:
    name: Upload VirusTotal
    environment: release
    runs-on:
      - self-hosted
      - linux
      - repo-release
    steps:

    - name: Checkout Salt
      uses: actions/checkout@v4

    - name: Set Up Python 3.10
      uses: actions/setup-python@v5
      with:
        python-version: "3.10"

    - name: Setup Python Tools Scripts
      uses: ./.github/actions/setup-python-tools-scripts
      with:
        cache-prefix: virus-total

    - name: Upload to VirusTotal
      env:
        VIRUSTOTAL_API_KEY: ${{ secrets.VIRUSTOTAL_API_KEY }}
      run: |
        tools release upload-virustotal ${{ inputs.salt-version || github.ref_name }}


Install Test Dependencies

---
name: Install Test Dependencies

on:
  workflow_call:
    inputs:
      nox-session:
        required: true
        type: string
        description: The nox session to run
      salt-version:
        type: string
        required: true
        description: The Salt version to set prior to running tests.
      cache-prefix:
        required: true
        type: string
        description: Seed used to invalidate caches
      nox-version:
        required: true
        type: string
        description: The nox version to install
      nox-archive-hash:
        required: true
        type: string
        description: Nox Tarball Cache Hash
      python-version:
        required: false
        type: string
        description: The python version to run tests with
        default: "3.10"
      package-name:
        required: false
        type: string
        description: The onedir package name to use
        default: salt

env:
  COLUMNS: 190
  AWS_MAX_ATTEMPTS: "10"
  AWS_RETRY_MODE: "adaptive"
  PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/
  PIP_EXTRA_INDEX_URL: https://pypi.org/simple
  PIP_DISABLE_PIP_VERSION_CHECK: "1"
  RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"

jobs:

  generate-matrix:
    name: Generate Matrix
    runs-on: ubuntu-latest
    outputs:
      matrix-include: ${{ steps.generate-matrix.outputs.matrix }}
    steps:

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t"

      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: Setup Python Tools Scripts
        uses: ./.github/actions/setup-python-tools-scripts
        with:
          cache-prefix: ${{ inputs.cache-prefix }}

      - name: Generate Test Matrix
        id: generate-matrix
        run: |
          tools ci deps-matrix

  linux-dependencies:
    name: Linux
    needs:
      - generate-matrix
    runs-on:
      - self-hosted
      - linux
      - bastion
    env:
      USE_S3_CACHE: 'true'
    timeout-minutes: 90
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include)['linux'] }}
    steps:

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t"

      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: Cache nox.linux.${{ matrix.arch }}.tar.* for session ${{ inputs.nox-session }}
        id: nox-dependencies-cache
        uses: ./.github/actions/cache
        with:
          path: nox.linux.${{ matrix.arch }}.tar.*
          key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|linux|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{ inputs.nox-archive-hash }}

      - name: Download Onedir Tarball as an Artifact
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        uses: actions/download-artifact@v4
        with:
          name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-linux-${{ matrix.arch }}.tar.xz
          path: artifacts/

      - name: Decompress Onedir Tarball
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        shell: bash
        run: |
          python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
          cd artifacts
          tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-linux-${{ matrix.arch }}.tar.xz

      - name: PyPi Proxy
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          sed -i '7s;^;--index-url=https://pypi-proxy.saltstack.net/root/local/+simple/ --extra-index-url=https://pypi.org/simple\n;' requirements/static/ci/*/*.txt

      - name: Setup Python Tools Scripts
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        uses: ./.github/actions/setup-python-tools-scripts
        with:
          cache-prefix: ${{ inputs.cache-prefix }}-build-deps-ci

      - name: Get Salt Project GitHub Actions Bot Environment
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30")
          SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment)
          echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV"

      - name: Start VM
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        id: spin-up-vm
        run: |
          tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ matrix.distro-slug }}

      - name: List Free Space
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          tools --timestamps vm ssh ${{ matrix.distro-slug }} -- df -h || true

      - name: Upload Checkout To VM
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          tools --timestamps vm rsync ${{ matrix.distro-slug }}

      - name: Install Dependencies
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          tools --timestamps vm install-dependencies --nox-session=${{ inputs.nox-session }} ${{ matrix.distro-slug }}

      - name: Cleanup .nox Directory
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          tools --timestamps vm pre-archive-cleanup ${{ matrix.distro-slug }}

      - name: Compress .nox Directory
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          tools --timestamps vm compress-dependencies ${{ matrix.distro-slug }}

      - name: Download Compressed .nox Directory
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          tools --timestamps vm download-dependencies ${{ matrix.distro-slug }}

      - name: Destroy VM
        if: always() && steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          tools --timestamps vm destroy --no-wait ${{ matrix.distro-slug }}

      - name: Upload Nox Requirements Tarball
        uses: actions/upload-artifact@v4
        with:
          name: nox-linux-${{ matrix.arch }}-${{ inputs.nox-session }}
          path: nox.linux.${{ matrix.arch }}.tar.*

  macos-dependencies:
    name: MacOS
    needs:
      - generate-matrix
    runs-on: ${{ matrix.distro-slug == 'macos-13-arm64' && 'macos-13-xlarge' || matrix.distro-slug }}
    timeout-minutes: 90
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include)['macos'] }}
    steps:

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t"

      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: Cache nox.macos.${{ matrix.arch }}.tar.* for session ${{ inputs.nox-session }}
        id: nox-dependencies-cache
        uses: ./.github/actions/cache
        with:
          path: nox.macos.${{ matrix.arch }}.tar.*
          key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|macos|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{ inputs.nox-archive-hash }}

      - name: Download Onedir Tarball as an Artifact
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        uses: actions/download-artifact@v4
        with:
          name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-macos-${{ matrix.arch }}.tar.xz
          path: artifacts/

      - name: Decompress Onedir Tarball
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        shell: bash
        run: |
          python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
          cd artifacts
          tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-macos-${{ matrix.arch }}.tar.xz

      - name: Set up Python ${{ inputs.python-version }}
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        uses: actions/setup-python@v5
        with:
          python-version: "${{ inputs.python-version }}"

      - name: Install System Dependencies
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          brew install openssl@3

      - name: Install Nox
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          python3 -m pip install 'nox==${{ inputs.nox-version }}'

      - name: Install Dependencies
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        env:
          PRINT_TEST_SELECTION: "0"
          PRINT_SYSTEM_INFO: "0"
        run: |
          export PYCURL_SSL_LIBRARY=openssl
          export LDFLAGS="-L/usr/local/opt/openssl@3/lib"
          export CPPFLAGS="-I/usr/local/opt/openssl@3/include"
          export PKG_CONFIG_PATH="/usr/local/opt/openssl@3/lib/pkgconfig"
          nox --install-only -e ${{ inputs.nox-session }}

      - name: Cleanup .nox Directory
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          nox --force-color -e "pre-archive-cleanup(pkg=False)"

      - name: Compress .nox Directory
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          nox --force-color -e compress-dependencies -- macos ${{ matrix.arch }}

      - name: Upload Nox Requirements Tarball
        uses: actions/upload-artifact@v4
        with:
          name: nox-macos-${{ matrix.arch }}-${{ inputs.nox-session }}
          path: nox.macos.${{ matrix.arch }}.tar.*

  windows-dependencies:
    needs:
      - generate-matrix
    name: Windows
    runs-on:
      - self-hosted
      - linux
      - bastion
    env:
      USE_S3_CACHE: 'true'
    timeout-minutes: 90
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include)['windows'] }}
    steps:

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t"

      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: Cache nox.windows.${{ matrix.arch }}.tar.* for session ${{ inputs.nox-session }}
        id: nox-dependencies-cache
        uses: ./.github/actions/cache
        with:
          path: nox.windows.${{ matrix.arch }}.tar.*
          key: ${{ inputs.cache-prefix }}|testrun-deps|${{ matrix.arch }}|windows|${{ inputs.nox-session }}|${{ inputs.python-version }}|${{ inputs.nox-archive-hash }}

      - name: Download Onedir Tarball as an Artifact
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        uses: actions/download-artifact@v4
        with:
          name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-windows-${{ matrix.arch }}.tar.xz
          path: artifacts/

      - name: Decompress Onedir Tarball
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        shell: bash
        run: |
          python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
          cd artifacts
          tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-windows-${{ matrix.arch }}.tar.xz

      - name: PyPi Proxy
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          sed -i '7s;^;--index-url=https://pypi-proxy.saltstack.net/root/local/+simple/ --extra-index-url=https://pypi.org/simple\n;' requirements/static/ci/*/*.txt

      - name: Setup Python Tools Scripts
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        uses: ./.github/actions/setup-python-tools-scripts
        with:
          cache-prefix: ${{ inputs.cache-prefix }}-build-deps-ci

      - name: Get Salt Project GitHub Actions Bot Environment
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30")
          SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment)
          echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV"

      - name: Start VM
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        id: spin-up-vm
        run: |
          tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ matrix.distro-slug }}

      - name: List Free Space
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          tools --timestamps vm ssh ${{ matrix.distro-slug }} -- df -h || true

      - name: Upload Checkout To VM
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          tools --timestamps vm rsync ${{ matrix.distro-slug }}

      - name: Install Dependencies
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          tools --timestamps vm install-dependencies --nox-session=${{ inputs.nox-session }} ${{ matrix.distro-slug }}

      - name: Cleanup .nox Directory
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          tools --timestamps vm pre-archive-cleanup ${{ matrix.distro-slug }}

      - name: Compress .nox Directory
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          tools --timestamps vm compress-dependencies ${{ matrix.distro-slug }}

      - name: Download Compressed .nox Directory
        if: steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          tools --timestamps vm download-dependencies ${{ matrix.distro-slug }}

      - name: Destroy VM
        if: always() && steps.nox-dependencies-cache.outputs.cache-hit != 'true'
        run: |
          tools --timestamps vm destroy --no-wait ${{ matrix.distro-slug }}

      - name: Upload Nox Requirements Tarball
        uses: actions/upload-artifact@v4
        with:
          name: nox-windows-${{ matrix.arch }}-${{ inputs.nox-session }}
          path: nox.windows.${{ matrix.arch }}.tar.*


Test Download Packages

name: Test Download Packages

on:
  workflow_call:
    inputs:
      salt-version:
        type: string
        required: true
        description: The Salt version of the packages to install and test
      cache-prefix:
        required: true
        type: string
        description: Seed used to invalidate caches
      environment:
        required: true
        type: string
        description: The environment to run tests against
      latest-release:
        required: true
        type: string
        description: The latest salt release
      nox-version:
        required: true
        type: string
        description: The nox version to install
      python-version:
        required: false
        type: string
        description: The python version to run tests with
        default: "3.10"
      package-name:
        required: false
        type: string
        description: The onedir package name to use
        default: salt
      skip-code-coverage:
        required: false
        type: boolean
        description: Skip code coverage
        default: false
      nox-session:
        required: false
        type: string
        description: The nox session to run
        default: ci-test-onedir

env:
  COLUMNS: 190
  AWS_MAX_ATTEMPTS: "10"
  AWS_RETRY_MODE: "adaptive"
  PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/
  PIP_EXTRA_INDEX_URL: https://pypi.org/simple
  PIP_DISABLE_PIP_VERSION_CHECK: "1"
  RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"

jobs:

  generate-matrix:
    name: Generate Matrix
    runs-on: ubuntu-latest
    outputs:
      matrix-include: ${{ steps.generate-matrix.outputs.matrix }}
    steps:

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t"

      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: Setup Python Tools Scripts
        uses: ./.github/actions/setup-python-tools-scripts
        with:
          cache-prefix: ${{ inputs.cache-prefix }}

      - name: Generate Test Matrix
        id: generate-matrix
        run: |
          tools ci pkg-downloads-matrix

  linux:
    name: Linux
    needs:
      - generate-matrix
    runs-on:
      - self-hosted
      - linux
      - bastion
    env:
      USE_S3_CACHE: 'true'
    environment: ${{ inputs.environment }}
    timeout-minutes: 120  # 2 Hours - More than this and something is wrong
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include)['linux'] }}

    steps:

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t"

      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: Download Onedir Tarball as an Artifact
        uses: actions/download-artifact@v4
        with:
          name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-linux-${{ matrix.arch == 'aarch64' && 'arm64' || matrix.arch }}.tar.xz
          path: artifacts/

      - name: Decompress Onedir Tarball
        shell: bash
        run: |
          python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
          cd artifacts
          tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-linux-${{ matrix.arch == 'aarch64' && 'arm64' || matrix.arch }}.tar.xz

      - name: Download nox.linux.${{ matrix.arch == 'aarch64' && 'arm64' || matrix.arch }}.tar.* artifact for session ${{ inputs.nox-session }}
        uses: actions/download-artifact@v4
        with:
          name: nox-linux-${{ matrix.arch == 'aarch64' && 'arm64' || matrix.arch }}-${{ inputs.nox-session }}

      - name: Setup Python Tools Scripts
        uses: ./.github/actions/setup-python-tools-scripts
        with:
          cache-prefix: ${{ inputs.cache-prefix }}-pkg-download-linux

      - name: Get Salt Project GitHub Actions Bot Environment
        run: |
          TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30")
          SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment)
          echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV"

      - name: Start VM
        id: spin-up-vm
        run: |
          tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ matrix.distro-slug }}

      - name: List Free Space
        run: |
          tools --timestamps vm ssh ${{ matrix.distro-slug }} -- df -h || true

      - name: Upload Checkout To VM
        run: |
          tools --timestamps vm rsync ${{ matrix.distro-slug }}

      - name: Decompress .nox Directory
        run: |
          tools --timestamps vm decompress-dependencies ${{ matrix.distro-slug }}

      - name: Show System Info
        run: |
          tools --timestamps --timeout-secs=1800 vm test --skip-requirements-install --print-system-information-only \
            --nox-session=${{ inputs.nox-session }}-pkgs ${{ matrix.distro-slug }} -- download-pkgs

      - name: Run Package Download Tests
        env:
          SALT_RELEASE: "${{ inputs.salt-version }}"
          SALT_REPO_ARCH: ${{ matrix.arch }}
          SALT_REPO_TYPE: ${{ inputs.environment }}
          SALT_REPO_USER: ${{ secrets.SALT_REPO_USER }}
          SALT_REPO_PASS: ${{ secrets.SALT_REPO_PASS }}
          SALT_REPO_DOMAIN_RELEASE: ${{ vars.SALT_REPO_DOMAIN_RELEASE || 'repo.saltproject.io' }}
          SALT_REPO_DOMAIN_STAGING: ${{ vars.SALT_REPO_DOMAIN_STAGING || 'staging.repo.saltproject.io' }}
          SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
          LATEST_SALT_RELEASE: "${{ inputs.latest-release }}"
          DOWNLOAD_TEST_PACKAGE_TYPE: ${{ matrix.pkg-type }}
        run: |
          tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
          -E SALT_RELEASE -E SALT_REPO_ARCH -E SALT_REPO_TYPE -E SALT_REPO_USER -E SALT_REPO_PASS \
          -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING  -E LATEST_SALT_RELEASE -E DOWNLOAD_TEST_PACKAGE_TYPE \
            --nox-session=${{ inputs.nox-session }}-pkgs --rerun-failures ${{ matrix.distro-slug }} -- download-pkgs

      - name: Combine Coverage Reports
        if: always() && inputs.skip-code-coverage == false && steps.spin-up-vm.outcome == 'success' && job.status != 'cancelled'
        run: |
          tools --timestamps vm combine-coverage ${{ matrix.distro-slug }}

      - name: Download Test Run Artifacts
        id: download-artifacts-from-vm
        if: always() && steps.spin-up-vm.outcome == 'success'
        run: |
          tools --timestamps vm download-artifacts ${{ matrix.distro-slug }}
          # Delete the salt onedir, we won't need it anymore and it will prevent
          # from it showing in the tree command below
          rm -rf artifacts/salt*
          tree -a artifacts

      - name: Destroy VM
        if: always()
        run: |
          tools --timestamps vm destroy --no-wait ${{ matrix.distro-slug }} || true

      - name: Fix file ownership
        run: |
          sudo chown -R "$(id -un)" .

      - name: Install Codecov CLI
        if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled'
        run: |
          # We can't yet use tokenless uploads with the codecov CLI
          # python3 -m pip install codecov-cli
          #
          curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --import
          curl -Os https://uploader.codecov.io/latest/linux/codecov
          curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM
          curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM.sig
          gpg --verify codecov.SHA256SUM.sig codecov.SHA256SUM
          shasum -a 256 -c codecov.SHA256SUM
          chmod +x codecov

      - name: Upload Source Code Coverage To Codecov
        if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled'
        run: |
          if [ ! -s artifacts/coverage/salt.xml ]; then
            echo "The artifacts/coverage/salt.xml file does not exist"
            exit 1
          fi
          # We can't yet use tokenless uploads with the codecov CLI
          #codecovcli --auto-load-params-from GithubActions --verbose --token ${{ secrets.CODECOV_TOKEN }} \
          #  do-upload --git-service github --sha ${{ github.sha }} \
          #  --file artifacts/coverage/salt.xml \
          #  --flag salt --flag ${{ matrix.distro-slug }} --flag pkg \
          #  --name salt.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs
          n=0
          until [ "$n" -ge 5 ]
          do
          if ./codecov --file artifacts/coverage/salt.xml \
              --sha ${{ github.event.pull_request.head.sha || github.sha }} ${{ github.event_name == 'pull_request' && format('--parent {0}', github.event.pull_request.base.sha) }} \
              --flags salt,${{ matrix.distro-slug }},pkg \
              --name salt.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs --nonZero; then
              rc=$?
              break
          fi
          rc=$?
          n=$((n+1))
          sleep 15
          done
          if [ "$rc" -ne 0 ]; then
            echo "Failed to upload codecov stats"
            exit 1
          fi

      - name: Upload Tests Code Coverage To Codecov
        if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled'
        run: |
          if [ ! -s artifacts/coverage/tests.xml ]; then
            echo "The artifacts/coverage/tests.xml file does not exist"
            exit 1
          fi
          # We can't yet use tokenless uploads with the codecov CLI
          #codecovcli --auto-load-params-from GithubActions --verbose --token ${{ secrets.CODECOV_TOKEN }} \
          #  do-upload --git-service github --sha ${{ github.sha }} \
          #  --file artifacts/coverage/tests.xml \
          #  --flag tests --flag ${{ matrix.distro-slug }} --flag pkg \
          #  --name tests.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs
          n=0
          until [ "$n" -ge 5 ]
          do
          if ./codecov --file artifacts/coverage/tests.xml \
              --sha ${{ github.event.pull_request.head.sha || github.sha }} ${{ github.event_name == 'pull_request' && format('--parent {0}', github.event.pull_request.base.sha) }} \
              --flags tests,${{ matrix.distro-slug }},pkg \
              --name tests.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs --nonZero; then
              rc=$?
              break
          fi
          rc=$?
          n=$((n+1))
          sleep 15
          done
          if [ "$rc" -ne 0 ]; then
            echo "Failed to upload codecov stats"
            exit 1
          fi

      - name: Upload Test Run Artifacts
        if: always() && steps.download-artifacts-from-vm.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: pkg-testrun-artifacts-${{ matrix.distro-slug }}-${{ matrix.arch }}-${{ matrix.pkg-type }}
          path: |
            artifacts
            !artifacts/salt/*
            !artifacts/salt-*.tar.*

  macos:
    name: MacOS
    needs:
      - generate-matrix
    runs-on: ${{ matrix.distro-slug == 'macos-13-arm64' && 'macos-13-xlarge' || matrix.distro-slug }}
    env:
      USE_S3_CACHE: 'false'
    environment: ${{ inputs.environment }}
    timeout-minutes: 120  # 2 Hours - More than this and something is wrong
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include)['macos'] }}

    steps:

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t"

      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: Download Onedir Tarball as an Artifact
        uses: actions/download-artifact@v4
        with:
          name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-macos-${{ matrix.arch }}.tar.xz
          path: artifacts/

      - name: Install System Dependencies
        run: |
          brew install tree

      - name: Decompress Onedir Tarball
        shell: bash
        run: |
          python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
          cd artifacts
          tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-macos-${{ matrix.arch }}.tar.xz

      - name: Set up Python ${{ inputs.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: "${{ inputs.python-version }}"
          update-environment: true

      - name: Install Nox
        run: |
          python3 -m pip install 'nox==${{ inputs.nox-version }}'

      - name: Download nox.macos.${{ matrix.arch }}.tar.* artifact for session ${{ inputs.nox-session }}
        uses: actions/download-artifact@v4
        with:
          name: nox-macos-${{ matrix.arch }}-${{ inputs.nox-session }}

      - name: Decompress .nox Directory
        run: |
          nox --force-color -e decompress-dependencies -- macos ${{ matrix.arch }}

      - name: Show System Info
        env:
          SKIP_REQUIREMENTS_INSTALL: "1"
          PRINT_SYSTEM_INFO_ONLY: "1"
        run: |
          sudo -E nox --force-color -e ${{ inputs.nox-session }}-pkgs -- download-pkgs

      - name: Run Package Download Tests
        env:
          SKIP_REQUIREMENTS_INSTALL: "1"
          PRINT_TEST_SELECTION: "0"
          PRINT_TEST_PLAN_ONLY: "0"
          PRINT_SYSTEM_INFO: "0"
          RERUN_FAILURES: "1"
          GITHUB_ACTIONS_PIPELINE: "1"
          SKIP_INITIAL_GH_ACTIONS_FAILURES: "1"
          SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
          COVERAGE_CONTEXT: ${{ matrix.distro-slug }}
          SALT_RELEASE: "${{ inputs.salt-version }}"
          SALT_REPO_ARCH: ${{ matrix.arch }}
          LATEST_SALT_RELEASE: "${{ inputs.latest-release }}"
          SALT_REPO_TYPE: ${{ inputs.environment }}
          SALT_REPO_USER: ${{ secrets.SALT_REPO_USER }}
          SALT_REPO_PASS: ${{ secrets.SALT_REPO_PASS }}
          SALT_REPO_DOMAIN_RELEASE: ${{ vars.SALT_REPO_DOMAIN_RELEASE || 'repo.saltproject.io' }}
          SALT_REPO_DOMAIN_STAGING: ${{ vars.SALT_REPO_DOMAIN_STAGING || 'staging.repo.saltproject.io' }}
          DOWNLOAD_TEST_PACKAGE_TYPE: ${{ matrix.pkg-type }}
        run: |
          sudo -E nox --force-color -e ${{ inputs.nox-session }}-pkgs -- download-pkgs

      - name: Fix file ownership
        run: |
          sudo chown -R "$(id -un)" .

      - name: Combine Coverage Reports
        if: always() && inputs.skip-code-coverage == false && job.status != 'cancelled'
        run: |
          nox --force-color -e combine-coverage

      - name: Prepare Test Run Artifacts
        id: download-artifacts-from-vm
        if: always() && job.status != 'cancelled'
        run: |
          # Delete the salt onedir, we won't need it anymore and it will prevent
          # from it showing in the tree command below
          rm -rf artifacts/salt*
          tree -a artifacts

      - name: Install Codecov CLI
        if: always() && inputs.skip-code-coverage == false && job.status != 'cancelled'
        run: |
          # We can't yet use tokenless uploads with the codecov CLI
          # python3 -m pip install codecov-cli
          #
          curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --import
          curl -Os https://uploader.codecov.io/latest/macos/codecov
          curl -Os https://uploader.codecov.io/latest/macos/codecov.SHA256SUM
          curl -Os https://uploader.codecov.io/latest/macos/codecov.SHA256SUM.sig
          gpg --verify codecov.SHA256SUM.sig codecov.SHA256SUM
          shasum -a 256 -c codecov.SHA256SUM
          chmod +x codecov

      - name: Upload Source Code Coverage To Codecov
        if: always() && inputs.skip-code-coverage == false && job.status != 'cancelled'
        run: |
          if [ ! -s artifacts/coverage/salt.xml ]; then
            echo "The artifacts/coverage/salt.xml file does not exist"
            exit 1
          fi
          # We can't yet use tokenless uploads with the codecov CLI
          #codecovcli --auto-load-params-from GithubActions --verbose --token ${{ secrets.CODECOV_TOKEN }} \
          #  do-upload --git-service github --sha ${{ github.sha }} \
          #  --file artifacts/coverage/salt.xml \
          #  --flag salt --flag ${{ matrix.distro-slug }} --flag pkg \
          #  --name salt.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs
          n=0
          until [ "$n" -ge 5 ]
          do
          if ./codecov --file artifacts/coverage/salt.xml \
              --sha ${{ github.event.pull_request.head.sha || github.sha }} ${{ github.event_name == 'pull_request' && format('--parent {0}', github.event.pull_request.base.sha) }} \
              --flags salt,${{ matrix.distro-slug }},pkg \
              --name salt.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs --nonZero; then
              rc=$?
              break
          fi
          rc=$?
          n=$((n+1))
          sleep 15
          done
          if [ "$rc" -ne 0 ]; then
            echo "Failed to upload codecov stats"
            exit 1
          fi

      - name: Upload Tests Code Coverage To Codecov
        if: always() && inputs.skip-code-coverage == false && job.status != 'cancelled'
        run: |
          if [ ! -s artifacts/coverage/tests.xml ]; then
            echo "The artifacts/coverage/tests.xml file does not exist"
            exit 1
          fi
          # We can't yet use tokenless uploads with the codecov CLI
          #codecovcli --auto-load-params-from GithubActions --verbose --token ${{ secrets.CODECOV_TOKEN }} \
          #  do-upload --git-service github --sha ${{ github.sha }} \
          #  --file artifacts/coverage/tests.xml \
          #  --flag tests --flag ${{ matrix.distro-slug }} --flag pkg \
          #  --name tests.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs
          n=0
          until [ "$n" -ge 5 ]
          do
          if ./codecov --file artifacts/coverage/tests.xml \
              --sha ${{ github.event.pull_request.head.sha || github.sha }} ${{ github.event_name == 'pull_request' && format('--parent {0}', github.event.pull_request.base.sha) }} \
              --flags tests,${{ matrix.distro-slug }},pkg \
              --name tests.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs --nonZero; then
              rc=$?
              break
          fi
          rc=$?
          n=$((n+1))
          sleep 15
          done
          if [ "$rc" -ne 0 ]; then
            echo "Failed to upload codecov stats"
            exit 1
          fi

      - name: Upload Test Run Artifacts
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: pkg-testrun-artifacts-${{ matrix.distro-slug }}-${{ matrix.arch }}-${{ matrix.pkg-type }}
          path: |
            artifacts
            !artifacts/salt/*
            !artifacts/salt-*.tar.*

  windows:
    name: Windows
    needs:
      - generate-matrix
    env:
      USE_S3_CACHE: 'true'
    runs-on:
      - self-hosted
      - linux
      - bastion
    environment: ${{ inputs.environment }}
    timeout-minutes: 120  # 2 Hours - More than this and something is wrong
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include)['windows'] }}

    steps:
      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: Download Onedir Tarball as an Artifact
        uses: actions/download-artifact@v4
        with:
          name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-windows-${{ matrix.arch }}.tar.xz
          path: artifacts/

      - name: Decompress Onedir Tarball
        shell: bash
        run: |
          python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
          cd artifacts
          tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-windows-${{ matrix.arch }}.tar.xz

      - name: Download nox.windows.${{ matrix.arch }}.tar.* artifact for session ${{ inputs.nox-session }}
        uses: actions/download-artifact@v4
        with:
          name: nox-windows-${{ matrix.arch }}-${{ inputs.nox-session }}

      - name: Setup Python Tools Scripts
        uses: ./.github/actions/setup-python-tools-scripts
        with:
          cache-prefix: ${{ inputs.cache-prefix }}-pkg-download-windows

      - name: Get Salt Project GitHub Actions Bot Environment
        run: |
          TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30")
          SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment)
          echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV"

      - name: Start VM
        id: spin-up-vm
        run: |
          tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ matrix.distro-slug }}

      - name: List Free Space
        run: |
          tools --timestamps vm ssh ${{ matrix.distro-slug }} -- df -h || true

      - name: Upload Checkout To VM
        run: |
          tools --timestamps vm rsync ${{ matrix.distro-slug }}

      - name: Decompress .nox Directory
        run: |
          tools --timestamps vm decompress-dependencies ${{ matrix.distro-slug }}

      - name: Show System Info
        run: |
          tools --timestamps --timeout-secs=1800 vm test --skip-requirements-install --print-system-information-only \
            --nox-session=${{ inputs.nox-session }}-pkgs ${{ matrix.distro-slug }} -- download-pkgs

      - name: Run Package Download Tests
        env:
          SALT_RELEASE: "${{ inputs.salt-version }}"
          SALT_REPO_ARCH: ${{ matrix.arch }}
          LATEST_SALT_RELEASE: "${{ inputs.latest-release }}"
          SALT_REPO_TYPE: ${{ inputs.environment }}
          SALT_REPO_USER: ${{ secrets.SALT_REPO_USER }}
          SALT_REPO_PASS: ${{ secrets.SALT_REPO_PASS }}
          SALT_REPO_DOMAIN_RELEASE: ${{ vars.SALT_REPO_DOMAIN_RELEASE || 'repo.saltproject.io' }}
          SALT_REPO_DOMAIN_STAGING: ${{ vars.SALT_REPO_DOMAIN_STAGING || 'staging.repo.saltproject.io' }}
          SKIP_CODE_COVERAGE: "${{ inputs.skip-code-coverage && '1' || '0' }}"
          DOWNLOAD_TEST_PACKAGE_TYPE: ${{ matrix.pkg-type }}
        run: |
          tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
          -E SALT_RELEASE -E SALT_REPO_ARCH -E SALT_REPO_TYPE -E SALT_REPO_USER -E SALT_REPO_PASS \
          -E SALT_REPO_DOMAIN_RELEASE -E SALT_REPO_DOMAIN_STAGING -E LATEST_SALT_RELEASE -E DOWNLOAD_TEST_PACKAGE_TYPE \
            --nox-session=${{ inputs.nox-session }}-pkgs --rerun-failures ${{ matrix.distro-slug }} -- download-pkgs

      - name: Combine Coverage Reports
        if: always() && inputs.skip-code-coverage == false && steps.spin-up-vm.outcome == 'success' && job.status != 'cancelled'
        run: |
          tools --timestamps vm combine-coverage ${{ matrix.distro-slug }}

      - name: Download Test Run Artifacts
        id: download-artifacts-from-vm
        if: always() && steps.spin-up-vm.outcome == 'success'
        run: |
          tools --timestamps vm download-artifacts ${{ matrix.distro-slug }}
          # Delete the salt onedir, we won't need it anymore and it will prevent
          # from it showing in the tree command below
          rm -rf artifacts/salt*
          tree -a artifacts

      - name: Destroy VM
        if: always()
        run: |
          tools --timestamps vm destroy --no-wait ${{ matrix.distro-slug }} || true

      - name: Fix file ownership
        run: |
          sudo chown -R "$(id -un)" .

      - name: Install Codecov CLI
        if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled'
        run: |
          # We can't yet use tokenless uploads with the codecov CLI
          # python3 -m pip install codecov-cli
          #
          curl https://keybase.io/codecovsecurity/pgp_keys.asc | gpg --no-default-keyring --import
          curl -Os https://uploader.codecov.io/latest/linux/codecov
          curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM
          curl -Os https://uploader.codecov.io/latest/linux/codecov.SHA256SUM.sig
          gpg --verify codecov.SHA256SUM.sig codecov.SHA256SUM
          shasum -a 256 -c codecov.SHA256SUM
          chmod +x codecov

      - name: Upload Source Code Coverage To Codecov
        if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled'
        run: |
          if [ ! -s artifacts/coverage/salt.xml ]; then
            echo "The artifacts/coverage/salt.xml file does not exist"
            exit 1
          fi
          # We can't yet use tokenless uploads with the codecov CLI
          #codecovcli --auto-load-params-from GithubActions --verbose --token ${{ secrets.CODECOV_TOKEN }} \
          #  do-upload --git-service github --sha ${{ github.sha }} \
          #  --file artifacts/coverage/salt.xml \
          #  --flag salt --flag ${{ matrix.distro-slug }} --flag pkg \
          #  --name salt.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs
          n=0
          until [ "$n" -ge 5 ]
          do
          if ./codecov --file artifacts/coverage/salt.xml \
              --sha ${{ github.event.pull_request.head.sha || github.sha }} ${{ github.event_name == 'pull_request' && format('--parent {0}', github.event.pull_request.base.sha) }} \
              --flags salt,${{ matrix.distro-slug }},pkg \
              --name salt.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs --nonZero; then
              rc=$?
              break
          fi
          rc=$?
          n=$((n+1))
          sleep 15
          done
          if [ "$rc" -ne 0 ]; then
            echo "Failed to upload codecov stats"
            exit 1
          fi

      - name: Upload Tests Code Coverage To Codecov
        if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled'
        run: |
          if [ ! -s artifacts/coverage/tests.xml ]; then
            echo "The artifacts/coverage/tests.xml file does not exist"
            exit 1
          fi
          # We can't yet use tokenless uploads with the codecov CLI
          #codecovcli --auto-load-params-from GithubActions --verbose --token ${{ secrets.CODECOV_TOKEN }} \
          #  do-upload --git-service github --sha ${{ github.sha }} \
          #  --file artifacts/coverage/tests.xml \
          #  --flag tests --flag ${{ matrix.distro-slug }} --flag pkg \
          #  --name tests.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs
          n=0
          until [ "$n" -ge 5 ]
          do
          if ./codecov --file artifacts/coverage/tests.xml \
              --sha ${{ github.event.pull_request.head.sha || github.sha }} ${{ github.event_name == 'pull_request' && format('--parent {0}', github.event.pull_request.base.sha) }} \
              --flags tests,${{ matrix.distro-slug }},pkg \
              --name tests.${{ matrix.distro-slug }}.${{ inputs.nox-session }}.download-pkgs --nonZero; then
              rc=$?
              break
          fi
          rc=$?
          n=$((n+1))
          sleep 15
          done
          if [ "$rc" -ne 0 ]; then
            echo "Failed to upload codecov stats"
            exit 1
          fi

      - name: Upload Test Run Artifacts
        if: always() && steps.download-artifacts-from-vm.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: pkg-testrun-artifacts-${{ matrix.distro-slug }}-${{ matrix.arch }}-${{ matrix.pkg-type }}
          path: |
            artifacts
            !artifacts/salt/*
            !artifacts/salt-*.tar.*


Test Artifact

---
name: Test Artifact

on:
  workflow_call:
    inputs:
      distro-slug:
        required: true
        type: string
        description: The OS slug to run tests against
      nox-session:
        required: true
        type: string
        description: The nox session to run
      testrun:
        required: true
        type: string
        description: JSON string containing information about what and how to run the test suite
      salt-version:
        type: string
        required: true
        description: The Salt version to set prior to running tests.
      cache-prefix:
        required: true
        type: string
        description: Seed used to invalidate caches
      platform:
        required: true
        type: string
        description: The platform being tested
      arch:
        required: true
        type: string
        description: The platform arch being tested
      nox-version:
        required: true
        type: string
        description: The nox version to install
      timeout-minutes:
        required: true
        type: number
        description: Timeout, in minutes, for the test job
      gh-actions-python-version:
        required: false
        type: string
        description: The python version to run tests with
        default: "3.10"
      fips:
        required: false
        type: boolean
        default: false
        description: Test run with FIPS enabled
      package-name:
        required: false
        type: string
        description: The onedir package name to use
        default: salt
      skip-code-coverage:
        required: false
        type: boolean
        description: Skip code coverage
        default: false
      workflow-slug:
        required: false
        type: string
        description: Which workflow is running.
        default: ci

env:
  COLUMNS: 190
  AWS_MAX_ATTEMPTS: "10"
  AWS_RETRY_MODE: "adaptive"
  PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/
  PIP_EXTRA_INDEX_URL: https://pypi.org/simple
  PIP_DISABLE_PIP_VERSION_CHECK: "1"
  RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"

jobs:

  generate-matrix:
    name: Test Matrix
    runs-on: ubuntu-latest
    outputs:
      matrix-include: ${{ steps.generate-matrix.outputs.matrix }}
      build-reports: ${{ steps.generate-matrix.outputs.build-reports }}
    steps:

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t"

      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: Setup Python Tools Scripts
        uses: ./.github/actions/setup-python-tools-scripts
        with:
          cache-prefix: ${{ inputs.cache-prefix }}

      - name: Generate Test Matrix
        id: generate-matrix
        run: |
          tools ci matrix --workflow=${{ inputs.workflow-slug }} ${{ fromJSON(inputs.testrun)['type'] == 'full' && '--full ' || '' }}${{ inputs.distro-slug }}

  test:
    name: Test
    runs-on:
      - self-hosted
      - linux
      - bastion
    timeout-minutes: ${{ inputs.timeout-minutes }}
    needs:
      - generate-matrix
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include) }}
    env:
      SALT_TRANSPORT: ${{ matrix.transport }}
      TEST_GROUP: ${{ matrix.test-group || 1 }}

    steps:

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t"

      - name: "Set `TIMESTAMP` environment variable"
        shell: bash
        run: |
          echo "TIMESTAMP=$(date +%s)" | tee -a "$GITHUB_ENV"

      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: Setup Salt Version
        run: |
          echo "${{ inputs.salt-version }}" > salt/_version.txt

      - name: Download Onedir Tarball as an Artifact
        uses: actions/download-artifact@v4
        with:
          name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
          path: artifacts/

      - name: Decompress Onedir Tarball
        shell: bash
        run: |
          python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
          cd artifacts
          tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz

      - name: Download nox.linux.${{ inputs.arch }}.tar.* artifact for session ${{ inputs.nox-session }}
        uses: actions/download-artifact@v4
        with:
          name: nox-linux-${{ inputs.arch }}-${{ inputs.nox-session }}

      - name: PyPi Proxy
        run: |
          sed -i '7s;^;--index-url=https://pypi-proxy.saltstack.net/root/local/+simple/ --extra-index-url=https://pypi.org/simple\n;' requirements/static/ci/*/*.txt

      - name: Setup Python Tools Scripts
        uses: ./.github/actions/setup-python-tools-scripts
        with:
          cache-prefix: ${{ inputs.cache-prefix }}

      - name: Download testrun-changed-files.txt
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }}
        uses: actions/download-artifact@v4
        with:
          name: testrun-changed-files.txt

      - name: Get Salt Project GitHub Actions Bot Environment
        run: |
          TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30")
          SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment)
          echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV"

      - name: Start VM
        id: spin-up-vm
        env:
          TESTS_CHUNK: ${{ matrix.tests-chunk }}
        run: |
          tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ inputs.distro-slug }}

      - name: List Free Space
        run: |
          tools --timestamps vm ssh ${{ inputs.distro-slug }} -- df -h || true

      - name: Upload Checkout To VM
        run: |
          tools --timestamps vm rsync ${{ inputs.distro-slug }}

      - name: Decompress .nox Directory
        run: |
          tools --timestamps vm decompress-dependencies ${{ inputs.distro-slug }}

      - name: Show System Info
        run: |
          tools --timestamps --timeout-secs=1800 vm test --skip-requirements-install --print-system-information-only \
            --nox-session=${{ inputs.nox-session }} ${{ inputs.distro-slug }} \
            ${{ matrix.tests-chunk }}

      - name: Run Changed Tests
        id: run-fast-changed-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }}
        run: |
          tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
            --nox-session=${{ inputs.nox-session }} --rerun-failures -E SALT_TRANSPORT ${{ inputs.fips && '--fips ' || '' }}${{ inputs.distro-slug }} \
            ${{ matrix.tests-chunk }} -- --core-tests --slow-tests --suppress-no-test-exit-code \
            --from-filenames=testrun-changed-files.txt

      - name: Run Fast Tests
        id: run-fast-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['fast'] }}
        run: |
          tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
            --nox-session=${{ inputs.nox-session }} --rerun-failures -E SALT_TRANSPORT ${{ (inputs.skip-code-coverage && matrix.tests-chunk != 'unit') && '--skip-code-coverage' || '' }} \
            ${{ inputs.fips && '--fips ' || '' }}${{ inputs.distro-slug }} ${{ matrix.tests-chunk }}

      - name: Run Slow Tests
        id: run-slow-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['slow'] }}
        run: |
          tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
            --nox-session=${{ inputs.nox-session }} --rerun-failures -E SALT_TRANSPORT ${{ inputs.fips && '--fips ' || '' }}${{ inputs.distro-slug }} \
            ${{ matrix.tests-chunk }} -- --no-fast-tests --slow-tests

      - name: Run Core Tests
        id: run-core-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['core'] }}
        run: |
          tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
            --nox-session=${{ inputs.nox-session }} --rerun-failures -E SALT_TRANSPORT ${{ inputs.fips && '--fips ' || '' }}${{ inputs.distro-slug }} \
            ${{ matrix.tests-chunk }} -- --no-fast-tests --core-tests

      - name: Run Flaky Tests
        id: run-flaky-tests
        if: ${{ fromJSON(inputs.testrun)['selected_tests']['flaky'] }}
        run: |
          tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
            --nox-session=${{ inputs.nox-session }} --rerun-failures -E SALT_TRANSPORT ${{ inputs.fips && '--fips ' || '' }}${{ inputs.distro-slug }} \
            ${{ matrix.tests-chunk }} -- --no-fast-tests --flaky-jail

      - name: Run Full Tests
        id: run-full-tests
        if: ${{ fromJSON(inputs.testrun)['type'] == 'full' }}
        run: |
          tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
            --nox-session=${{ inputs.nox-session }} --rerun-failures -E SALT_TRANSPORT ${{ (inputs.skip-code-coverage && matrix.tests-chunk != 'unit') && '--skip-code-coverage' || '' }} \
            -E TEST_GROUP ${{ inputs.fips && '--fips ' || '' }}${{ inputs.distro-slug }} ${{ matrix.tests-chunk }} -- --slow-tests --core-tests \
            --test-group-count=${{ matrix.test-group-count || 1 }} --test-group=${{ matrix.test-group || 1 }}

      - name: Combine Coverage Reports
        if: always() && inputs.skip-code-coverage == false && steps.spin-up-vm.outcome == 'success'
        run: |
          tools --timestamps vm combine-coverage ${{ inputs.distro-slug }}

      - name: Download Test Run Artifacts
        id: download-artifacts-from-vm
        if: always() && steps.spin-up-vm.outcome == 'success'
        run: |
          tools --timestamps vm download-artifacts ${{ inputs.distro-slug }}
          # Delete the salt onedir, we won't need it anymore and it will prevent
          # from it showing in the tree command below
          rm -rf artifacts/salt*
          tree -a artifacts
          if [ "${{ inputs.skip-code-coverage }}" != "true" ]; then
            mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ inputs.distro-slug }}${{ inputs.fips && '.fips' || '' }}.${{ inputs.nox-session }}.${{ matrix.transport }}.${{ matrix.tests-chunk }}.grp${{ matrix.test-group || '1' }}
          fi

      - name: Destroy VM
        if: always()
        run: |
          tools --timestamps vm destroy --no-wait ${{ inputs.distro-slug }} || true

      - name: Upload Code Coverage Test Run Artifacts
        if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled'
        uses: actions/upload-artifact@v4
        with:
          name: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }}-${{ env.TIMESTAMP }}
          path: |
            artifacts/coverage/

      - name: Upload JUnit XML Test Run Artifacts
        if: always() && steps.download-artifacts-from-vm.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: testrun-junit-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }}-${{ env.TIMESTAMP }}
          path: |
            artifacts/xml-unittests-output/

      - name: Upload Test Run Log Artifacts
        if: always() && steps.download-artifacts-from-vm.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: testrun-log-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }}-${{ env.TIMESTAMP }}
          path: |
            artifacts/logs

  report:
    name: Test Reports
    if: always() && fromJSON(needs.generate-matrix.outputs.build-reports) && needs.test.result != 'cancelled' && needs.test.result != 'skipped'
    runs-on: ubuntu-latest
    needs:
      - test
      - generate-matrix

    steps:
      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t"

      - name: Merge JUnit XML Test Run Artifacts
        if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped'
        continue-on-error: true
        uses: actions/upload-artifact/merge@v4
        with:
          name: testrun-junit-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}
          pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-*
          separate-directories: false
          delete-merged: true

      - name: Merge Log Test Run Artifacts
        if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped'
        continue-on-error: true
        uses: actions/upload-artifact/merge@v4
        with:
          name: testrun-log-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}
          pattern: testrun-log-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-*
          separate-directories: false
          delete-merged: true

      - name: Merge Code Coverage Test Run Artifacts
        if: ${{ inputs.skip-code-coverage == false }}
        continue-on-error: true
        uses: actions/upload-artifact/merge@v4
        with:
          name: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}
          pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}-*
          separate-directories: false
          delete-merged: true

      - name: Download Code Coverage Test Run Artifacts
        uses: actions/download-artifact@v4
        if: ${{ inputs.skip-code-coverage == false }}
        id: download-coverage-artifacts
        with:
          name: testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}
          path: artifacts/coverage/

      - name: Show Downloaded Test Run Artifacts
        if: ${{ inputs.skip-code-coverage == false }}
        run: |
          tree -a artifacts

      - name: Install Nox
        run: |
          python3 -m pip install 'nox==${{ inputs.nox-version }}'

      - name: Create XML Coverage Reports
        if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success' && job.status != 'cancelled'
        run: |
          nox --force-color -e create-xml-coverage-reports
          mv artifacts/coverage/salt.xml artifacts/coverage/salt..${{ inputs.distro-slug }}${{ inputs.fips && '..fips' || '' }}..${{ inputs.nox-session }}.xml
          mv artifacts/coverage/tests.xml artifacts/coverage/tests..${{ inputs.distro-slug }}${{ inputs.fips && '..fips' || '' }}..${{ inputs.nox-session }}.xml

      - name: Report Salt Code Coverage
        if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success'
        continue-on-error: true
        run: |
          nox --force-color -e report-coverage -- salt

      - name: Report Combined Code Coverage
        if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success'
        continue-on-error: true
        run: |
          nox --force-color -e report-coverage

      - name: Rename Code Coverage DB
        if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success'
        continue-on-error: true
        run: |
          mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ inputs.distro-slug }}${{ inputs.fips && '.fips' || '' }}.${{ inputs.nox-session }}

      - name: Upload Code Coverage DB
        if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: all-testrun-coverage-artifacts-${{ inputs.distro-slug }}${{ inputs.fips && '-fips' || '' }}-${{ inputs.nox-session }}
          path: artifacts/coverage


---
name: Test Artifact

on:
  workflow_call:
    inputs:
      distro-slug:
        required: true
        type: string
        description: The OS slug to run tests against
      nox-session:
        required: true
        type: string
        description: The nox session to run
      testrun:
        required: true
        type: string
        description: JSON string containing information about what and how to run the test suite
      salt-version:
        type: string
        required: true
        description: The Salt version to set prior to running tests.
      cache-prefix:
        required: true
        type: string
        description: Seed used to invalidate caches
      platform:
        required: true
        type: string
        description: The platform being tested
      arch:
        required: true
        type: string
        description: The platform arch being tested
      nox-version:
        required: true
        type: string
        description: The nox version to install
      timeout-minutes:
        required: true
        type: number
        description: Timeout, in minutes, for the test job
      gh-actions-python-version:
        required: false
        type: string
        description: The python version to run tests with
        default: "3.10"
      fips:
        required: false
        type: boolean
        default: false
        description: Test run with FIPS enabled
      package-name:
        required: false
        type: string
        description: The onedir package name to use
        default: salt
      skip-code-coverage:
        required: false
        type: boolean
        description: Skip code coverage
        default: false
      workflow-slug:
        required: false
        type: string
        description: Which workflow is running.
        default: ci

env:
  COLUMNS: 190
  AWS_MAX_ATTEMPTS: "10"
  AWS_RETRY_MODE: "adaptive"
  PIP_INDEX_URL: https://pypi-proxy.saltstack.net/root/local/+simple/
  PIP_EXTRA_INDEX_URL: https://pypi.org/simple
  PIP_DISABLE_PIP_VERSION_CHECK: "1"
  RAISE_DEPRECATIONS_RUNTIME_ERRORS: "1"

jobs:

  generate-matrix:
    name: Test Matrix
    runs-on: ubuntu-latest
    outputs:
      matrix-include: ${{ steps.generate-matrix.outputs.matrix }}
      build-reports: ${{ steps.generate-matrix.outputs.build-reports }}
    steps:

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t"

      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: Setup Python Tools Scripts
        uses: ./.github/actions/setup-python-tools-scripts
        with:
          cache-prefix: ${{ inputs.cache-prefix }}

      - name: Generate Test Matrix
        id: generate-matrix
        run: |
          tools ci matrix --workflow=${{ inputs.workflow-slug }} ${{ fromJSON(inputs.testrun)['type'] == 'full' && '--full ' || '' }}${{ inputs.fips && '--fips ' || '' }}${{ inputs.distro-slug }}

  test:
    name: Test
    runs-on:
      - self-hosted
      - linux
      - bastion
    timeout-minutes: ${{ inputs.timeout-minutes }}
    needs:
      - generate-matrix
    strategy:
      fail-fast: false
      matrix:
        include: ${{ fromJSON(needs.generate-matrix.outputs.matrix-include) }}
    env:
      SALT_TRANSPORT: ${{ matrix.transport }}
      TEST_GROUP: ${{ matrix.test-group || 1 }}

    steps:

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(python3 -c 'import random, sys; sys.stdout.write(str(random.randint(1, 15)))'); echo "Sleeping $t seconds"; sleep "$t"

      - name: "Set `TIMESTAMP` environment variable"
        shell: bash
        run: |
          echo "TIMESTAMP=$(date +%s)" | tee -a "$GITHUB_ENV"

      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: Setup Salt Version
        run: |
          echo "${{ inputs.salt-version }}" > salt/_version.txt

      - name: Download Onedir Tarball as an Artifact
        uses: actions/download-artifact@v4
        with:
          name: ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz
          path: artifacts/

      - name: Decompress Onedir Tarball
        shell: bash
        run: |
          python3 -c "import os; os.makedirs('artifacts', exist_ok=True)"
          cd artifacts
          tar xvf ${{ inputs.package-name }}-${{ inputs.salt-version }}-onedir-${{ inputs.platform }}-${{ inputs.arch }}.tar.xz

      - name: Download nox.windows.${{ inputs.arch }}.tar.* artifact for session ${{ inputs.nox-session }}
        uses: actions/download-artifact@v4
        with:
          name: nox-windows-${{ inputs.arch }}-${{ inputs.nox-session }}

      - name: PyPi Proxy
        run: |
          sed -i '7s;^;--index-url=https://pypi-proxy.saltstack.net/root/local/+simple/ --extra-index-url=https://pypi.org/simple\n;' requirements/static/ci/*/*.txt

      - name: Setup Python Tools Scripts
        uses: ./.github/actions/setup-python-tools-scripts
        with:
          cache-prefix: ${{ inputs.cache-prefix }}

      - name: Download testrun-changed-files.txt
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }}
        uses: actions/download-artifact@v4
        with:
          name: testrun-changed-files.txt

      - name: Get Salt Project GitHub Actions Bot Environment
        run: |
          TOKEN=$(curl -sS -f -X PUT "http://169.254.169.254/latest/api/token" -H "X-aws-ec2-metadata-token-ttl-seconds: 30")
          SPB_ENVIRONMENT=$(curl -sS -f -H "X-aws-ec2-metadata-token: $TOKEN" http://169.254.169.254/latest/meta-data/tags/instance/spb:environment)
          echo "SPB_ENVIRONMENT=$SPB_ENVIRONMENT" >> "$GITHUB_ENV"

      - name: Start VM
        id: spin-up-vm
        env:
          TESTS_CHUNK: ${{ matrix.tests-chunk }}
        run: |
          tools --timestamps vm create --environment "${SPB_ENVIRONMENT}" --retries=2 ${{ inputs.distro-slug }}

      - name: List Free Space
        run: |
          tools --timestamps vm ssh ${{ inputs.distro-slug }} -- df -h || true

      - name: Upload Checkout To VM
        run: |
          tools --timestamps vm rsync ${{ inputs.distro-slug }}

      - name: Decompress .nox Directory
        run: |
          tools --timestamps vm decompress-dependencies ${{ inputs.distro-slug }}

      - name: Show System Info
        run: |
          tools --timestamps --timeout-secs=1800 vm test --skip-requirements-install --print-system-information-only \
            --nox-session=${{ inputs.nox-session }} ${{ inputs.distro-slug }} \
            ${{ matrix.tests-chunk }}

      - name: Run Changed Tests
        id: run-fast-changed-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' }}
        run: |
          tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
            --nox-session=${{ inputs.nox-session }} --rerun-failures -E SALT_TRANSPORT ${{ matrix.fips && '--fips ' || '' }}${{ inputs.distro-slug }} \
            ${{ matrix.tests-chunk }} -- --core-tests --slow-tests --suppress-no-test-exit-code \
            --from-filenames=testrun-changed-files.txt

      - name: Run Fast Tests
        id: run-fast-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['fast'] }}
        run: |
          tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
            --nox-session=${{ inputs.nox-session }} --rerun-failures -E SALT_TRANSPORT ${{ (inputs.skip-code-coverage && matrix.tests-chunk != 'unit') && '--skip-code-coverage' || '' }} \
            ${{ matrix.fips && '--fips ' || '' }}${{ inputs.distro-slug }} ${{ matrix.tests-chunk }}

      - name: Run Slow Tests
        id: run-slow-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['slow'] }}
        run: |
          tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
            --nox-session=${{ inputs.nox-session }} --rerun-failures -E SALT_TRANSPORT ${{ matrix.fips && '--fips ' || '' }}${{ inputs.distro-slug }} \
            ${{ matrix.tests-chunk }} -- --no-fast-tests --slow-tests

      - name: Run Core Tests
        id: run-core-tests
        if: ${{ fromJSON(inputs.testrun)['type'] != 'full' && fromJSON(inputs.testrun)['selected_tests']['core'] }}
        run: |
          tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
            --nox-session=${{ inputs.nox-session }} --rerun-failures -E SALT_TRANSPORT ${{ matrix.fips && '--fips ' || '' }}${{ inputs.distro-slug }} \
            ${{ matrix.tests-chunk }} -- --no-fast-tests --core-tests

      - name: Run Flaky Tests
        id: run-flaky-tests
        if: ${{ fromJSON(inputs.testrun)['selected_tests']['flaky'] }}
        run: |
          tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
            --nox-session=${{ inputs.nox-session }} --rerun-failures -E SALT_TRANSPORT ${{ matrix.fips && '--fips ' || '' }}${{ inputs.distro-slug }} \
            ${{ matrix.tests-chunk }} -- --no-fast-tests --flaky-jail

      - name: Run Full Tests
        id: run-full-tests
        if: ${{ fromJSON(inputs.testrun)['type'] == 'full' }}
        run: |
          tools --timestamps --no-output-timeout-secs=1800 --timeout-secs=14400 vm test --skip-requirements-install \
            --nox-session=${{ inputs.nox-session }} --rerun-failures -E SALT_TRANSPORT ${{ (inputs.skip-code-coverage && matrix.tests-chunk != 'unit') && '--skip-code-coverage' || '' }} \
            -E TEST_GROUP ${{ matrix.fips && '--fips ' || '' }}${{ inputs.distro-slug }} ${{ matrix.tests-chunk }} -- --slow-tests --core-tests \
            --test-group-count=${{ matrix.test-group-count || 1 }} --test-group=${{ matrix.test-group || 1 }}

      - name: Combine Coverage Reports
        if: always() && inputs.skip-code-coverage == false && steps.spin-up-vm.outcome == 'success'
        run: |
          tools --timestamps vm combine-coverage ${{ inputs.distro-slug }}

      - name: Download Test Run Artifacts
        id: download-artifacts-from-vm
        if: always() && steps.spin-up-vm.outcome == 'success'
        run: |
          tools --timestamps vm download-artifacts ${{ inputs.distro-slug }}
          # Delete the salt onedir, we won't need it anymore and it will prevent
          # from it showing in the tree command below
          rm -rf artifacts/salt*
          tree -a artifacts
          if [ "${{ inputs.skip-code-coverage }}" != "true" ]; then
            mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ inputs.distro-slug }}.${{ inputs.nox-session }}.${{ matrix.transport }}.${{ matrix.tests-chunk }}.grp${{ matrix.test-group || '1' }}
          fi

      - name: Destroy VM
        if: always()
        run: |
          tools --timestamps vm destroy --no-wait ${{ inputs.distro-slug }} || true

      - name: Upload Code Coverage Test Run Artifacts
        if: always() && inputs.skip-code-coverage == false && steps.download-artifacts-from-vm.outcome == 'success' && job.status != 'cancelled'
        uses: actions/upload-artifact@v4
        with:
          name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }}-${{ env.TIMESTAMP }}
          path: |
            artifacts/coverage/

      - name: Upload JUnit XML Test Run Artifacts
        if: always() && steps.download-artifacts-from-vm.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }}-${{ env.TIMESTAMP }}
          path: |
            artifacts/xml-unittests-output/

      - name: Upload Test Run Log Artifacts
        if: always() && steps.download-artifacts-from-vm.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-${{ matrix.transport }}-${{ matrix.tests-chunk }}-grp${{ matrix.test-group || '1' }}-${{ env.TIMESTAMP }}
          path: |
            artifacts/logs

  report:
    name: Test Reports
    if: always() && fromJSON(needs.generate-matrix.outputs.build-reports) && needs.test.result != 'cancelled' && needs.test.result != 'skipped'
    runs-on: ubuntu-latest
    needs:
      - test
      - generate-matrix

    steps:
      - name: Checkout Source Code
        uses: actions/checkout@v4

      - name: "Throttle Builds"
        shell: bash
        run: |
          t=$(shuf -i 1-30 -n 1); echo "Sleeping $t seconds"; sleep "$t"

      - name: Merge JUnit XML Test Run Artifacts
        if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped'
        continue-on-error: true
        uses: actions/upload-artifact/merge@v4
        with:
          name: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}
          pattern: testrun-junit-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-*
          separate-directories: false
          delete-merged: true

      - name: Merge Log Test Run Artifacts
        if: always() && needs.test.result != 'cancelled' && needs.test.result != 'skipped'
        continue-on-error: true
        uses: actions/upload-artifact/merge@v4
        with:
          name: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}
          pattern: testrun-log-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-*
          separate-directories: false
          delete-merged: true

      - name: Merge Code Coverage Test Run Artifacts
        if: ${{ inputs.skip-code-coverage == false }}
        continue-on-error: true
        uses: actions/upload-artifact/merge@v4
        with:
          name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}
          pattern: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}-*
          separate-directories: false
          delete-merged: true

      - name: Download Code Coverage Test Run Artifacts
        uses: actions/download-artifact@v4
        if: ${{ inputs.skip-code-coverage == false }}
        id: download-coverage-artifacts
        with:
          name: testrun-coverage-artifacts-${{ inputs.distro-slug }}-${{ inputs.nox-session }}
          path: artifacts/coverage/

      - name: Show Downloaded Test Run Artifacts
        if: ${{ inputs.skip-code-coverage == false }}
        run: |
          tree -a artifacts

      - name: Install Nox
        run: |
          python3 -m pip install 'nox==${{ inputs.nox-version }}'

      - name: Create XML Coverage Reports
        if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success' && job.status != 'cancelled'
        run: |
          nox --force-color -e create-xml-coverage-reports
          mv artifacts/coverage/salt.xml artifacts/coverage/salt..${{ inputs.distro-slug }}..${{ inputs.nox-session }}.xml
          mv artifacts/coverage/tests.xml artifacts/coverage/tests..${{ inputs.distro-slug }}..${{ inputs.nox-session }}.xml

      - name: Report Salt Code Coverage
        if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success'
        continue-on-error: true
        run: |
          nox --force-color -e report-coverage -- salt

      - name: Report Combined Code Coverage
        if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success'
        continue-on-error: true
        run: |
          nox --force-color -e report-coverage

      - name: Rename Code Coverage DB
        if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success'
        continue-on-error: true
        run: |
          mv artifacts/coverage/.coverage artifacts/coverage/.coverage.${{ inputs.distro-slug }}.${{ inputs.nox-session }}

      - name: Upload Code Coverage DB
        if: always() && inputs.skip-code-coverage == false && steps.download-coverage-artifacts.outcome == 'success'
        uses: actions/upload-artifact@v4
        with:
          name: all-testrun-coverage-artifacts-${{ inputs.distro-slug }}.${{ inputs.nox-session }}
          path: artifacts/coverage