Container Scan

name: Container Scan

on:
  workflow_call:

  workflow_dispatch:
    inputs:
      none:
        description: "Run Tests Manually"
        required: false

jobs:
  scan-backend-trivy:
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      # free 10GB of space
      - name: Remove unnecessary files
        if: matrix.os == 'ubuntu-latest'
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
          docker image prune --all --force
          docker builder prune --all --force
          docker system prune --all --force

      # Build the docker image for testing
      - name: Build a Docker image
        run: DOCKER_BUILDKIT=1 docker build -f packages/grid/backend/backend.dockerfile packages -t backend:${{ github.sha }} --no-cache
        # Runs Snyk Container (Container and SCA) analysis and uploads result to Snyk.

      - name: Run Trivy vulnerability scanner
        continue-on-error: true
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: "backend:${{ github.sha }}"
          format: "template"
          template: "@/contrib/sarif.tpl"
          output: "trivy-results.sarif"
          severity: "CRITICAL,HIGH"
          timeout: "10m0s"

      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: "trivy-results.sarif"

  scan-backend-snyk:
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Snyk CLI to check for security issues
        # Snyk can be used to break the build when it detects security issues.
        # In this case we want to upload the SAST issues to GitHub Code Scanning
        uses: snyk/actions/setup@master
        env:
          # This is where you will need to introduce the Snyk API token created with your Snyk account
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}

      # free 10GB of space
      - name: Remove unnecessary files
        if: matrix.os == 'ubuntu-latest'
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
          docker image prune --all --force
          docker builder prune --all --force
          docker system prune --all --force

      # Build the docker image for testing
      - name: Build a Docker image
        shell: bash
        run: DOCKER_BUILDKIT=1 docker build -f packages/grid/backend/backend.dockerfile packages -t backend:${{ github.sha }} --no-cache
        # Runs Snyk Container (Container and SCA) analysis and uploads result to Snyk.

      - name: Snyk auth
        shell: bash
        run: snyk config set api=$SNYK_TOKEN
        env:
          # This is where you will need to introduce the Snyk API token created with your Snyk account
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}

      - name: Snyk Container test
        continue-on-error: true
        shell: bash
        run: snyk container test backend:${{ github.sha }} --file=packages/grid/backend/backend.dockerfile --sarif --sarif-file-output=snyk-code.sarif
        env:
          # This is where you will need to introduce the Snyk API token created with your Snyk account
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}

        # Push the Snyk Code results into GitHub Code Scanning tab
      - name: Upload result to GitHub Code Scanning
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: snyk-code.sarif

  scan-frontend-trivy:
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      # Build the docker image for testing
      - name: Build a Docker image
        shell: bash
        run: DOCKER_BUILDKIT=1 docker build -f packages/grid/frontend/frontend.dockerfile packages/grid/frontend -t frontend:${{ github.sha }} --no-cache
        # Runs Snyk Container (Container and SCA) analysis and uploads result to Snyk.

      - name: Run Trivy vulnerability scanner
        continue-on-error: true
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: "frontend:${{ github.sha }}"
          format: "template"
          template: "@/contrib/sarif.tpl"
          output: "trivy-results.sarif"
          severity: "CRITICAL,HIGH"
          timeout: "10m0s"

      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: "trivy-results.sarif"

  scan-frontend-snyk:
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Snyk CLI to check for security issues
        # Snyk can be used to break the build when it detects security issues.
        # In this case we want to upload the SAST issues to GitHub Code Scanning
        uses: snyk/actions/setup@master
        env:
          # This is where you will need to introduce the Snyk API token created with your Snyk account
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}

      # Build the docker image for testing
      - name: Build a Docker image
        shell: bash
        run: DOCKER_BUILDKIT=1 docker build -f packages/grid/frontend/frontend.dockerfile packages/grid/frontend -t frontend:${{ github.sha }} --no-cache
        # Runs Snyk Container (Container and SCA) analysis and uploads result to Snyk.

      - name: Snyk auth
        shell: bash
        run: snyk config set api=$SNYK_TOKEN
        env:
          # This is where you will need to introduce the Snyk API token created with your Snyk account
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}

      - name: Snyk Container test
        continue-on-error: true
        shell: bash
        run: snyk container test frontend:${{ github.sha }} --file=packages/grid/frontend/frontend.dockerfile --sarif --sarif-file-output=snyk-code.sarif
        env:
          # This is where you will need to introduce the Snyk API token created with your Snyk account
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}

        # Push the Snyk Code results into GitHub Code Scanning tab
      - name: Upload result to GitHub Code Scanning
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: snyk-code.sarif

  scan-syft-requirements:
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      #Generate SBOM
      - name: Generate SBOM
        run: |
          python3 --version
          python3 -m pip install ./packages/syft
          python3 -m pip install cyclonedx-bom
          python3 -m pip freeze > requirements.txt
          cyclonedx-py requirements -o syft.sbom.json

        #Trivy scan SBOM
      - name: Run Trivy vulnerability scanner
        continue-on-error: true
        run: |
          sudo apt-get install wget apt-transport-https gnupg lsb-release
          wget -qO - https://aquasecurity.github.io/trivy-repo/deb/public.key | sudo apt-key add -
          echo deb https://aquasecurity.github.io/trivy-repo/deb $(lsb_release -sc) main | sudo tee -a /etc/apt/sources.list.d/trivy.list
          sudo apt-get update
          sudo apt-get install trivy
          trivy sbom syft.sbom.json --format sarif --output trivy-results.sarif --severity CRITICAL,HIGH --timeout 10m0s

        #Upload SBOM to GitHub Security tab
      - name: Upload SBOM to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: "trivy-results.sarif"

          #upload SBOM to github artifacts
      - name: Upload SBOM to GitHub Artifacts
        uses: actions/upload-artifact@v4
        with:
          name: syft.sbom.json
          path: syft.sbom.json

  scan-mongo-latest-trivy:
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      #Generate SBOM
      - name: Run Trivy vulnerability scanner
        continue-on-error: true
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: "mongo:7.0.0"
          format: "cyclonedx"
          output: "mongo-trivy-results.sbom.json"
          timeout: "10m0s"

        #Upload SBOM to GitHub Artifacts
      - name: Upload SBOM to GitHub Artifacts
        uses: actions/upload-artifact@v4
        with:
          name: mongo-trivy-results.sbom.json
          path: mongo-trivy-results.sbom.json

          #Generate sarif file
      - name: Run Trivy vulnerability scanner
        continue-on-error: true
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: "mongo:7.0.0"
          format: "sarif"
          output: "trivy-results.sarif"
          timeout: "10m0s"

          #Upload sarif file to GitHub Security tab
      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: "trivy-results.sarif"

  scan-mongo-latest-snyk:
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Snyk Container test
        uses: snyk/actions/docker@master
        continue-on-error: true
        env:
          # This is where you will need to introduce the Snyk API token created with your Snyk account
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
        with:
          image: mongo:7.0.0
          args: --sarif-file-output=snyk-code.sarif

        # Replace any "undefined" security severity values with 0. The undefined value is used in the case
        # of license-related findings, which do not do not indicate a security vulnerability.
        # See https://github.com/github/codeql-action/issues/2187 for more context.
      - name: Post-process sarif output
        run: |
          sed -i 's/"security-severity": "undefined"/"security-severity": "0"/g' snyk-code.sarif

        # Replace any "null" security severity values with 0. The undefined value is used in the case
        # the NVD CVSS Score is not available.
        # See  https://github.com/Erikvl87/docker-languagetool/issues/90 and https://github.com/github/codeql-action/issues/2187 for more context.
      - name: Post-process sarif output for security severities set to "null"
        run: |
          sed -i 's/"security-severity": "null"/"security-severity": "0"/g' snyk-code.sarif

      - name: Upload result to GitHub Code Scanning
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: snyk-code.sarif

  scan-traefik-trivy:
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      #Generate SBOM
      - name: Run Trivy vulnerability scanner
        continue-on-error: true
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: "traefik:v2.11.0"
          format: "cyclonedx"
          output: "traefik-trivy-results.sbom.json"
          timeout: "10m0s"
          #Upload SBOM to GitHub Artifacts
      - name: Upload SBOM to GitHub Artifacts
        uses: actions/upload-artifact@v4
        with:
          name: traefik-trivy-results.sbom.json
          path: traefik-trivy-results.sbom.json

        #Generate sarif file
      - name: Run Trivy vulnerability scanner
        continue-on-error: true
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: "traefik:v2.11.0"
          format: "sarif"
          output: "trivy-results.sarif"
          severity: "CRITICAL,HIGH"
          timeout: "10m0s"

          #Upload sarif file to GitHub Security tab
      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: "trivy-results.sarif"

  scan-traefik-snyk:
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    runs-on: ubuntu-latest
    steps:
      - name: Snyk Container test
        uses: snyk/actions/docker@master
        continue-on-error: true
        env:
          # This is where you will need to introduce the Snyk API token created with your Snyk account
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
        with:
          image: traefik:v2.11.0
          args: --sarif-file-output=snyk-code.sarif

        # Replace any "undefined" security severity values with 0. The undefined value is used in the case
        # of license-related findings, which do not do not indicate a security vulnerability.
        # See https://github.com/github/codeql-action/issues/2187 for more context.
      - name: Post-process sarif output
        run: |
          sed -i 's/"security-severity": "undefined"/"security-severity": "0"/g' snyk-code.sarif

        # Replace any "null" security severity values with 0. The undefined value is used in the case
        # the NVD CVSS Score is not available.
        # See  https://github.com/Erikvl87/docker-languagetool/issues/90 and https://github.com/github/codeql-action/issues/2187 for more context.
      - name: Post-process sarif output for security severities set to "null"
        run: |
          sed -i 's/"security-severity": "null"/"security-severity": "0"/g' snyk-code.sarif

        # Push the Snyk Code results into GitHub Code Scanning tab
      - name: Upload result to GitHub Code Scanning
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: snyk-code.sarif

  scan-seaweedfs-trivy:
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Remove unnecessary files
        if: matrix.os == 'ubuntu-latest'
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
          docker image prune --all --force
          docker builder prune --all --force
          docker system prune --all --force

      # Build the docker image for testing
      - name: Build a Docker image
        run: DOCKER_BUILDKIT=1 docker build -f packages/grid/seaweedfs/seaweedfs.dockerfile ./packages/grid/seaweedfs -t seaweedfs:${{ github.sha }} --no-cache
        # Runs Snyk Container (Container and SCA) analysis and uploads result to Snyk.

      - name: Run Trivy vulnerability scanner
        continue-on-error: true
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: "seaweedfs:${{ github.sha }}"
          format: "template"
          template: "@/contrib/sarif.tpl"
          output: "trivy-results.sarif"
          severity: "CRITICAL,HIGH"
          timeout: "10m0s"

      - name: Upload Trivy scan results to GitHub Security tab
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: "trivy-results.sarif"

  scan-seaweedfs-snyk:
    permissions:
      contents: read # for actions/checkout to fetch code
      security-events: write # for github/codeql-action/upload-sarif to upload SARIF results
      actions: read # only required for a private repository by github/codeql-action/upload-sarif to get the Action run status
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Snyk CLI to check for security issues
        # Snyk can be used to break the build when it detects security issues.
        # In this case we want to upload the SAST issues to GitHub Code Scanning
        uses: snyk/actions/setup@master
        env:
          # This is where you will need to introduce the Snyk API token created with your Snyk account
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}

      # free 10GB of space
      - name: Remove unnecessary files
        if: matrix.os == 'ubuntu-latest'
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
          docker image prune --all --force
          docker builder prune --all --force
          docker system prune --all --force

      # Build the docker image for testing
      - name: Build a Docker image
        shell: bash
        run: DOCKER_BUILDKIT=1 docker build -f packages/grid/seaweedfs/seaweedfs.dockerfile ./packages/grid/seaweedfs -t seaweedfs:${{ github.sha }} --no-cache

      - name: Snyk auth
        shell: bash
        run: snyk config set api=$SNYK_TOKEN
        env:
          # This is where you will need to introduce the Snyk API token created with your Snyk account
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}

      - name: Snyk Container test
        continue-on-error: true
        shell: bash
        run: snyk container test seaweedfs:${{ github.sha }} --file=packages/grid/seaweedfs/seaweedfs.dockerfile --sarif --sarif-file-output=snyk-code.sarif
        env:
          # This is where you will need to introduce the Snyk API token created with your Snyk account
          SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}

        # Push the Snyk Code results into GitHub Code Scanning tab
      - name: Upload result to GitHub Code Scanning
        uses: github/codeql-action/upload-sarif@v3
        with:
          sarif_file: snyk-code.sarif


CD - Syft-CLI

name: CD - Syft-CLI

on:
  schedule:
    - cron: "00 10 * * */7" # At 10:00 UTC on every three days

  workflow_dispatch:
    inputs:
      skip_tests:
        description: "If true, skip tests"
        required: false
        default: "false"

jobs:
  call-pr-tests-linting:
    if: github.repository == 'OpenMined/PySyft' && (github.event.inputs.skip_tests == 'false' || github.event_name == 'schedule') # don't run on forks
    uses: OpenMined/PySyft/.github/workflows/pr-tests-linting.yml@dev

  # call-pr-tests-syft:
  #   if: github.repository == 'OpenMined/PySyft' && (github.event.inputs.skip_tests == 'false' || github.event_name == 'schedule') # don't run on forks
  #   uses: OpenMined/PySyft/.github/workflows/pr-tests-syft.yml@dev

  # call-pr-tests-stack:
  #   if: github.repository == 'OpenMined/PySyft' && (github.event.inputs.skip_tests == 'false' || github.event_name == 'schedule') # don't run on forks
  #   uses: OpenMined/PySyft/.github/workflows/pr-tests-stack.yml@dev
  #   secrets: inherit

  deploy-syft-cli:
    needs: [call-pr-tests-linting]
    if: always() && (needs.call-pr-tests-linting.result == 'success' || github.event.inputs.skip_tests == 'true')
    runs-on: ubuntu-latest

    outputs:
      deployed_version: ${{ steps.bump-version.outputs.deployed_version }}
      current_hash: ${{ steps.get-hashes.outputs.current_hash }}
      previous_hash: ${{ steps.get-hashes.outputs.previous_hash }}

    steps:
      - uses: actions/checkout@v4
        with:
          token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }}

      - name: Install checksumdir
        run: |
          pip install --upgrade checksumdir

      - name: Get the hashes
        id: get-hashes
        shell: bash
        run: |
          echo "current_hash=$(checksumdir ./packages/syftcli/syftcli -e version.py)" >> $GITHUB_OUTPUT
          echo "previous_hash=$(cat ./scripts/syftcli_hash)" >> $GITHUB_OUTPUT

      - name: Current Hash
        run: echo ${{steps.get-hashes.outputs.current_hash}}

      - name: Previous Hash
        run: echo ${{steps.get-hashes.outputs.previous_hash}}

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install dependencies
        if: ${{steps.get-hashes.outputs.current_hash != steps.get-hashes.outputs.previous_hash }}
        run: |
          python -m pip install --upgrade pip==24.0
          pip install --upgrade tox setuptools wheel twine bump2version PyYAML

      - name: Bump the Version
        if: ${{steps.get-hashes.outputs.current_hash != steps.get-hashes.outputs.previous_hash }}
        id: bump-version
        run: |
          python3 syftcli/version.py
          bump2version patch --allow-dirty --no-commit
          tox -e lint || true
          echo "deployed_version=$(python3 syftcli/version.py)" >> $GITHUB_OUTPUT
          python3 syftcli/version.py
        working-directory: ./packages/syftcli

      - name: Write the new hash
        if: ${{steps.get-hashes.outputs.current_hash != steps.get-hashes.outputs.previous_hash }}
        run: echo $(checksumdir packages/syftcli/syftcli -e version.py) > ./scripts/syftcli_hash

      - name: Commit changes to Syft CLI
        if: ${{steps.get-hashes.outputs.current_hash != steps.get-hashes.outputs.previous_hash }}
        uses: EndBug/add-and-commit@v9
        with:
          author_name: ${{ secrets.OM_BOT_NAME }}
          author_email: ${{ secrets.OM_BOT_EMAIL }}
          message: "[syftcli]bump version"
          add: "['./packages/syftcli/.bumpversion.cfg','./packages/syftcli/setup.py','./packages/syftcli/syftcli/version.py', './scripts/syftcli_hash']"

      - name: Build and publish
        if: ${{steps.get-hashes.outputs.current_hash != steps.get-hashes.outputs.previous_hash }}
        env:
          TWINE_USERNAME: __token__
          TWINE_PASSWORD: ${{ secrets.OM_SYFTCLI_PYPI_TOKEN }}
        run: |
          tox -e syftcli.publish
          twine upload packages/syftcli/dist/*

  create-syftcli-binaries:
    needs: [deploy-syft-cli]
    if: always() && (needs.deploy-syft-cli.result == 'success' && (needs.deploy-syft-cli.outputs.current_hash != needs.deploy-syft-cli.outputs.previous_hash))
    strategy:
      max-parallel: 99
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
      fail-fast: true
    runs-on: ${{matrix.os}}

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install build dependencies for syftcli
        run: |
          pip install --upgrade pip==24.0

      - name: Install Tox
        run: |
          pip install -U tox

      - name: Build syftcli
        env:
          SYFT_CLI_VERSION: ${{ needs.deploy-syft-cli.outputs.deployed_version }}
        run: |
          tox -e syftcli.build

      - name: upload binaries
        uses: actions/upload-artifact@v4
        with:
          name: syftcli-binaries-${{needs.deploy-syft-cli.outputs.deployed_version}}-${{matrix.os}}
          path: ./packages/syftcli/dist/cli/*
          if-no-files-found: error

  create-syftcli-github-release:
    needs: [create-syftcli-binaries, deploy-syft-cli]
    if: always() && (needs.create-syftcli-binaries.result == 'success')
    runs-on: ubuntu-latest
    permissions:
      contents: write # For tag and release notes.

    steps:
      - uses: actions/checkout@v4

      - name: Deployed Version
        run: echo ${{needs.deploy-syft-cli.outputs.deployed_version}}

      - name: Check if version is released on Pypi
        id: check-pypi
        run: |
          VERSION=${{needs.deploy-syft-cli.outputs.deployed_version}}

          RESPONSE=$(curl -s --head https://pypi.org/pypi/syftcli/$VERSION/json)

          if [[ $RESPONSE == *"HTTP/2 200"* ]]; then
            echo "released=true" >> $GITHUB_OUTPUT
          else
            echo "released=false" >> $GITHUB_OUTPUT
          fi

      - name: Make sure directory exists
        if: steps.check-pypi.outputs.released == 'true'
        run: |
          mkdir -p ./downloaded-binaries

      - name: Download binary
        if: steps.check-pypi.outputs.released == 'true'
        uses: actions/download-artifact@v4
        with:
          pattern: syftcli-binaries-${{needs.deploy-syft-cli.outputs.deployed_version}}-*
          path: ./downloaded-binaries/
          merge-multiple: true

      - name: GitHub Release
        if: steps.check-pypi.outputs.released == 'true'
        uses: softprops/action-gh-release@v2
        with:
          name: syftcli-v${{ needs.deploy-syft-cli.outputs.deployed_version }}
          generate_release_notes: false
          prerelease: false
          files: |
            ./downloaded-binaries/*
          tag_name: syftcli-v${{ needs.deploy-syft-cli.outputs.deployed_version }}


CD - Kubernetes - Dev

name: CD - Kubernetes - Dev

on:
  schedule:
    - cron: "0 */3 * * *"

  workflow_dispatch:
    inputs:
      check-cache:
        type: boolean
        description: "Check workflow cache"
        default: false
      deploy-helm:
        type: boolean
        description: "Deploy Helm Charts"
        default: false

jobs:
  check-last-run:
    # runs-on: [self-hosted, Linux]
    runs-on: ubuntu-latest

    outputs:
      same-commit: ${{ steps.cache.outputs.cache-hit }}

    steps:
      - name: Check for new changes
        if: github.event_name == 'schedule' || github.event.inputs.check-cache == 'true'
        id: cache
        uses: actions/cache@v4
        with:
          path: /tmp/k8s_dev_hash.txt # we don't care about the file, just the key
          key: k8s-dev-${{ github.sha }}
          lookup-only: true

      - name: Save Commit SHA
        if: github.event_name == 'schedule' || github.event.inputs.check-cache == 'true'
        # only for making the cache action happy
        run: echo "${{ github.sha }}" > /tmp/k8s_dev_hash.txt

  deploy-syft-dev:
    needs: check-last-run
    if: needs.check-last-run.outputs.same-commit != 'true'

    # runs-on: [self-hosted, Linux]
    runs-on: ubuntu-latest

    steps:
      - name: Permission to home directory
        run: |
          sudo chown -R $USER:$USER $HOME

      - uses: actions/checkout@v4

      # Checkout Infra repo (nested)
      - name: Checkout Infra Repo
        uses: actions/checkout@v4
        with:
          repository: ${{ secrets.INFRA_REPO }}
          ref: "main"
          token: ${{ secrets.INFRA_BOT_COMMIT_TOKEN }}
          path: infrastructure

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to Azure Container Registry
        uses: docker/login-action@v3
        with:
          registry: ${{ secrets.ACR_SERVER }}
          username: ${{ secrets.ACR_USERNAME }}
          password: ${{ secrets.ACR_PASSWORD }}

      - name: Set Grid package version
        id: grid
        shell: bash
        run: |
          echo "GRID_VERSION=$(python packages/grid/VERSION)" >> $GITHUB_OUTPUT

      - name: Build and push `syft` image to registry
        uses: docker/build-push-action@v5
        with:
          context: ./packages
          file: ./packages/grid/syft-client/syft.Dockerfile
          push: true
          tags: |
            ${{ secrets.ACR_SERVER }}/openmined/syft-client:dev
            ${{ secrets.ACR_SERVER }}/openmined/syft-client:dev-${{ github.sha }}
            ${{ secrets.ACR_SERVER }}/openmined/syft-client:${{ steps.grid.outputs.GRID_VERSION }}

      - name: Build and push `grid-backend` image to registry
        uses: docker/build-push-action@v5
        with:
          context: ./packages
          file: ./packages/grid/backend/backend.dockerfile
          push: true
          target: backend
          tags: |
            ${{ secrets.ACR_SERVER }}/openmined/grid-backend:dev
            ${{ secrets.ACR_SERVER }}/openmined/grid-backend:dev-${{ github.sha }}
            ${{ secrets.ACR_SERVER }}/openmined/grid-backend:${{ steps.grid.outputs.GRID_VERSION }}

      - name: Build and push `grid-frontend` image to registry
        uses: docker/build-push-action@v5
        with:
          context: ./packages/grid/frontend
          file: ./packages/grid/frontend/frontend.dockerfile
          push: true
          tags: |
            ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:dev
            ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:dev-${{ github.sha }}
            ${{ secrets.ACR_SERVER }}/openmined/grid-frontend:${{ steps.grid.outputs.GRID_VERSION }}
          target: grid-ui-development

      - name: Build and push `grid-seaweedfs` image to registry
        uses: docker/build-push-action@v5
        with:
          context: ./packages/grid/seaweedfs
          file: ./packages/grid/seaweedfs/seaweedfs.dockerfile
          push: true
          tags: |
            ${{ secrets.ACR_SERVER }}/openmined/grid-seaweedfs:dev
            ${{ secrets.ACR_SERVER }}/openmined/grid-seaweedfs:dev-${{ github.sha }}
            ${{ secrets.ACR_SERVER }}/openmined/grid-seaweedfs:${{ steps.grid.outputs.GRID_VERSION }}

      - name: Build and push `grid-enclave-attestation` image to registry
        uses: docker/build-push-action@v5
        with:
          context: ./packages/grid/enclave/attestation
          file: ./packages/grid/enclave/attestation/attestation.dockerfile
          push: true
          tags: |
            ${{ secrets.ACR_SERVER }}/openmined/grid-enclave-attestation:dev
            ${{ secrets.ACR_SERVER }}/openmined/grid-enclave-attestation:dev-${{ github.sha }}
            ${{ secrets.ACR_SERVER }}/openmined/grid-enclave-attestation:${{ steps.grid.outputs.GRID_VERSION }}

      - name: Build Helm Chart & Copy to infra
        if: github.ref == 'refs/heads/dev' || github.event.inputs.deploy-helm == 'true'
        shell: bash
        run: |
          K3D_VERSION=v5.6.3
          DEVSPACE_VERSION=v6.3.12

          # install k3d
          wget https://github.com/k3d-io/k3d/releases/download/${K3D_VERSION}/k3d-linux-amd64
          mv k3d-linux-amd64 k3d
          chmod +x k3d
          export PATH=`pwd`:$PATH
          k3d version

          # Install devspace
          curl -sSL https://github.com/loft-sh/devspace/releases/download/${DEVSPACE_VERSION}/devspace-linux-amd64 -o ./devspace
          chmod +x devspace
          devspace version

          # Install helm
          curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3 | bash
          helm version

          # install tox
          python -m pip install --upgrade pip==24.0
          pip install tox

          tox -e syft.build.helm

          rm -rf infrastructure/gitops/environments/dev/
          mkdir -p infrastructure/gitops/environments/dev/

          cp -R packages/grid/helm/syft/. infrastructure/gitops/environments/dev/

          # write version to VERSION.txt file
          echo "dev-${{github.sha}}" > infrastructure/gitops/environments/dev/VERSION.txt

      - name: Commit & push changes to Infra Repo
        if: github.ref == 'refs/heads/dev' || github.event.inputs.deploy-helm == 'true'
        uses: EndBug/add-and-commit@v9
        with:
          author_name: ${{ secrets.OM_BOT_NAME }}
          author_email: ${{ secrets.OM_BOT_EMAIL }}
          message: "[env] Update dev helm charts"
          add: "."
          push: "origin main"
          cwd: "./infrastructure/"


Post Merge Tasks

name: Post Merge Tasks

on:
  workflow_call:

  push:
    branches:
      - dev
      - main

jobs:
  post-merge-cleanup-notebooks:
    strategy:
      matrix:
        python-version: ["3.12"]
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          token: ${{ secrets.SYFT_BOT_COMMIT_TOKEN }}

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install pip packages
        run: |
          python -m pip install --upgrade --user pip tox

      - name: Check and Bump Protocol Version
        run: |
          tox -e syft.protocol.check

      - name: Commit changes to bump protocol and remove notebooks
        uses: EndBug/add-and-commit@v9
        with:
          author_name: ${{ secrets.OM_BOT_NAME }}
          author_email: ${{ secrets.OM_BOT_EMAIL }}
          message: "bump protocol and remove notebooks"
          remove: "-r notebooks/Experimental/ --ignore-unmatch"
          add: "['packages/syft/src/syft/protocol/protocol_version.json']"
          commit: "-a"


name: CD - Post Release Tests

on:
  workflow_dispatch:
    inputs:
      syft_version:
        description: "Syft version to test"
        required: true
        type: string

      release_platform:
        description: "Release Platform"
        required: true
        type: choice
        options:
          - "REAL_PYPI"
          - "TEST_PYPI"

  workflow_call:
    inputs:
      syft_version:
        description: "Syft version to test"
        required: true
        type: string

      release_platform:
        description: "Release Platform"
        required: true
        type: string
        default: "REAL_PYPI"

jobs:
  syft-install-check:
    strategy:
      max-parallel: 99
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest, macos-14]
        python-version: ["3.12", "3.11", "3.10"]
    runs-on: ${{ matrix.os }}
    steps:
      - name: System Architecture
        run: |
          echo "System Architecture: $(uname -m)"
          echo "System Version: $(uname -a)"

      - uses: actions/checkout@v4

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5

        with:
          python-version: ${{ matrix.python-version }}

      - name: Upgrade pip
        run: |
          python -m pip install --upgrade --user pip

      - name: Get pip cache dir
        id: pip-cache
        shell: bash
        run: |
          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT

      - name: pip cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.pip-cache.outputs.dir }}
          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
          restore-keys: |
            ${{ runner.os }}-pip-py${{ matrix.python-version }}-

      - name: Install Syft
        shell: bash
        run: |
          if [[ ${{inputs.release_platform}} == "REAL_PYPI" ]]; then
            pip install syft==${{inputs.syft_version}}
          fi
          if [[ ${{ inputs.release_platform }} == "TEST_PYPI" ]]; then
            pip install --extra-index-url https://test.pypi.org/simple/ syft==${{inputs.syft_version}}
          fi

      - name: Check Syft version
        run: |
          python -c "import syft; print(syft.__version__)"

  notebook-test-k8s-k3d:
    if: github.event.inputs.release_platform == 'REAL_PYPI'
    strategy:
      max-parallel: 99
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.12"]

    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4

      # free 10GB of space
      - name: Remove unnecessary files
        if: matrix.os == 'ubuntu-latest'
        run: |
          sudo rm -rf /usr/share/dotnet
          sudo rm -rf "$AGENT_TOOLSDIRECTORY"
          docker image prune --all --force
          docker builder prune --all --force
          docker system prune --all --force

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5

        with:
          python-version: ${{ matrix.python-version }}

      - name: Install K3d
        run: |
          K3D_VERSION=v5.6.3
          wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=$K3D_VERSION bash

      - name: Get pip cache dir
        id: pip-cache
        shell: bash
        run: |
          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT

      - name: pip cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.pip-cache.outputs.dir }}
          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
          restore-keys: |
            ${{ runner.os }}-pip-py${{ matrix.python-version }}-

      - name: Install tox
        run: |
          pip install --upgrade pip==24.0 uv==0.2.13 tox tox-uv==1.9.0

      - name: Run K8s tests
        env:
          SYFT_VERSION: ${{ inputs.syft_version }}
        run: |
          tox -e syft.test.helm

  # This job is used to test the syft unit tests on Test PyPi
  syft-unit-tests:
    strategy:
      max-parallel: 99
      matrix:
        os: [ubuntu-latest, macos-latest]
        python-version: ["3.12", "3.11", "3.10"]
    runs-on: ${{ matrix.os }}
    steps:
      - name: System Architecture
        run: |
          echo "System Architecture: $(uname -m)"
          echo "System Version: $(uname -a)"

      - uses: actions/checkout@v4

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5

        with:
          python-version: ${{ matrix.python-version }}

      - name: Upgrade pip
        run: |
          python -m pip install --upgrade --user pip

      - name: Get pip cache dir
        id: pip-cache
        shell: bash
        run: |
          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT

      - name: pip cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.pip-cache.outputs.dir }}
          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
          restore-keys: |
            ${{ runner.os }}-pip-py${{ matrix.python-version }}-

      - name: Install Syft
        env:
          SYFT_VERSION: ${{ inputs.syft_version }}
        run: |
          if [[ ${{ inputs.release_platform }} == "TEST_PYPI" ]]; then
            pip install --extra-index-url https://test.pypi.org/simple/ syft[data_science,dev]==${{ inputs.syft_version }}
          fi
          if [[ ${{ inputs.release_platform }} == "REAL_PYPI" ]]; then
            pip install syft[data_science,dev]==${{ inputs.syft_version }}
          fi

      - name: Install tox and uv
        run: |
          pip install --upgrade pip==24.0 uv==0.2.13 tox tox-uv==1.9.0 tox-current-env

      - name: Run unit tests
        run: |
          tox -e syft.test.unit --current-env


E2E - Notebook Tests

name: E2E - Notebook Tests

on:
  workflow_dispatch:
    inputs:
      syft_version:
        description: "Syft version to test"
        required: true
        type: string
      node_url:
        description: "Node URL to use"
        required: true
        type: string
      node_port:
        description: "Node port"
        required: true
        type: number
      exclude_notebooks:
        description: "Notebooks to exclude ex: not 11-container-images-k8s.ipynb"
        required: false
        type: string

  workflow_call:
    inputs:
      syft_version:
        description: "Syft version to test"
        required: true
        type: string
      node_url:
        description: "Node URL to use"
        required: true
        type: string
      node_port:
        description: "Node port"
        required: true
        type: number
      exclude_notebooks:
        description: "Notebooks to exclude ex: not 11-container-images-k8s.ipynb"
        required: false
        type: string

jobs:
  notebook-test-e2e:
    strategy:
      max-parallel: 99
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.12"]

    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5

        with:
          python-version: ${{ matrix.python-version }}

      - name: Upgrade pip
        run: |
          python -m pip install --upgrade --user pip

      - name: Install Deps
        run: |
          pip install --upgrade pip==24.0 uv==0.2.13 tox tox-uv==1.9.0

      - name: Get pip cache dir
        id: pip-cache
        shell: bash
        run: |
          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT

      - name: pip cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.pip-cache.outputs.dir }}
          key: ${{ runner.os }}-pip-py${{ matrix.python-version }}-${{ hashFiles('setup.cfg') }}
          restore-keys: |
            ${{ runner.os }}-pip-py${{ matrix.python-version }}-

      - name: Install tox
        run: |
          pip install tox

      - name: Run Notebook tests
        env:
          SYFT_VERSION: ${{ inputs.syft_version }}
          NODE_URL: ${{ inputs.node_url }}
          NODE_PORT: ${{ inputs.node_port }}
          EXCLUDE_NOTEBOOKS: ${{ inputs.exclude_notebooks }}
        run: |
          tox -e e2e.test.notebook


PR Tests - Helm Upgrade

name: PR Tests - Helm Upgrade

on:
  # Re-enable when we have a stable helm chart
  # pull_request:
  #   branches:
  #     - dev
  #   paths:
  #     - packages/grid/helm/syft/**

  workflow_dispatch:
    inputs:
      upgrade_type:
        description: "Select upgrade path type"
        required: false
        default: "BetaToDev"
        type: choice
        options:
          - BetaToDev
          - ProdToBeta
          - ProdToDev

concurrency:
  group: pr-tests-helm-upgrade
  cancel-in-progress: true

jobs:
  pr-tests-helm-upgrade:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Install dependencies
        run: |
          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
          brew update

          # Install python deps
          pip install --upgrade pip==24.0
          pip install tox

          # Install kubernetes
          brew install helm k3d devspace kubectl

      - name: Setup cluster
        run: |
          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"

          tox -e dev.k8s.start

      - name: Upgrade helm chart
        run: |
          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"

          # default upgrade is beta to dev, but override with input if provided
          UPGRADE_TYPE_INPUT=${{ github.event.inputs.upgrade_type }}
          export UPGRADE_TYPE=${UPGRADE_TYPE_INPUT:-BetaToDev}
          tox -e syft.test.helm.upgrade

      - name: Destroy cluster
        if: always()
        run: |
          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"

          tox -e dev.k8s.destroyall


PR Tests - Lint Helm Charts

name: PR Tests - Lint Helm Charts

on:
  # temporary disabled
  # pull_request:
  #   branches:
  #     - dev
  #   paths:
  #     - packages/grid/helm/syft/**

  workflow_dispatch:
    inputs:
      none:
        description: "Run Tests Manually"
        required: false

concurrency:
  group: pr-tests-helm-lint
  cancel-in-progress: true

jobs:
  pr-tests-helm-lint:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Install dependencies
        run: |
          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
          brew update
          brew tap FairwindsOps/tap
          brew install kube-linter FairwindsOps/tap/polaris

          # Install python deps
          pip install --upgrade pip==24.0
          pip install tox

          kube-linter version
          polaris version

      - name: Run Polaris
        run: |
          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
          polaris audit --helm-chart packages/grid/helm/syft --helm-values packages/grid/helm/syft/values.yaml --format=pretty --only-show-failed-tests

      - name: Run Linter
        run: |
          eval "$(/home/linuxbrew/.linuxbrew/bin/brew shellenv)"
          tox -e syft.lint.helm


Lint

name: Lint

on: [pull_request]

jobs:
  lintrunner:
    runs-on: ubuntu-latest
    steps:
      - name: Pull DGL
        uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Checkout master and HEAD
        run: |
          git checkout -t origin/master
          git checkout ${{ github.event.pull_request.head.sha }}

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: '3.8'

      - name: Install requirements
        run: |
          python -m pip install --upgrade pip
          pip install lintrunner --user

      - name: Initialize lint dependencies
        run: lintrunner init

      - name: Run lintrunner on all changed files
        run: |
          set +e
          if ! lintrunner --force-color -m master --tee-json=lint.json; then
              echo ""
              echo -e "\e[1m\e[36mYou can reproduce these results locally by using \`lintrunner\`.\e[0m"
              echo -e "\e[1m\e[36mSee https://github.com/pytorch/pytorch/wiki/lintrunner for setup instructions.\e[0m"
              exit 1
          fi

      - name: Store annotations
        if: always() && github.event_name == 'pull_request'
        # Don't show this as an error; the above step will have already failed.
        continue-on-error: true
        run: |
          # Use jq to massage the JSON lint output into GitHub Actions workflow commands.
          jq --raw-output \
            '"::\(if .severity == "advice" or .severity == "disabled" then "warning" else .severity end) file=\(.path),line=\(.line),col=\(.char),title=\(.code) \(.name)::" + (.description | gsub("\\n"; "%0A"))' \
            lint.json

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.sha }}-${{ github.event_name == 'workflow_dispatch' }}
  cancel-in-progress: true


Mark stale issues and pull requests

# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time.
#
# You can adjust the behavior by modifying this file.
# For more information, see:
# https://github.com/actions/stale
name: Mark stale issues and pull requests

on:
  schedule:
  - cron: '0 1 * * *'

jobs:
  stale:

    runs-on: ubuntu-latest
    permissions:
      issues: write
      pull-requests: write

    steps:
    - uses: actions/stale@v4.1.0
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
        days-before-issue-stale: 30
        days-before-issue-close: -1 # disable issue close
        days-before-pr-stale: -1 # disable stale bot on pr
        days-before-pr-close: -1 # disable stale bot on pr
        stale-issue-message: 'This issue has been automatically marked as stale due to lack of activity. It will be closed if no further activity occurs. Thank you'
        close-issue-message: 'This issue is closed due to lack of activity. Feel free to reopen it if you still have questions.'
        stale-issue-label: 'stale-issue'
        exempt-issue-labels: 'bug:confirmed,feature request,help wanted,Work Item'
        exempt-all-issue-milestones: true


PR

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# GH actions.
# We use it to cover windows and mac builds
# Jenkins is still the primary CI

name: PR

on:
  # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target
  pull_request_target:
    types: [assigned, opened, synchronize, reopened, edited, ready_for_review]

concurrency:
  group: PR-${{ github.event.pull_request.number }}
  cancel-in-progress: true

jobs:
  cc-reviewers:
    if: github.repository == 'apache/tvm'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: "recursive"
      - name: Add cc'ed reviewers
        env:
          PR: ${{ toJson(github.event.pull_request) }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -eux
          python ci/scripts/github/github_cc_reviewers.py || echo step failed


CI

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# GH actions.
# We use it to cover windows and mac builds
# Jenkins is still the primary CI

name: CI

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
  workflow_dispatch:

concurrency:
  group: CI-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

jobs:
  MacOS:
    if: ${{ github.repository == 'apache/tvm' }}
    runs-on: macOS-latest
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: 'recursive'
      - name: Set up environment
        uses: ./.github/actions/setup
      - name: Conda Build
        shell: bash -l {0}
        run: >-
          conda build --output-folder=conda/pkg  conda/recipe &&
          conda install tvm -c ./conda/pkg
      - name: Build iOS RPC
        run: |
          IOS_VERSION="14.0"
          CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Release \
                       -DCMAKE_SYSTEM_NAME=iOS \
                       -DCMAKE_SYSTEM_VERSION=${IOS_VERSION} \
                       -DCMAKE_OSX_SYSROOT=iphonesimulator \
                       -DCMAKE_OSX_ARCHITECTURES=x86_64 \
                       -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
                       -DCMAKE_BUILD_WITH_INSTALL_NAME_DIR=ON \
                       -DUSE_IOS_RPC=ON"

          mkdir build-ios-simulator
          cd build-ios-simulator
          cmake .. ${CMAKE_FLAGS}
          cmake --build . --target ios_rpc
      - name: Test
        shell: bash -l {0}
        run: >-
          python -m pytest -v tests/python/all-platform-minimal-test
      - name: Minimal Metal Compile-Only
        shell: bash -l {0}
        run: |
          python -m pytest -v -s 'tests/python/codegen/test_gpu_codegen_allreduce.py::test_allreduce_sum_compile'
          python -m pytest -v -s 'tests/python/codegen/test_target_codegen_metal.py::test_func_with_trailing_pod_params'
      - name: Minimal Metal Compile-and-Run
        shell: bash -l {0}
        run: >-
          python -m pytest -v -s 'tests/python/codegen/test_target_codegen_metal.py'
          python -m pytest -v -s 'tests/python/codegen/test_target_codegen_gpu_common.py'
          python -m pytest -v -s 'tests/python/codegen/test_gpu_codegen_allreduce.py::test_allreduce_sum[dims0-metal]'
#      - name: Test iOS RPC
#        shell: bash -l {0}
#        run: >-
#          python -m pip install tornado psutil cloudpickle &&
#          export PYTHONPATH=tests/python/contrib:${PYTHONPATH} &&
#          export BUNDLE_ID=org.apache.tvmrpc &&
#          export BUNDLE_PATH=build-ios-simulator/apps/ios_rpc/ios_rpc/src/ios_rpc-build/Release-iphonesimulator/tvmrpc.app &&
#          python -m pytest -v tests/python/contrib/test_rpc_server_device.py

  Windows:
    if: ${{ github.repository == 'apache/tvm' }}
    runs-on: windows-2019
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: 'recursive'
      - name: Set up environment
        uses: ./.github/actions/setup
      - name: Conda Build
        shell: cmd /C call {0}
        run: >-
          conda build --output-folder=conda/pkg conda/recipe &&
          conda install tvm -c ./conda/pkg
      - name: Test
        shell: cmd /C call {0}
        run: >-
          python -m pytest -v tests/python/all-platform-minimal-test

  # Disabled due to https://github.com/apache/tvm/issues/13950
  # Windows-Static-Runtime:
  #   if: ${{ github.repository == 'apache/tvm' }}
  #   runs-on: windows-2019
  #   steps:
  #     - uses: actions/checkout@v2
  #       with:
  #         submodules: 'recursive'
  #     - name: Set up environment
  #       uses: ./.github/actions/setup
  #     - name: Build static TVM runtime
  #       shell: bash -l {0}
  #       run: |
  #         tests/scripts/task_config_build_static.sh build
  #         cd build
  #         cmake .. -A x64 -DCMAKE_CONFIGURATION_TYPES="Release"
  #         cmake --build . --config Release --target runtime

  Linux-Static-Runtime:
    if: ${{ github.repository == 'apache/tvm' }}
    runs-on: Ubuntu-20.04
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: 'recursive'
      - name: Set up environment
        uses: ./.github/actions/setup
      - name: Build static TVM runtime
        shell: bash -l {0}
        run: |
          tests/scripts/task_config_build_static.sh build
          cd build
          cmake ..
          cmake --build . --config Release --target runtime

  Android:
    if: ${{ github.repository == 'apache/tvm' }}
    runs-on: Ubuntu-20.04
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: 'recursive'
      - name: Set up environment
        uses: ./.github/actions/setup
      - name: Set up java
        uses: actions/setup-java@v3
        with:
          distribution: 'zulu'
          java-version: '11'
      - name: Build TVM
        shell: bash -l {0}
        run: |
          mkdir build
          cd build
          ../tests/scripts/task_config_build_jvm.sh .
          cmake ..
          make
      - name: Build TVM4J
        run: |
          make jvmpkg
      - name: Build android_rpc
        working-directory: apps/android_rpc
        run: |
          set -eux
          export PATH="${ANDROID_NDK_LATEST_HOME}:$PATH"
          gradle clean build
      - name: Upload android_rpc APK
        uses: actions/upload-artifact@v2
        with:
          name: android_rpc-debug.apk
          path: ./apps/android_rpc/app/build/outputs/apk/debug/app-debug.apk
      - name: Build android_deploy
        working-directory: apps/android_deploy
        run: |
          set -eux
          export PATH="${ANDROID_NDK_LATEST_HOME}:$PATH"
          gradle clean build
      - name: Upload android_deploy APK
        uses: actions/upload-artifact@v2
        with:
          name: android_deploy-debug.apk
          path: ./apps/android_deploy/app/build/outputs/apk/debug/app-debug.apk


Nightly Docker Update

name: Nightly Docker Update
on:
  schedule:
    - cron: "0 0 * * *"
  workflow_dispatch:

concurrency:
  group: nightly-docker-update
  cancel-in-progress: true

jobs:
  open_update_pr:
    permissions:
      actions: write
      checks: write
      contents: write
      id-token: write
      issues: write
      pull-requests: write
      statuses: write
    if: github.repository == 'apache/tvm'
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v2
      - name: Open PR to update Docker images
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -eux
          python ci/scripts/jenkins/open_docker_update_pr.py


Ping Reviewers

name: Ping Reviewers
on:
  schedule:
    - cron: "0/15 * * * *"
  workflow_dispatch:

concurrency:
  group: ping
  cancel-in-progress: true

jobs:
  ping:
    if: github.repository == 'apache/tvm'
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v2
      - name: Ping reviewers
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -eux
          python ci/scripts/github/ping_reviewers.py --wait-time-minutes 10080 || echo failed


Teams

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: Teams

on:
  issues:
    types: [opened, edited, reopened, labeled]

concurrency:
  group: Teams-${{ github.event.pull_request.number }}-${{ github.event.issue.number }}
  cancel-in-progress: true

jobs:
  tag-teams:
    if: github.repository == 'apache/tvm'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: Tag people from relevant teams
        env:
          PR: ${{ toJson(github.event.pull_request) }}
          ISSUE: ${{ toJson(github.event.issue) }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -eux
          python ci/scripts/github/github_tag_teams.py || echo failed


tvm-bot

name: tvm-bot
on:
  pull_request_review:
    types:
      - submitted
  issue_comment:

concurrency:
  group: merge-${{ github.event.pull_request.number }}-${{ github.event.issue.number }}
  cancel-in-progress: true

jobs:
  run-tvm-bot:
    permissions:
      actions: write
      checks: write
      contents: write
      id-token: write
      issues: write
      pull-requests: write
      statuses: write
    if: ${{ github.event.issue.pull_request && github.repository == 'apache/tvm' }}
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v2
      - name: Run tvm-bot
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          GH_ACTIONS_TOKEN: ${{ secrets.GH_ACTIONS_TOKEN }}
          TVM_BOT_JENKINS_TOKEN: ${{ secrets.TVM_BOT_JENKINS_TOKEN }}
          PR_NUMBER: ${{ github.event.issue.number }}
          ISSUE_COMMENT: ${{ toJson(github.event.comment) }}
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: |
          set -eux
          python ci/scripts/github/github_tvmbot.py --pr "$PR_NUMBER" --run-url "$RUN_URL" --trigger-comment-json "$ISSUE_COMMENT"


Update last-successful branch

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# GH actions.
# We use it to cover windows and mac builds
# Jenkins is still the primary CI

name: Update last-successful branch

on:
  schedule:
    - cron: "0/15 * * * *"
  workflow_dispatch:

concurrency:
  group: update-last-successful-branch
  cancel-in-progress: true

jobs:
  update-last-successful-branch:
    if: github.repository == 'apache/tvm'
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v2
      - name: Update last-successful branch
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -eux
          python ci/scripts/github/update_branch.py || echo step failed


Update nightly branch

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# Update a branch for nightly test results
name: Update nightly branch

on:
  schedule:
    # 9 PM PST
    - cron: "0 5 * * *"
  workflow_dispatch:

concurrency:
  group: update-nightly-branch
  cancel-in-progress: true

jobs:
  update-nightly-branch:
    if: github.repository == 'apache/tvm'
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v2
      - name: Update nightly branch
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -eux
          git checkout -B nightly
          git log -5
          git push origin --force nightly


Upload CI Resource

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
name: Upload CI Resource

on:
  workflow_dispatch:
    inputs:
      url:
        description: 'URL of the file (e.g. "https://example.com/file.zip")'
        required: true
        type: string
      sha256:
        description: 'SHA256 of the file'
        required: true
        type: string
      upload_path:
        description: 'Path of the file in S3 (e.g. "my_folder/something.zip")'
        required: true
        type: string

concurrency:
  group: upload-ci-resource
  cancel-in-progress: true

jobs:
  upload-ci-resource:
    if: github.repository == 'apache/tvm'
    runs-on: ubuntu-20.04
    steps:
      - name: Download item and upload to S3
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          AWS_ACCESS_KEY_ID: ${{ secrets.CI_RESOURCES_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_RESOURCES_AWS_SECRET_ACCESS_KEY }}
          URL: ${{ inputs.url }}
          SHA256: ${{ inputs.sha256 }}
          UPLOAD_PATH: ${{ inputs.upload_path }}
          AWS_DEFAULT_REGION: us-west-2
        run: |
          set -eux
          curl -L -o downloaded_file "$URL"
          echo "$SHA256 downloaded_file" | sha256sum --check
          aws s3 cp downloaded_file "s3://tvm-ci-resources/$UPLOAD_PATH"
          echo "The item is available at https://tvm-ci-resources.s3.us-west-2.amazonaws.com/$UPLOAD_PATH"
          echo "Add this line to tests/scripts/request_hook/request_hook.py"
          echo "    \"$URL\": f\"{BASE}/$UPLOAD_PATH\","


Backport

name: Backport
on:
  pull_request:
    types:
      - closed
      - labeled

jobs:
  backport:
    runs-on: ubuntu-latest
    # Only react to merged PRs for security reasons.
    # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target.
    if: >
      github.event.pull_request.merged
      && (
        github.event.action == 'closed'
        || (
          github.event.action == 'labeled'
          && contains(github.event.label.name, 'backport')
        )
      )
    name: Backport
    steps:
      - name: Backport
        uses: tibdex/backport@v2
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}


CodeQL

name: "CodeQL"

on:
  push:
    branches: [ 'main' ]
  pull_request:
    # The branches below must be a subset of the branches above
    branches: [ 'main' ]
  schedule:
    - cron: '25 2 * * 0'

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write

    strategy:
      fail-fast: false
      matrix:
        language: [ 'python', 'javascript' ]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v3
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
        # By default, queries listed here will override any specified in a config file.
        # Prefix the list here with "+" to use these queries and those in the config file.

        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
        queries: +security-and-quality

    # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
      uses: github/codeql-action/autobuild@v3

    # ℹ Command-line programs to run using the OS shell.
    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun

    #   If the Autobuild fails above, remove it and uncomment the following three lines.
    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.

    # - run: |
    #   echo "Run, Build Application using script"
    #   ./location_of_script_within_repo/buildscript.sh

    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v3
      with:
        category: "/language:${{matrix.language}}"


Generate Documentation

name: Generate Documentation

on:
  release:
    types: [published]
  push:
    branches:
      - main
  pull_request:

permissions: {}

jobs:
  build:
    permissions:
      contents: write # for git push

    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.10"]
    steps:
    - name: Checkout
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Checkout statsmodels.github.io without token
      uses: actions/checkout@v4
      with:
        fetch-depth: 1
        repository: statsmodels/statsmodels.github.io
        path: statsmodels.github.io
      if: ${{ github.event_name == 'pull_request' }}
    - name: Checkout statsmodels.github.io with token
      uses: actions/checkout@v4
      with:
        fetch-depth: 1
        repository: statsmodels/statsmodels.github.io
        path: statsmodels.github.io
        token: ${{ secrets.DOCBUILD_TOKEN }}
      if: ${{ github.event_name == 'release' || github.event_name == 'push' }}
    - name: Event type
      run: echo $EVENT_TYPE
      env:
        EVENT_TYPE: ${{ github.event_name }}
    - name: Install pandoc
      uses: r-lib/actions/setup-pandoc@v2
    - name: Install graphviz
      uses: ts-graphviz/setup-graphviz@v2
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
        cache: "pip"
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip wheel setuptools
        python -m pip install -r requirements.txt
        python -m pip install -r requirements-dev.txt
        python -m pip install -r requirements-doc.txt
        python -m pip list
    - name: Install statsmodels
      run: python -m pip install . -vv
    - name: Build documentation
      run: |
        pushd docs
        make html
        popd
    - name: Move and commit documentation
      env:
        GIT_TAG: ${{ github.event.release.tag_name }}
      run: |
        source tools/ci/docbuild-commit.sh
    - name: Push changes
      env:
        PERSONAL_ACCESS_TOKEN: ${{ secrets.DOCBUILD_TOKEN }}
      run: |
        source tools/ci/docbuild-push.sh
      if: ${{ github.event_name == 'release' || github.event_name == 'push' }}


Create Subrepo Conda Packages

on:
  push:
    branches:
      - 'master'
    paths:
      - '**.gitrepo'
      - '.github/workflows/build-subrepos.yml'

  workflow_call:

  workflow_dispatch:

concurrency:
  group: build-subrepos-${{ github.ref }}
  cancel-in-progress: true

name: Create Subrepo Conda Packages

jobs:
  build-pkgs:
    name: Build ${{ matrix.pkg }} ${{ matrix.cache-arch }} Python-${{ matrix.python-version }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
        pkg: [python-lsp-server, qtconsole, spyder-kernels]
        include:
          - python-version: '3.11'
          - cache-arch: noarch
          - os: ubuntu-latest
            pkg: spyder-kernels
            cache-arch: unix
          - os: windows-latest
            pkg: spyder-kernels
            python-version: '3.11'
            cache-arch: win-64
    defaults:
      run:
        shell: bash -l {0}
        working-directory: ${{ github.workspace }}/installers-conda
    env:
      pkg: ${{ matrix.pkg }}

    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Cache ${{ matrix.pkg }} ${{ matrix.cache-arch }} Conda Build
        id: cache
        uses: actions/cache@v4
        with:
          path: installers-conda/build/conda-bld/**/*.tar.bz2
          key: ${{ matrix.pkg }}_${{ matrix.cache-arch }}_${{ matrix.python-version }}_${{ hashFiles(format('external-deps/{0}/.gitrepo', env.pkg)) }}
          lookup-only: true
          enableCrossOsArchive: true

      - name: Setup Build Environment
        if: steps.cache.outputs.cache-hit != 'true'
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: installers-conda/build-environment.yml
          environment-name: spy-inst
          create-args: >-
            --channel-priority=flexible
            python=${{ matrix.python-version }}
          cache-downloads: true
          cache-environment: true

      - name: Build ${{ matrix.pkg }} Conda Package
        if: steps.cache.outputs.cache-hit != 'true'
        env:
          CONDA_BLD_PATH: ${{ github.workspace }}/installers-conda/build/conda-bld
        run: python build_conda_pkgs.py --build $pkg


Chatops Binder

name: Chatops Binder
on: [issue_comment] # issues and PRs are equivalent in terms of comments for the GitHub API

jobs:
  trigger-chatops:
    # Make sure the comment is from one in our organization, it's on a PR, and contains the command "/binder"
    if: |
      (
        (github.event.issue.pull_request != null) &&
        (
          contains(github.event.comment.body, '/Show binder') ||
          contains(github.event.comment.body, '/show binder')
        )
      )
    runs-on: ubuntu-latest
    steps:
      # Use the GitHub API to:
      #  (1) Get the branch name of the PR that has been commented on with "/binder"
      #  (2) make a comment on the PR with the binder badge
    - name: comment on PR with Binder link
      uses: actions/github-script@v1
      with:
        github-token: ${{secrets.GITHUB_TOKEN}}
        script: |
          // Get the branch name
          github.pulls.get({
            owner: context.repo.owner,
            repo: context.repo.repo,
            pull_number: context.payload.issue.number
          }).then( (pr) => {

            // use the branch name to make a comment  on the PR with a Binder badge
            var BRANCH_NAME = pr.data.head.ref;
            var USER_NAME = pr.data.head.user.login;
            var BASE_NAME = pr.data.base.ref
            github.issues.createComment({
              issue_number: context.payload.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: `[![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/spyder-ide/binder-environments/${BASE_NAME}?urlpath=git-pull%3Frepo%3Dhttps%253A%252F%252Fgithub.com%252F${USER_NAME}%252Fspyder%26urlpath%3Ddesktop%252F%26branch%3D${BRANCH_NAME}%26depth%3D500) :point_left: Launch a Binder instance on this branch`
            })
          })


Close duplicate issues

name: Close duplicate issues

on: [issues]

jobs:
  example:
    runs-on: ubuntu-latest
    steps:
      - uses: spyder-ide/action-close-duplicate-issues@master
        with:
          repo-token: ${{ secrets.SECRET_TOKEN }}
          items: >-
            [
             {
               "pattern": "ValueError: cannot set WRITEABLE flag to True of this array",
               "reply": "Thanks for reporting. This issue is fixed in our latest Spyder 3 version. Please update by opening the Anaconda Prompt and running there:

`conda install spyder=3.3.6`",
               "labels": ["resolution:Duplicate"],
               "close": true
             },
             {
               "pattern": "TypeError: _start_server() missing 1 required positional argument",
               "reply": "Thanks for reporting. This issue is fixed in our latest Spyder 3 version. Please update by opening the Anaconda Prompt and running there:

`conda install spyder=3.3.6`",
               "labels": ["resolution:Duplicate"],
               "close": true
             },
             {
               "pattern": "AttributeError: 'NoneType' object has no attribute 'execute'",
               "reply": "Thanks for reporting. This issue is fixed in our latest Spyder 3 version. Please update by opening the Anaconda Prompt and running there:

`conda install spyder=3.3.6`",
               "labels": ["resolution:Duplicate"],
               "close": true
             },
             {
               "pattern": "AttributeError: module 'dateutil.tz' has no attribute 'UTC'",
               "reply": "Thanks for reporting. This issue is fixed in our latest Spyder 3 version. Please update by opening the Anaconda Prompt and running there:

`conda install spyder=3.3.6`",
               "labels": ["resolution:Duplicate"],
               "close": true
             },
             {
               "pattern": "AttributeError: '_ModuleMock' object has no attribute 'PLUGIN_CLASS'",
               "reply": "Thanks for reporting. This issue is fixed in our latest Spyder 3 version. Please update by opening the Anaconda Prompt and running there:

`conda install spyder=3.3.6`",
               "labels": ["resolution:Duplicate"],
               "close": true
             },
             {
               "pattern": "TypeError: _handle_remote_call_reply() missing 1 required positional argument: 'load_exception'",
               "reply": "Thanks for reporting. This issue is fixed in our latest Spyder 4 version. Please update by opening the Anaconda Prompt and running there:

`conda install spyder=4.1.*`",
               "labels": ["resolution:Duplicate"],
               "close": true
             },
             {
               "pattern": "psutil.NoSuchProcess: psutil.NoSuchProcess process no longer exists",
               "reply": "Thanks for reporting. This issue is a duplicate of #11872 and fixed in our latest Spyder 4 version. Please update by opening the Anaconda Prompt and running there:

`conda install spyder=4.1.*`",
               "labels": ["resolution:Duplicate"],
               "close": true
             },
             {
               "pattern": "current_pos_text = text[offset - utf16_diff - 1]",
               "reply": "Thanks for reporting. This issue is a duplicate of #12266 and fixed in our latest version. Please update by opening the Anaconda Prompt and running there:

`conda install spyder=4.1.*`",
               "labels": ["resolution:Duplicate"],
               "close": true
             },
             {
               "pattern": "TimeoutError: Timeout while waiting for comm port.",
               "reply": "Thanks for reporting. This issue is a duplicate of #13288 and fixed in our latest version. Please update by opening the Anaconda Prompt and running there:

`conda update anaconda`
`conda install spyder=4.2.*`",
               "labels": ["resolution:Duplicate"],
               "close": true
             },
             {
               "pattern": "TypeError: __init__() got an unexpected keyword argument 'size'",
               "reply": "Thanks for reporting. This issue is a duplicate of #11875 and fixed in our latest version. Please update by opening the Anaconda Prompt and running there:

`conda install spyder=4.1.*`",
               "labels": ["resolution:Duplicate"],
               "close": true
             },
             {
               "pattern": "TypeError: CodeEditor.sig_display_object_info[str, bool].emit(): argument 1 has unexpected type 'list'",
               "reply": "Thanks for reporting. This issue is a duplicate of #13297 and fixed in our latest version. Please update by opening the Anaconda Prompt and running there:

`conda update anaconda`
`conda install spyder=4.1.5`",
               "labels": ["resolution:Duplicate"],
               "close": true
             },
             {
               "pattern": "triggered=lambda v: self.open_project()",
               "reply": "Thanks for reporting. This issue is a duplicate of #13346 and fixed in our latest version. Please update by opening the Anaconda Prompt and running there:

`conda update anaconda`
`conda install spyder=4.1.5`",
               "labels": ["resolution:Duplicate"],
               "close": true
             },
             {
               "pattern": "AttributeError: module 'qdarkstyle.palette' has no attribute 'DarkPalette'",
               "reply": "Thanks for reporting. To fix this, please close Spyder, open the Anaconda Prompt and run there:

`conda install qdarkstyle=2.8.1`",
               "labels": ["resolution:Duplicate"],
               "close": true
             },
             {
               "pattern": "AttributeError: 'SpyderKernelManager' object has no attribute 'kernel'",
               "reply": "Thanks for reporting. To fix this, please close Spyder, open the Anaconda Prompt and run there:

`conda install jupyter_client=6.1.12 freetype=2.10.4`",
               "labels": ["resolution:Duplicate"],
               "close": true
             },
             {
               "pattern": "TypeError: _show_prompt() got an unexpected keyword argument 'separator'",
               "reply": "Thanks for reporting. To fix this error, please close Spyder, open the Anaconda Prompt (on Windows) or a terminal (on macOS or Linux) and run there the commands mentioned in [our documentation](http://docs.spyder-ide.org/current/installation.html#updating-with-anaconda).",
               "labels": ["resolution:Duplicate"],
               "close": true
             },
             {
               "pattern": "AttributeError: '_asyncio.Future' object has no attribute 'index'",
               "reply": "Thanks for reporting. To fix this error, please close Spyder, open the Anaconda Prompt and run the following command there:

`conda install jupyter_client=7.3.1`",
               "labels": ["resolution:Duplicate"],
               "close": true
             },
             {
               "pattern": "AttributeError: 'DataFrame' object has no attribute 'iteritems'",
               "reply": "Thanks for reporting. To fix this error, please close Spyder, open the Anaconda Prompt (on Windows) or a terminal (on macOS or Linux) and run there the commands mentioned in [our documentation](http://docs.spyder-ide.org/current/installation.html#updating-with-anaconda).",
               "labels": ["resolution:Duplicate"],
               "close": true
             }
             ]


Nightly conda-based installers

on:
  schedule:
    - cron: '30 6 * * *'

  pull_request:
    paths:
      - 'installers-conda/**'
      - '.github/workflows/installers-conda.yml'
      - 'requirements/*.yml'
      - 'MANIFEST.in'

  release:
    types:
      - created

  workflow_dispatch:
    inputs:
      pre:
        description: 'Build as release candidate'
        required: false
        default: true
        type: boolean
      ssh:
        description: 'Enable ssh debugging'
        required: false
        default: false
        type: boolean
      macos-x86_64:
        description: 'Build macOS x86_64 installer'
        required: false
        default: true
        type: boolean
      macos-arm64:
        description: 'Build macOS arm64 installer'
        required: false
        default: true
        type: boolean
      linux:
        description: 'Build Linux installer'
        required: false
        default: true
        type: boolean
      win:
        description: 'Build Windows installer'
        required: false
        default: true
        type: boolean

concurrency:
  group: installers-conda-${{ github.ref }}
  cancel-in-progress: true

name: Nightly conda-based installers

env:
  IS_RELEASE: ${{ github.event_name == 'release' }}
  IS_PRE_RELEASE: ${{ github.event_name == 'workflow_dispatch' && inputs.pre }}
  ENABLE_SSH: ${{ github.event_name == 'workflow_dispatch' && inputs.ssh }}
  BUILD_MAC: ${{ github.event_name != 'workflow_dispatch' || inputs.macos-x86_64 }}
  BUILD_ARM: ${{ github.event_name != 'workflow_dispatch' || inputs.macos-arm64 }}
  BUILD_LNX: ${{ github.event_name != 'workflow_dispatch' || inputs.linux }}
  BUILD_WIN: ${{ github.event_name != 'workflow_dispatch' || inputs.win }}
  USE_SUBREPOS: ${{ github.event_name == 'schedule' || github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && ! inputs.pre) }}
  NOTARIZE: ${{ github.event_name == 'schedule' || github.event_name == 'release' || (github.event_name == 'workflow_dispatch' && inputs.pre) }}

jobs:
  build-matrix:
    name: Determine Build Matrix
    runs-on: ubuntu-latest
    outputs:
      target_platform: ${{ steps.build-matrix.outputs.target_platform }}
      include: ${{ steps.build-matrix.outputs.include }}
      python_version: ${{ steps.build-matrix.outputs.python_version }}

    steps:
    - name: Determine Build Matrix
      id: build-matrix
      run: |
        if [[ $BUILD_MAC == "true" ]]; then
            target_platform="'osx-64'"
            include="{'os': 'macos-12', 'target-platform': 'osx-64', 'spyk-arch': 'unix'}"
        fi
        if [[ $BUILD_ARM == "true" ]]; then
            target_platform=${target_platform:+"$target_platform, "}"'osx-arm64'"
            include=${include:+"$include, "}"{'os': 'macos-14', 'target-platform': 'osx-arm64', 'spyk-arch': 'unix'}"
        fi
        if [[ $BUILD_LNX == "true" ]]; then
            target_platform=${target_platform:+"$target_platform, "}"'linux-64'"
            include=${include:+"$include, "}"{'os': 'ubuntu-latest', 'target-platform': 'linux-64', 'spyk-arch': 'unix'}"
        fi
        if [[ $BUILD_WIN == "true" ]]; then
            target_platform=${target_platform:+"$target_platform, "}"'win-64'"
            include=${include:+"$include, "}"{'os': 'windows-latest', 'target-platform': 'win-64', 'spyk-arch': 'win-64'}"
        fi

        echo "target_platform=[$target_platform]" >> $GITHUB_OUTPUT
        echo "include=[$include]" >> $GITHUB_OUTPUT

  build-subrepos:
    name: Build Subrepos
    # env.USE_SUBREPOS is not available at job level; must copy-paste here
    if: github.event_name == 'schedule' || github.event_name == 'pull_request' || (github.event_name == 'workflow_dispatch' && ! inputs.pre)
    uses: ./.github/workflows/build-subrepos.yml

  build-installers:
    name: Build installer for ${{ matrix.target-platform }} Python-${{ matrix.python-version }}
    if: ${{ ! failure() && ! cancelled() }}
    runs-on: ${{ matrix.os }}
    needs:
      - build-matrix
      - build-subrepos
    strategy:
      fail-fast: false
      matrix:
        target-platform: ${{fromJson(needs.build-matrix.outputs.target_platform)}}
        python-version: ['3.11']
        include: ${{fromJson(needs.build-matrix.outputs.include)}}
    defaults:
      run:
        shell: bash -le {0}
        working-directory: ${{ github.workspace }}/installers-conda
    env:
      DISTDIR: ${{ github.workspace }}/installers-conda/dist
      MACOS_CERTIFICATE_PWD: ${{ secrets.MACOS_CERTIFICATE_PWD }}
      MACOS_CERTIFICATE: ${{ secrets.MACOS_CERTIFICATE }}
      MACOS_INSTALLER_CERTIFICATE: ${{ secrets.MACOS_INSTALLER_CERTIFICATE }}
      APPLICATION_PWD: ${{ secrets.APPLICATION_PWD }}
      CONSTRUCTOR_TARGET_PLATFORM: ${{ matrix.target-platform }}

    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Setup Remote SSH Connection
        if: env.ENABLE_SSH == 'true'
        uses: mxschmitt/action-tmate@v3
        timeout-minutes: 10
        with:
          detached: true

      - name: Restore python-lsp-server Cache
        if: env.USE_SUBREPOS == 'true'
        uses: actions/cache/restore@v4
        with:
          path: installers-conda/build/conda-bld/**/*.tar.bz2
          key: python-lsp-server_noarch_${{ matrix.python-version }}_${{ hashFiles('external-deps/python-lsp-server/.gitrepo') }}
          enableCrossOsArchive: true
          fail-on-cache-miss: true

      - name: Restore qtconsole Cache
        if: env.USE_SUBREPOS == 'true'
        uses: actions/cache/restore@v4
        with:
          path: installers-conda/build/conda-bld/**/*.tar.bz2
          key: qtconsole_noarch_${{ matrix.python-version }}_${{ hashFiles('external-deps/qtconsole/.gitrepo') }}
          enableCrossOsArchive: true
          fail-on-cache-miss: true

      - name: Restore ${{ matrix.spyk-arch }} spyder-kernels Cache
        if: env.USE_SUBREPOS == 'true'
        uses: actions/cache/restore@v4
        with:
          path: installers-conda/build/conda-bld/**/*.tar.bz2
          key: spyder-kernels_${{ matrix.spyk-arch }}_${{ matrix.python-version }}_${{ hashFiles('external-deps/spyder-kernels/.gitrepo') }}
          enableCrossOsArchive: true
          fail-on-cache-miss: true

      - name: Setup Build Environment (Windows)
        if: runner.os == 'Windows'
        uses: mamba-org/setup-micromamba@v1
        with:
          condarc: |
            conda_build:
              pkg_format: '2'
              zstd_compression_level: '19'
          environment-file: installers-conda/build-environment.yml
          environment-name: spy-inst
          create-args: >-
            --channel-priority=flexible
            python=${{ matrix.python-version }}
            nsis>=3.08=*_log_*
          cache-downloads: true
          cache-environment: true

      - name: Setup Build Environment (macOS & Linux)
        if: runner.os != 'Windows'
        uses: mamba-org/setup-micromamba@v1
        with:
          condarc: |
            conda_build:
              pkg_format: '2'
              zstd_compression_level: '19'
          environment-file: installers-conda/build-environment.yml
          environment-name: spy-inst
          create-args: >-
            --channel-priority=flexible
            python=${{ matrix.python-version }}
          cache-downloads: true
          cache-environment: true

      - name: Env Variables
        run: |
          NSIS_USING_LOG_BUILD=1
          [[ "$IS_RELEASE" == "true" || "$IS_PRE_RELEASE" == "true" ]] && NSIS_USING_LOG_BUILD=0
          CONDA_BLD_PATH=${RUNNER_TEMP}/conda-bld

          echo "NSIS_USING_LOG_BUILD=$NSIS_USING_LOG_BUILD" >> $GITHUB_ENV
          echo "CONDA_BLD_PATH=$CONDA_BLD_PATH" >> $GITHUB_ENV

          env | sort

      - name: Build ${{ matrix.target-platform }} spyder Conda Package
        run: |
          # Copy built packages to new build location because spyder cannot be
          # built in workspace
          [[ -d build/conda-bld ]] && cp -Rv build/conda-bld $CONDA_BLD_PATH

          python build_conda_pkgs.py --build spyder

      - name: Create Local Conda Channel
        run: |
          conda config --set bld_path $CONDA_BLD_PATH
          conda index $CONDA_BLD_PATH
          mamba search -c local --override-channels || true

      - name: Create Keychain
        if: runner.os == 'macOS' && env.NOTARIZE == 'true'
        run: |
          _codesign=$(which codesign)
          if [[ $_codesign =~ ${CONDA_PREFIX}.* ]]; then
              # Find correct codesign
              echo "Moving $_codesign..."
              mv $_codesign ${_codesign}.bak
          fi

          ./certkeychain.sh "${MACOS_CERTIFICATE_PWD}" "${MACOS_CERTIFICATE}" "${MACOS_INSTALLER_CERTIFICATE}"
          CNAME=$(security find-identity -p codesigning -v | pcre2grep -o1 "\(([0-9A-Z]+)\)")
          echo "CNAME=$CNAME" >> $GITHUB_ENV

      - name: Load signing certificate (Windows)
        if: runner.os == 'Windows' && env.NOTARIZE == 'true'
        run: |
          echo "${MACOS_CERTIFICATE}" > "${{ runner.temp }}/certificate.b64.txt"
          certutil.exe -decode "${{ runner.temp }}/certificate.b64.txt" "${{ runner.temp }}/certificate.pfx"

          echo "CONSTRUCTOR_SIGNING_CERTIFICATE=${{ runner.temp }}/certificate.pfx" >> $GITHUB_ENV
          echo "CONSTRUCTOR_PFX_CERTIFICATE_PASSWORD=${MACOS_CERTIFICATE_PWD}" >> $GITHUB_ENV
          echo "CONSTRUCTOR_SIGNTOOL_PATH=C:/Program Files (x86)/Windows Kits/10/bin/10.0.17763.0/x86/signtool.exe" >> $GITHUB_ENV

      - name: Build Package Installer
        run: |
          [[ -n $CNAME ]] && args=("--cert-id" "$CNAME") || args=()
          python build_installers.py ${args[@]}

          PKG_NAME=$(ls $DISTDIR | grep Spyder-)
          LCK_NAME=$(ls $DISTDIR | grep conda-)
          echo "PKG_NAME=$PKG_NAME" >> $GITHUB_ENV
          echo "LCK_NAME=$LCK_NAME" >> $GITHUB_ENV
          echo "ARTIFACT_NAME=${PKG_NAME%.*}" >> $GITHUB_ENV
          echo "PKG_PATH=$DISTDIR/$PKG_NAME" >> $GITHUB_ENV

      - name: Test macOS or Linux Installer
        if: runner.os != 'Windows'
        run: ${{ github.workspace }}/.github/scripts/installer_test.sh

      - name: Test Windows Installer
        if: runner.os == 'Windows'
        shell: cmd
        run: |
          set base_prefix=%USERPROFILE%\AppData\Local\spyder-6
          start /wait %PKG_PATH% /InstallationType=JustMe /NoRegistry=1 /S
          if exist %base_prefix%\install.log type %base_prefix%\install.log

          set mode=system
          for /F "tokens=*" %%i in (
              '%base_prefix%\python %base_prefix%\Scripts\menuinst_cli.py shortcut --mode=%mode%'
          ) do (
              set shortcut=%%~fi
          )
          if exist "%shortcut%" (
              echo "Spyder installed successfully"
          ) else (
              echo "Spyder NOT installed successfully"
              EXIT /B 1
          )
          EXIT /B %ERRORLEVEL%

      - name: Notarize or Compute Checksum
        if: env.NOTARIZE == 'true'
        run: |
          if [[ $RUNNER_OS == "macOS" ]]; then
              ./notarize.sh -p $APPLICATION_PWD $PKG_PATH
          else
              cd $DISTDIR
              echo $(sha256sum $PKG_NAME) > "${ARTIFACT_NAME}-sha256sum.txt"
          fi

      - name: Upload Artifact
        if: env.IS_RELEASE == 'false'
        uses: actions/upload-artifact@v4
        with:
          path: ${{ env.DISTDIR }}
          name: ${{ env.ARTIFACT_NAME }}

      - name: Get Release
        if: env.IS_RELEASE == 'true'
        uses: bruceadams/get-release@v1.3.2
        id: get_release
        env:
          GITHUB_TOKEN: ${{ github.token }}

      - name: Upload Release Asset
        if: env.IS_RELEASE == 'true'
        uses: shogo82148/actions-upload-release-asset@v1
        env:
          GITHUB_TOKEN: ${{ github.token }}
        with:
          upload_url: ${{ steps.get_release.outputs.upload_url }}
          asset_path: ${{ env.DISTDIR }}/*.*


File tests

name: File tests

on:
  push:
    branches:
      - master
      - 5.*
      - 4.x
    paths:
      - '.github/scripts/*.sh'
      - '.github/workflows/*.yml'
      - 'requirements/*.yml'
      - 'MANIFEST.in'
      - '**.bat'
      - '**.py'
      - '**.sh'
      - '!installers-conda/**'
      - '!.github/workflows/installers-conda.yml'
      - '!.github/workflows/build-subrepos.yml'

  pull_request:
    branches:
      - master
      - 5.*
      - 4.x
    paths:
      - '.github/scripts/*.sh'
      - '.github/workflows/*.yml'
      - 'requirements/*.yml'
      - 'MANIFEST.in'
      - '**.bat'
      - '**.py'
      - '**.sh'
      - '!installers-conda/**'
      - '!.github/workflows/installers-conda.yml'
      - '!.github/workflows/build-subrepos.yml'

  workflow_dispatch:

concurrency:
  group: test-files-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build:
    # Use this to disable the workflow
    # if: false
    name: Linux - Py${{ matrix.PYTHON_VERSION }}
    runs-on: ubuntu-latest
    env:
      CI: 'true'
      USE_CONDA: 'true'
      OS: 'linux'
      PYTHON_VERSION: ${{ matrix.PYTHON_VERSION }}
    strategy:
      fail-fast: false
      matrix:
        PYTHON_VERSION: ['3.9', '3.10']
    timeout-minutes: 30
    steps:
      - name: Checkout Pull Requests
        if: github.event_name == 'pull_request'
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - name: Checkout Push
        if: github.event_name != 'pull_request'
        uses: actions/checkout@v4
      - name: Fetch branches
        if: github.event_name == 'pull_request'
        run: git fetch --prune --unshallow
      - name: Install dependencies
        shell: bash
        run: |
          sudo apt-get update --fix-missing
          sudo apt-get install -qq pyqt5-dev-tools libxcb-xinerama0 xterm --fix-missing
      - name: Cache conda
        uses: actions/cache@v4
        env:
          # Increase this value to reset cache if requirements/*.yml has not changed
          CACHE_NUMBER: 0
        with:
          path: ~/conda_pkgs_dir
          key: ${{ runner.os }}-cacheconda-installconda-${{ matrix.PYTHON_VERSION }}-${{ env.CACHE_NUMBER }}-${{ hashFiles('requirements/*.yml') }}
      - name: Cache pip
        uses: actions/cache@v4
        with:
          path: ~/.cache/pip
          key: ${{ runner.os }}-cachepip-installconda-${{ env.CACHE_NUMBER }}-${{ hashFiles('setup.py') }}
      - name: Create conda test environment
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: requirements/main.yml
          environment-name: test
          cache-downloads: true
          create-args: python=${{ matrix.PYTHON_VERSION }}
      - name: Install additional dependencies
        shell: bash -l {0}
        run: bash -l .github/scripts/install.sh
      - name: Show test environment
        shell: bash -l {0}
        run: |
          conda info
          conda list
      - name: Run file tests
        shell: bash -l {0}
        run: xvfb-run --auto-servernum .github/scripts/modules_test.sh || xvfb-run --auto-servernum .github/scripts/modules_test.sh
      # Uncomment this to debug the CI job
      # - name: Setup tmate session
      #   if: ${{ failure() }}
      #   uses: mxschmitt/action-tmate@v3