ci-tests

# Reference:
#   - https://github.com/actions/checkout
#   - https://github.com/actions/download-artifact
#   - https://github.com/actions/setup-python
#   - https://github.com/actions/upload-artifact
#   - https://github.com/awalsh128/cache-apt-pkgs-action
#   - https://github.com/codecov/codecov-action
#   - https://github.com/conda-incubator/setup-miniconda

name: ci-tests

on:
  pull_request:

  push:
    branches:
      - "main"
      - "v*x"
      - "!conda-lock-auto-update"
      - "!pre-commit-ci-update-config"
      - "!dependabot/*"
    tags:
      - "v*"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  tests:
    name: "${{ matrix.session }} ${{ matrix.marker }} (${{ matrix.version }})"

    runs-on: ${{ matrix.os }}

    defaults:
      run:
        shell: bash -l {0}

    env:
      CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
      ENV_NAME: "ci-tests"

    strategy:
      fail-fast: false
      matrix:
        os: ["ubuntu-latest"]
        version: ["py310", "py311", "py312"]
        session: ["tests"]
        marker: ["image", "not image"]
        posargs: ["--xvfb-backend xvfb --durations=10"]
        include:
          - version: "py312"
            coverage: "--cov-report=xml --cov=geovista"

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - name: "environment configure"
      env:
        # Maximum cache period (in weeks) before forcing a cache refresh.
        CACHE_WEEKS: 2
      run: |
        echo "CACHE_PERIOD=$(date +%Y).$(expr $(date +%U) / ${CACHE_WEEKS})" >> ${GITHUB_ENV}
        echo "LOCK_FILE=requirements/locks/${{ matrix.version }}-lock-linux-64.txt" >> ${GITHUB_ENV}

    - name: "conda package cache"
      uses: ./.github/workflows/composite/conda-pkg-cache
      with:
        cache_period: ${{ env.CACHE_PERIOD }}
        env_name: ${{ env.ENV_NAME }}

    - name: "conda install"
      uses: conda-incubator/setup-miniconda@v3
      with:
        miniforge-version: latest
        channels: conda-forge,defaults
        channel-priority: true
        auto-update-conda: true
        activate-environment: ${{ env.ENV_NAME }}
        use-only-tar-bz2: false

    - name: "conda environment cache"
      uses: ./.github/workflows/composite/conda-env-cache
      with:
        cache_period: ${{ env.CACHE_PERIOD }}
        env_name: ${{ env.ENV_NAME }}
        install_packages: "cartopy pip 'tox<4'"

    - name: "conda info"
      run: |
        conda info
        conda list

    - name: "cartopy cache"
      uses: ./.github/workflows/composite/cartopy-cache
      with:
        cache_period: ${{ env.CACHE_PERIOD }}
        env_name: ${{ env.ENV_NAME }}

    - name: "tox cache"
      uses: ./.github/workflows/composite/tox-cache
      with:
        lock_file: ${{ env.LOCK_FILE }}

    - name: "apt cache"
      uses: awalsh128/cache-apt-pkgs-action@v1.4.2
      with:
        packages: libgl1-mesa-glx xvfb
        version: 1.0

    - name: "geovista cache"
      uses: ./.github/workflows/composite/geovista-cache
      with:
        cache_period: ${{ env.CACHE_PERIOD }}

    - name: "${{ matrix.session }} ${{ matrix.marker }} (${{ matrix.version }})"
      env:
        PY_COLORS: "1"
      run: |
        tox -e ${{ matrix.version }}-${{ matrix.session }} -- -m "${{ matrix.marker }}" ${{ matrix.coverage }} ${{ matrix.posargs }}

    - name: "upload test images"
      if: ${{ failure() }}
      uses: actions/upload-artifact@v4
      with:
        name: test-images-${{ github.job }}-${{ strategy.job-index }}
        path: ${{ github.workspace }}/test_images

    - name: "prepare coverage"
      if: ${{ matrix.coverage }}
      run: |
        mv .coverage .coverage${{ strategy.job-index }}

    - if: ${{ matrix.coverage }}
      uses: actions/upload-artifact@v4
      with:
        name: coverage-artifacts-${{ github.job }}-${{ strategy.job-index }}
        path: ${{ github.workspace }}/.coverage*

  coverage:
    name: "upload coverage"
    needs: tests
    runs-on: ubuntu-latest

    defaults:
      run:
        shell: bash -l {0}

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - run: |
          python -m pip install --upgrade pip
          pip install coverage

      - uses: actions/download-artifact@v4
        with:
          pattern: coverage-artifacts-*
          merge-multiple: true

      - name: "create coverage report"
        run: |
          coverage combine .coverage*
          coverage xml --omit=*/_version.py

      - name: "upload coverage report"
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}


ci-wheels

# Reference:
#   - https://github.com/actions/cache
#   - https://github.com/actions/checkout
#   - https://github.com/actions/download-artifact
#   - https://github.com/actions/upload-artifact
#   - https://github.com/conda-incubator/setup-miniconda
#   - https://github.com/pypa/build
#   - https://github.com/pypa/gh-action-pypi-publish
#   - https://test.pypi.org/help/#apitoken

name: ci-wheels

on:
  pull_request:

  push:
    branches:
      - "main"
      - "v*x"
      - "!conda-lock-auto-update"
      - "!pre-commit-ci-update-config"
      - "!dependabot/*"
    tags:
      - "v*"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build-artifacts:
    name: "build pypi artifacts"

    runs-on: ubuntu-latest

    defaults:
      run:
        shell: bash -l {0}

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - name: "build sdist and wheel"
      run: |
        # geovista is a pure python package, so simply use pypa/build
        pipx run build

    - name: "show sdist and wheel"
      run: |
        ls -l ${{ github.workspace }}/dist

    - uses: actions/upload-artifact@v4
      with:
        name: pypi-artifacts-${{ github.job }}-${{ strategy.job-index }}
        path: ${{ github.workspace }}/dist

  test-artifacts:
    needs: [build-artifacts]

    name: "test wheel (${{ matrix.version }})"

    runs-on: ubuntu-latest

    defaults:
      run:
        shell: bash -l {0}

    env:
      ENV_NAME: "ci-wheels"

    strategy:
      fail-fast: false
      matrix:
        version: ["py310", "py311", "py312"]
        session: ["env"]

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - uses: actions/download-artifact@v4
      with:
        pattern: pypi-artifacts-*
        path: ${{ github.workspace }}/dist
        merge-multiple: true

    - name: "environment configure"
      env:
        # Maximum cache period (in weeks) before forcing a cache refresh.
        CACHE_WEEKS: 2
      run: |
        echo "CACHE_PERIOD=$(date +%Y).$(expr $(date +%U) / ${CACHE_WEEKS})" >> ${GITHUB_ENV}
        echo "LOCK_FILE=requirements/locks/${{ matrix.version }}-lock-linux-64.txt" >> ${GITHUB_ENV}

    - name: "conda package cache"
      uses: ./.github/workflows/composite/conda-pkg-cache
      with:
        cache_period: ${{ env.CACHE_PERIOD }}
        env_name: ${{ env.ENV_NAME }}

    - name: "conda install"
      uses: conda-incubator/setup-miniconda@v3
      with:
        miniforge-version: latest
        channels: conda-forge,defaults
        channel-priority: true
        auto-update-conda: true
        activate-environment: ${{ env.ENV_NAME }}
        use-only-tar-bz2: false

    - name: "conda environment cache"
      uses: ./.github/workflows/composite/conda-env-cache
      with:
        cache_period: ${{ env.CACHE_PERIOD }}
        env_name: ${{ env.ENV_NAME }}
        install_packages: "pip 'tox<4'"

    - name: "conda info"
      run: |
        conda info
        conda list

    - name: "tox cache"
      uses: ./.github/workflows/composite/tox-cache
      with:
        lock_file: ${{ env.LOCK_FILE }}

    - name: "test wheel (${{ matrix.version }})"
      env:
        TESTENV: "${{ matrix.version }}-${{ matrix.session }}"
      run: |
        WHEEL=$(ls -1 dist/*.whl)
        tox -e ${{ env.TESTENV }} --run-command "python -m pip uninstall --yes ${WHEEL}"
        tox -e ${{ env.TESTENV }} --run-command "python -m pip install --no-deps ${WHEEL}"
        tox -e ${{ env.TESTENV }} --run-command "python -c 'import geovista; print(geovista.__version__)'"

  show-artifacts:
    needs: [build-artifacts]
    name: "show artifacts"
    runs-on: ubuntu-latest
    steps:
    - uses: actions/download-artifact@v4
      with:
        pattern: pypi-artifacts-*
        path: ${{ github.workspace }}/dist
        merge-multiple: true

    - shell: bash
      run: |
        ls -l ${{ github.workspace }}/dist

  publish-artifacts-test-pypi:
    needs: [test-artifacts]
    name: "Publish to Test PyPI"
    runs-on: ubuntu-latest
    # upload to Test PyPI for every commit on main branch
    if: github.event_name == 'push' && github.event.ref == 'refs/heads/main'
    steps:
    - uses: actions/download-artifact@v4
      with:
        pattern: pypi-artifacts-*
        path: ${{ github.workspace }}/dist
        merge-multiple: true

    - uses: pypa/gh-action-pypi-publish@release/v1
      with:
        user: __token__
        password: ${{ secrets.TEST_PYPI_API_TOKEN }}
        repository_url: https://test.pypi.org/legacy/
        skip_existing: true
        print_hash: true

  publish-artifacts-pypi:
    needs: [test-artifacts]
    name: "Publish to PyPI"
    runs-on: ubuntu-latest
    # upload to PyPI for every tag starting with 'v'
    if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/v')
    steps:
    - uses: actions/download-artifact@v4
      with:
        pattern: pypi-artifacts-*
        path: ${{ github.workspace }}/dist
        merge-multiple: true

    - uses: pypa/gh-action-pypi-publish@release/v1
      with:
        user: __token__
        password: ${{ secrets.PYPI_API_TOKEN }}
        print_hash: true


CodeQL

# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"

on:
  push:
    branches: [ "main" ]
  pull_request:
    # The branches below must be a subset of the branches above
    branches: [ "main" ]
  schedule:
    - cron: '19 1 * * 3'

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write

    strategy:
      fail-fast: false
      matrix:
        language: [ 'python' ]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
        # Use only 'java' to analyze code written in Java, Kotlin or both
        # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both
        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v3
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
        # By default, queries listed here will override any specified in a config file.
        # Prefix the list here with "+" to use these queries and those in the config file.

        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
        # queries: security-extended,security-and-quality

    # Autobuild attempts to build any compiled languages  (C/C++, C#, Go, or Java).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
      uses: github/codeql-action/autobuild@v3

    # ℹ Command-line programs to run using the OS shell.
    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun

    #   If the Autobuild fails above, remove it and uncomment the following three lines.
    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.

    # - run: |
    #   echo "Run, Build Application using script"
    #   ./location_of_script_within_repo/buildscript.sh

    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v3
      with:
        category: "/language:${{matrix.language}}"


Nightly Release

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Nightly Release

on:
  schedule:
    - cron: '0 5 * * *'
  workflow_dispatch:

jobs:
  windows-build:
    runs-on: 7950X
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11"]

    steps:
    - uses: actions/checkout@v2
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v3
      with:
        python-version: ${{ matrix.python-version }}

    - name: Compute version
      shell: powershell
      run: |
        $package_version = $(Get-Date -UFormat "%Y%m%d")+"."+${{ github.run_number }}
        $package_version_ = $(Get-Date -UFormat "%Y%m%d")+"_"+${{ github.run_number }}
        $tag_name=$package_version
        echo "package_version=$package_version" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
        echo "package_version_=$package_version_" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append
        echo "tag_name=$tag_name" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf8 -Append

    - name: Create Release
      id: create_release
      uses: actions/create-release@v1
      env:
        GITHUB_TOKEN: ${{ secrets.NODAI_INVOCATION_TOKEN }}
      with:
        tag_name: ${{ env.tag_name }}
        release_name: nod.ai SHARK ${{ env.tag_name }}
        body: |
          Automatic snapshot release of nod.ai SHARK.
        draft: true
        prerelease: true

    - name: Build Package
      shell: powershell
      run: |
        ./setup_venv.ps1
        python process_skipfiles.py
        $env:SHARK_PACKAGE_VERSION=${{ env.package_version }}
        pip install -e .
        pip freeze -l
        pyinstaller .\apps\shark_studio\shark_studio.spec
        mv ./dist/nodai_shark_studio.exe ./dist/nodai_shark_studio_${{ env.package_version_ }}.exe
        signtool sign /f c:\g\shark_02152023.cer /fd certHash /csp "eToken Base Cryptographic Provider" /k "${{ secrets.CI_CERT }}" ./dist/nodai_shark_studio_${{ env.package_version_ }}.exe

    - name: Upload Release Assets
      id: upload-release-assets
      uses: dwenegar/upload-release-assets@v1
      env:
        GITHUB_TOKEN: ${{ secrets.NODAI_INVOCATION_TOKEN }}
      with:
        release_id: ${{ steps.create_release.outputs.id }}
        assets_path: ./dist/nodai*
        #asset_content_type: application/vnd.microsoft.portable-executable

    - name: Publish Release
      id: publish_release
      uses: eregon/publish-release@v1
      env:
        GITHUB_TOKEN: ${{ secrets.NODAI_INVOCATION_TOKEN }}
      with:
        release_id: ${{ steps.create_release.outputs.id }}


Publish releases page

# See: https://github.com/llvm/torch-mlir/issues/1374
name: Publish releases page

on:
  workflow_dispatch:

jobs:
  scrape_and_publish_releases:
    name: "Scrape and publish releases"
    runs-on: ubuntu-latest

    # Don't run this in everyone's forks.
    if: github.repository == 'nod-ai/SHARK'

    steps:
      - name: Checking out repository
        uses: actions/checkout@v2
        with:
          token: ${{ secrets.NODAI_INVOCATION_TOKEN }}
      - name: Run scrape releases script
        run: python ./build_tools/scrape_releases.py nod-ai SHARK > /tmp/index.html
        shell: bash
      - run: git fetch --all
      - run: git switch github-pages
      - run: git config --global user.email "none@none.com"
      - run: git config --global user.name "nod-ai"
      - run: mv /tmp/index.html package-index/index.html
      - run: git add package-index/index.html

      # Only try to make a commit if the file has changed.
      - run: git diff --cached --exit-code || git commit -m "Update releases."

      - name: GitHub Push
        uses: ad-m/github-push-action@v0.6.0
        with:
          github_token: ${{ secrets.NODAI_INVOCATION_TOKEN }}
          branch: github-pages


Validate Shark Studio

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Validate Shark Studio

on:
  push:
    branches: [ main ]
    paths-ignore:
      - '**.md'
      - 'shark/examples/**'
  pull_request:
    branches: [ main ]
    paths-ignore:
      - '**.md'
      - 'shark/examples/**'
  workflow_dispatch:

# Ensure that only a single job or workflow using the same
# concurrency group will run at a time. This would cancel
# any in-progress jobs in the same github workflow and github
# ref (e.g. refs/heads/main or refs/pull/<pr_number>/merge).
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build-validate:
    strategy:
      fail-fast: true
      matrix:
        os: [nodai-ubuntu-builder-large]
        suite: [cpu] #,cuda,vulkan]
        python-version: ["3.11"]
        include:
          - os: nodai-ubuntu-builder-large
            suite: lint

    runs-on: ${{ matrix.os }}

    steps:
    - uses: actions/checkout@v3

    - name: Set Environment Variables
      run: |
        echo "SHORT_SHA=`git rev-parse --short=4 HEAD`" >> $GITHUB_ENV
        echo "DATE=$(date +'%Y-%m-%d')" >> $GITHUB_ENV

    - name: Set up Python Version File ${{ matrix.python-version }}
      run: |
        echo ${{ matrix.python-version }} >> $GITHUB_WORKSPACE/.python-version

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: '${{ matrix.python-version }}'

    - name: Install dependencies
      if: matrix.suite == 'lint'
      run: |
        python -m pip install --upgrade pip
        python -m pip install flake8 pytest toml black

    - name: Lint with flake8
      if: matrix.suite == 'lint'
      run: |
        # black format check
        black --version
        black --check apps/shark_studio
        # stop the build if there are Python syntax errors or undefined names
        flake8 . --statistics
        # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
        flake8 . --isolated --count --exit-zero --max-complexity=10 --max-line-length=127 \
          --statistics --exclude lit.cfg.py

    - name: Validate Models on CPU
      if: matrix.suite == 'cpu'
      run: |
        cd $GITHUB_WORKSPACE
        python${{ matrix.python-version }} -m venv shark.venv
        source shark.venv/bin/activate
        pip install -r requirements.txt --no-cache-dir
        pip install -e .
        # Disabled due to hang when exporting test llama2
        # python apps/shark_studio/tests/api_test.py


installation

name: installation
on:
  pull_request:
    paths:
      - 'src/**'
      - 'install.py'
      - 'setup.py'
      - 'MANIFEST.in'
      - 'pyproject.toml'
      - '.github/workflows/installation.yaml'
      - '!src/rez/utils/_version.py'
      - '!**.md'
  push:
    paths:
      - 'src/**'
      - 'install.py'
      - 'setup.py'
      - 'MANIFEST.in'
      - 'pyproject.toml'
      - '.github/workflows/installation.yaml'
      - '!src/rez/utils/_version.py'
      - '!**.md'
  workflow_dispatch:

jobs:
  main:
    name: ${{ matrix.os }} - ${{ matrix.python-version }} - ${{ matrix.method }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
        python-version: ['3.8', '3.9', '3.10', '3.11']
        method: ['install' ,'pip']

        include:
        # ubuntu
        - os: ubuntu-latest
          method: install
          REZ_SET_PATH_COMMAND: 'export PATH=${PATH}:/opt/rez/bin/rez'
          REZ_INSTALL_COMMAND: python ./install.py /opt/rez
        - os: ubuntu-latest
          method: pip
          REZ_SET_PATH_COMMAND: 'export PATH=${PATH}:/opt/rez/bin PYTHONPATH=${PYTHONPATH}:/opt/rez'
          REZ_INSTALL_COMMAND: pip install --target /opt/rez .
        # macOS
        - os: macos-latest
          method: install
          REZ_SET_PATH_COMMAND: 'export PATH=${PATH}:~/rez/bin/rez'
          REZ_INSTALL_COMMAND: python ./install.py ~/rez
        - os: macos-latest
          method: pip
          REZ_SET_PATH_COMMAND: 'export PATH="$PATH:~/rez/bin" PYTHONPATH=$PYTHONPATH:$HOME/rez'
          REZ_INSTALL_COMMAND: pip install --target ~/rez .
        # macOS
        # Python 3.7 is not supported on Apple Silicon.
        # macos-13 is the last macos runner image to run on Intel CPUs.
        - os: macos-13
          python-version: '3.7'
          method: install
          REZ_SET_PATH_COMMAND: 'export PATH=${PATH}:~/rez/bin/rez'
          REZ_INSTALL_COMMAND: python ./install.py ~/rez
        - os: macos-13
          python-version: '3.7'
          method: pip
          REZ_SET_PATH_COMMAND: 'export PATH="$PATH:~/rez/bin" PYTHONPATH=$PYTHONPATH:$HOME/rez'
          REZ_INSTALL_COMMAND: pip install --target ~/rez .
        # windows
        - os: windows-latest
          method: install
          REZ_SET_PATH_COMMAND: '$env:PATH="$env:PATH;C:\ProgramData\rez\Scripts\rez"'
          REZ_INSTALL_COMMAND: python ./install.py C:\ProgramData\rez
        - os: windows-latest
          method: pip
          REZ_SET_PATH_COMMAND: '[System.Environment]::SetEnvironmentVariable("PATH","$env:PATH;C:\ProgramData\rez\bin"); $env:PYTHONPATH="$env:PYTHONPATH;C:\ProgramData\rez"'
          REZ_INSTALL_COMMAND: pip install --target C:\ProgramData\rez .

    steps:
    - uses: actions/checkout@v4

    - name: Setup python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install
      env:
        MATRIX_PYTHON_VERSION: ${{ matrix.python-version }}
      run: |
        ${{ matrix.REZ_INSTALL_COMMAND }}

    - name: Run rez-status
      env:
        MATRIX_PYTHON_VERSION: ${{ matrix.python-version }}
      run: |
        ${{ matrix.REZ_SET_PATH_COMMAND }}
        rez-status

    - name: Install rez with rez-pip
      env:
        MATRIX_PYTHON_VERSION: ${{ matrix.python-version }}
      run: |
        ${{ matrix.REZ_SET_PATH_COMMAND }}
        rez-pip --install .


pypi

name: pypi
on:
  release:
    types: [released]

jobs:
  publish:
    name: Publish to PyPI
    runs-on: ubuntu-latest

    permissions:
      # IMPORTANT: this permission is mandatory for trusted publishing
      id-token: write

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: 3.11

      - name: Install Dependencies
        run: |
          pip install build

      - name: Build rez
        run: |
          python -m build --sdist --wheel --outdir dist .

      # Note that we don't need credentials.
      # We rely on https://docs.pypi.org/trusted-publishers/.
      - name: Upload to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          packages-dir: dist


benchmark

# Run resolving benchmark and store results. Fail if any resolved runtimes
# change from previous result
#
name: benchmark
on:
  release:
    types: [released]
  pull_request:
    types: [opened, synchronize, reopened, labeled]

jobs:
  run_benchmark:
    name: run_benchmark
    runs-on: ubuntu-latest

    if: ${{ github.event_name == 'release' || contains(github.event.pull_request.labels.*.name, 'run-benchmarks') }}

    strategy:
      matrix:
        python-version:
          - '3.7'

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Setup python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Rez
        run: |
          mkdir ./installdir

          python ./install.py ./installdir

      - name: Run Benchmark
        run: |
          ./installdir/bin/rez/rez-benchmark --out ./out

          # remove benchmarking suite package repo
          rm -rf ./out/packages

      - name: Validate Result
        run: |
          python ./.github/scripts/validate_benchmark.py

      - uses: actions/upload-artifact@v4
        with:
          name: "benchmark-result-${{ matrix.python-version }}"
          path: ./out

  store_benchmark_result:
    name: store_benchmark_result
    runs-on: ubuntu-latest
    needs: run_benchmark

    strategy:
      matrix:
        python-version:
          - '3.7'

      # so we don't have jobs trying to push to git at the same time
      max-parallel: 1

    steps:
      - uses: actions/download-artifact@v4
        with:
          name: "benchmark-result-${{ matrix.python-version }}"
          path: .

      - name: Checkout (release)
        uses: actions/checkout@v4
        if: ${{ github.event_name =='release' }}
        with:
          ref: main
          path: src

          # This is a PAT from an Admin user. We do this in order to be able to write to a
          # protected branch (main) from this workflow.
          # See https://github.community/t/how-to-push-to-protected-branches-in-a-github-action/16101/14
          #
          # Disable for now until we find a better solution.
          # token: "${{ secrets.GH_ACTION_TOKEN }}"

      - name: Checkout (pr)
        uses: actions/checkout@v4
        if: ${{ github.event_name !='release' }}
        with:
          path: src

      - name: Setup python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      # Note failing due to
      # https://github.com/actions/virtual-environments/issues/675
      #
      # We'll just skip on failure, this just means the gnuplot doesn't render
      #
      - name: Install gnuplot
        run: |
          sudo apt-get update
          sudo apt-get install -y gnuplot || /bin/true

      - name: Store Benchmark Result
        run: |
          python ./.github/scripts/store_benchmark.py
        working-directory: src

      - name: Create summary
        run: |
          echo '' >> $GITHUB_STEP_SUMMARY
          echo 'Results' >> $GITHUB_STEP_SUMMARY
          cat metrics/benchmarking/RESULTS.md >> $GITHUB_STEP_SUMMARY
          echo '' >> $GITHUB_STEP_SUMMARY
        working-directory: src

      # - name: Setup git config
      #   if: ${{ github.event_name == 'release' }}
      #   run: |
      #     git config user.name 'github-actions[bot]'
      #     git config user.email 'github-actions[bot]@users.noreply.github.com'
      #   working-directory: src

      # - name: Git commit and push
      #   if: ${{ github.event_name == 'release' }}
      #   run: |
      #     if [[ "$(git status --porcelain)" == "" ]]; then
      #       echo "Nothing new to commit"
      #     else
      #       git add --all
      #       git commit -m "Generated from GitHub "${{ github.workflow }}" Workflow"
      #       git push origin main
      #     fi
      #   working-directory: src


name: flake8
on:
  pull_request:
    paths:
      - 'src/rez/**.py'
      - 'src/rezplugins/**.py'
      - '.github/workflows/flake8.yaml'
      - '!src/rez/utils/_version.py'
      - '!src/rez/data/**'
      - '!src/rez/vendor/**'
      - '!src/rez/backport/**'
  push:
    paths:
      - 'src/rez/**.py'
      - 'src/rezplugins/**.py'
      - '.github/workflows/flake8.yaml'
      - '!src/rez/utils/_version.py'
      - '!src/rez/data/**'
      - '!src/rez/vendor/**'
      - '!src/rez/backport/**'

jobs:
  lint:
    name: Run Linter
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: 3.7

      - name: Install Dependencies
        run: |
          pip install flake8

      - name: Run flake8
        run: >-
          find -name '*.py'
          -not -path './rez/vendor/*'
          -not -path './rez/data/*'
          -not -path './rez/backport/*'
          -not -path './build_utils/*'
          -not -path './support/*'
          -not -path './rezgui/*'
          | xargs flake8
        working-directory: src


copyright

name: copyright
on:
  pull_request:
    paths:
      - 'src/**'
      - '!**.md'
  push:
    paths:
      - 'src/**'
      - '!**.md'
  workflow_dispatch:

jobs:
  main:
    name: Enforce copyright notices
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: 3

      - name: Run copyright checker
        run: |
          bash ./src/build_utils/license/apply_copyright
          if [[ "$(git status | grep modified)" != "" ]]; then
            echo "Some sourcefiles are missing copyright notice!" 1>&2
            echo "Run ./src/build_utils/license/apply_copyright to apply." 1>&2
            exit 1
          fi


slack-notification

name: slack-notification

on:
  release:
    types:
      # published should cover both 'released' and 'prereleased'
      - published

jobs:
  publish:
    runs-on: ubuntu-latest
    timeout-minutes: 5
    steps:
    - name: Notify Slack
      id: slack
      with:
        project_name: "rez"
        slack_bot_token: ${{ secrets.SLACK_BOT_TOKEN }}
        slack_channel: "#release-announcements"
        project_logo: "https://artwork.aswf.io/projects/rez/icon/black/rez-icon-black.png"
      uses: jmertic/slack-release-notifier@main


tests

name: tests
on:
  pull_request:
    paths:
    - 'src/**'
    - 'install.py'
    - 'setup.py'
    - 'MANIFEST.in'
    - 'pyproject.toml'
    - '.github/workflows/tests.yaml'
    - '!src/rez/utils/_version.py'
    - '!**.md'
  push:
    paths:
    - 'src/**'
    - 'install.py'
    - 'setup.py'
    - 'MANIFEST.in'
    - 'pyproject.toml'
    - '.github/workflows/tests.yaml'
    - '!src/rez/utils/_version.py'
    - '!**.md'
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  core:
    name: Tests

    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        os: ['macos-latest', 'ubuntu-latest', 'windows-latest']
        python-version: ['3.8', '3.9', '3.10', '3.11']
        include:
        - os: macos-latest
          shells: 'sh,csh,bash,tcsh,zsh,pwsh'
          rez-path: /installdir/bin/rez
        - os: ubuntu-latest
          shells: 'sh,csh,bash,tcsh,zsh,pwsh'
          rez-path: /installdir/bin/rez
        - os: windows-latest
          shells: 'cmd,pwsh,gitbash'
          rez-path: \installdir\Scripts\rez
        # Python 3.7 is not supported on Apple Silicon.
        # macos-13 is the last macos runner image to run on Intel CPUs.
        - os: macos-13
          shells: 'sh,csh,bash,tcsh,zsh,pwsh'
          rez-path: /installdir/bin/rez
          python-version: '3.7'

      fail-fast: false

    steps:
    - name: Checkout
      uses: actions/checkout@v4

    - name: Setup python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install test system dependencies (linux)
      if: ${{ startsWith(matrix.os, 'ubuntu-') }}
      run: |
        set -ex
        sudo apt-get update

        cmake --version
        pwsh --version

        sudo apt-get install -y csh tcsh zsh

    - name: Install rez
      run: python ./install.py ./installdir

    - name: Setup environment variables
      shell: bash
      env:
        _GH_REZ_INSTALL_PATH: ${{ github.workspace }}${{ matrix.rez-path }}
      run: |
        echo "$_GH_REZ_INSTALL_PATH" >> $GITHUB_PATH

    - name: Install test dependencies
      run: rez-python -m pip install pytest-cov parameterized

    - name: Run tests
      id: tests
      run: rez-selftest -v -- --cov=rez --cov-report=xml:coverage.xml
      env:
        _REZ_ENSURE_TEST_SHELLS: ${{ matrix.shells }}

    - name: Upload coverage reports to Codecov
      uses: codecov/codecov-action@v4
      # Run on both success and failure, but only if coverage.xml exists.
      if: ${{ hashFiles('coverage.xml') != '' && (steps.tests.outcome == 'success' || steps.tests.outcome == 'failure') }}
      with:
        slug: AcademySoftwareFoundation/rez
        files: 'coverage.xml'
      env:
        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}


OpenSSF Scorecard

name: OpenSSF Scorecard
on:
  # For Branch-Protection check. Only the default branch is supported. See
  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
  branch_protection_rule:
  # To guarantee Maintained check is occasionally updated. See
  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
  schedule:
    - cron: '21 4 * * 3'
  push:
    branches: [ "main" ]

# Declare default permissions as read only.
permissions: read-all

jobs:
  analysis:
    name: Scorecard analysis
    runs-on: ubuntu-latest
    permissions:
      # Needed to upload the results to code-scanning dashboard.
      security-events: write
      # Needed to publish results and get a badge (see publish_results below).
      id-token: write

    steps:
      - name: "Checkout code"
        uses: actions/checkout@b4ffde65f46336ab88eb53be808477a3936bae11 # v4.1.1
        with:
          persist-credentials: false

      - name: "Run analysis"
        uses: ossf/scorecard-action@dc50aa9510b46c811795eb24b2f1ba02a914e534 # v2.3.3
        with:
          results_file: results.sarif
          results_format: sarif
          # (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
          # - you want to enable the Branch-Protection check on a *public* repository, or
          # - you are installing Scorecard on a *private* repository
          # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action?tab=readme-ov-file#authentication-with-fine-grained-pat-optional.
          # repo_token: ${{ secrets.SCORECARD_TOKEN }}

          # Public repositories:
          #   - Publish results to OpenSSF REST API for easy access by consumers
          #   - Allows the repository to include the Scorecard badge.
          #   - See https://github.com/ossf/scorecard-action#publishing-results.
          # For private repositories:
          #   - `publish_results` will always be set to `false`, regardless
          #     of the value entered here.
          publish_results: true

      # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
      # format to the repository Actions tab.
      - name: "Upload artifact"
        uses: actions/upload-artifact@1746f4ab65b179e0ea60a494b83293b640dd5bba # v4.3.2
        with:
          name: SARIF file
          path: results.sarif
          retention-days: 5

      # Upload the results to GitHub's code scanning dashboard (optional).
      # Commenting out will disable upload of results to your repo's Code Scanning dashboard
      - name: "Upload to code-scanning"
        uses: github/codeql-action/upload-sarif@23acc5c183826b7a8a97bce3cecc52db901f8251 # v3.25.10
        with:
          sarif_file: results.sarif


CodeQL

---
name: "CodeQL"

on: # yamllint disable-line rule:truthy
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
  schedule:
    - cron: "34 3 * * 2"

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write

    strategy:
      fail-fast: false
      matrix:
        language: ["python"]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support

    steps:
      - name: Checkout repository
        uses: actions/checkout@v3

      # Initializes the CodeQL tools for scanning.
      - name: Initialize CodeQL
        uses: github/codeql-action/init@v2
        with:
          languages: ${{ matrix.language }}
          # If you wish to specify custom queries, you can do so here or in a config file.
          # By default, queries listed here will override any specified in a config file.
          # Prefix the list here with "+" to use these queries and those in the config file.

          # yamllint disable-line rule:line-length
          # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
          # queries: security-extended,security-and-quality

      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
      # If this step fails, then you should remove it and run the build manually (see below)
      - name: Autobuild
        uses: github/codeql-action/autobuild@v2

      # ℹ Command-line programs to run using the OS shell.
      # yamllint disable-line rule:line-length
      # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun

      #   If the Autobuild fails above, remove it and uncomment the following three lines.
      #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE
      #   below for guidance.

      # - run: |
      #   echo "Run, Build Application using script"
      #   ./location_of_script_within_repo/buildscript.sh

      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v2


Test documentation

---
name: Test documentation

env:
  CI_FORCE_COLORS_SPHINX: --color

on: # yamllint disable-line rule:truthy
  workflow_dispatch:

  push:
    tags:
      - "!*"
    branches:
      - main
      - "test-me-*"

  pull_request:
    branches:
      - "**"

jobs:
  build:
    name: Tests on ${{ matrix.os }} with default python
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"

      - name: Display Python version
        run: python --version

      - name: Upgrade pip
        run: python -m pip install --upgrade pip

      - name: Install pipx
        run: python -m pip install --upgrade pipx

      - name: Install tox via pipx
        run: pipx install tox

      - name: Run tests with tox except linkcheck and spelling
        if: runner.os != 'Linux'
        run: tox -m docs

      - name: Run all tests with tox
        if: runner.os == 'Linux'
        run: tox -m docs-full


QA

---
name: QA

env:
  CI_FORCE_COLORS_PRE_COMMIT: --color always

on: # yamllint disable-line rule:truthy
  workflow_dispatch:

  push:
    tags:
      - "!*"
    branches:
      - main
      - "test-me-*"

  pull_request:
    branches:
      - "**"

jobs:
  build:
    name: Run QA Tools
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"

      - name: Display Python version
        run: python --version

      - name: Upgrade pip
        run: python -m pip install --upgrade pip

      - name: Install pipx
        run: python -m pip install --upgrade pipx

      - name: Install tox via pipx
        run: pipx install tox

      - name: Run pre-commit via tox
        run: tox -e pre-commit-run


Test code

---
name: Test code

env:
  CI_FORCE_COLORS_PYTEST: --color yes

on: # yamllint disable-line rule:truthy
  workflow_dispatch:

  push:
    tags:
      - "!*"
    branches:
      - main
      - "test-me-*"

  pull_request:
    branches:
      - "**"

jobs:
  build:
    name: Tests on ${{ matrix.os }} with python ${{ matrix.python-version }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: ["ubuntu-latest", "windows-latest", "macos-latest"]
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12-dev"]
    steps:
      - uses: actions/checkout@v3

      - name: Fetch origin/main
        #: https://github.com/Bachmann1234/diff_cover#troubleshooting
        run: git fetch --no-tags origin main:refs/remotes/origin/main

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Display Python version
        run: python --version

      - name: Upgrade pip
        run: python -m pip install --upgrade pip

      - name: Install pipx
        run: python -m pip install --upgrade pipx

      - name: Install tox via pipx
        run: pipx install tox

      - name: Run tests via tox (linux/macos)
        if: runner.os != 'Windows'
        run: |
          major=$(python -c "import sys;print(sys.version_info[0])")
          minor=$(python -c "import sys;print(sys.version_info[1])")
          tox -m py${major}.${minor}

      - name: Run tests via tox (windows)
        if: runner.os == 'Windows'
        run: |
          $major = python -c "import sys;print(sys.version_info[0])"
          $minor = python -c "import sys;print(sys.version_info[1])"
          tox -m py${major}.${minor}


Update AUTHORS.rst

---
name: Update AUTHORS.rst

# What this workflow does:
# 1. Update the AUTHORS.rst file
# 2. Git commit and push the file if there are changes.

on: # yamllint disable-line rule:truthy
  workflow_dispatch:

  push:
    tags:
      - "!*"
    branches:
      - main
      - "test-me-*"

jobs:
  update-authors:
    name: Update AUTHORS.rst file
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
        with:
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"

      - name: Update AUTHORS.rst file
        shell: python
        run: |
          import subprocess

          git_authors = subprocess.run(
              ["git", "log", "--format=%aN <%aE>"], capture_output=True, check=True
          ).stdout.decode()

          skip_list = (
              "Steven Myint",
              "dependabot",
              "pre-commit-ci",
              "github-action",
              "GitHub Actions",
          )
          authors = [
              author
              for author in set(git_authors.strip().split("\n"))
              if not author.startswith(skip_list)
          ]
          authors.sort()

          file_head = (
              ".. This file is automatically generated/updated by a github actions workflow.\n"
              ".. Every manual change will be overwritten on push to main.\n"
              ".. You can find it here: ``.github/workflows/update-authors.yaml``\n"
              ".. For more information see "
              "`https://github.com/rstcheck/rstcheck/graphs/contributors`\n\n"
              "Author\n"
              "------\n"
              "Steven Myint <git@stevenmyint.com>\n\n"
              "Additional contributions by (sorted by name)\n"
              "--------------------------------------------\n"
          )

          with open("AUTHORS.rst", "w") as authors_file:
              authors_file.write(file_head)
              authors_file.write("- ")
              authors_file.write("\n- ".join(authors))
              authors_file.write("\n")

      - name: Check if diff
        continue-on-error: true
        run: >
          git diff --exit-code AUTHORS.rst &&
          (echo "### No update" && exit 1) || (echo "### Commit update")

      - uses: EndBug/add-and-commit@v9
        name: Commit and push if diff
        if: success()
        with:
          add: AUTHORS.rst
          message: Update AUTHORS.rst file with new author(s)
          author_name: GitHub Actions
          author_email: action@github.com
          committer_name: GitHub Actions
          committer_email: actions@github.com
          push: true


Advanced Tests (on demand)

# run advanced tests when `test advanced` comment is entered
name: Advanced Tests (on demand)

on:
  issue_comment:
    types: [created, edited]

jobs:
  verify:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [3.8, 3.9, "3.10", "3.11"]
        poetry-version: [1.5.1]
        test-group: [1, 2, 3, 4]

    if: ${{ github.event.comment.body == 'test advanced'}}
    steps:
      - name: Message Initiating
        uses: actions/github-script@v7
        with:
          script: |
            github.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: 'Launching advanced tests 🚀',
            });
      - name: Get PR SHA
        id: sha
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { owner, repo, number } = context.issue;
            const pr = await github.pulls.get({
              owner,
              repo,
              pull_number: number,
            });
            return pr.data.head.sha

      - uses: actions/checkout@v4
        with:
          ref: ${{ steps.sha.outputs.result }}

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Poetry ${{ matrix.poetry-version }}
        uses: abatilo/actions-poetry@v3.0.0
        with:
          poetry-version: ${{ matrix.poetry-version }}

      - name: Install graphviz
        run: |
          sudo apt install graphviz libgraphviz-dev graphviz-dev pkg-config

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          echo "installing poetry dependencies"
          poetry install -E plotting -E pydot -E pygraphviz -E econml --with docs

      - name: Run Advanced Tests
        run: poetry run poe test_advanced  --splits 4 --group ${{ matrix.test-group }}

      - name: Message success
        if: ${{ success() }}
        uses: actions/github-script@v7
        with:
          script: |
            github.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: 'Advanced tests succeeded! ✅',
            });
      - name: Message failure
        if: ${{ failure() }}
        uses: actions/github-script@v7
        with:
          script: |
            github.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: 'Advanced tests failed! ❌',
            });


Build Docker image (docs)

name: Build Docker image (docs)

on:
  push:
    branches: [main]
    paths: ['docs/Dockerfile']
  pull_request:
    branches: [main]
    paths: ['docs/Dockerfile']

jobs:
  build-and-push-docker-image:
    runs-on: ubuntu-latest
    permissions:
      packages: write
    steps:
      - uses: actions/checkout@v4
        name: Checkout repository

      - uses: pmorelli92/github-container-registry-build-push@2.2.1
        name: Build and Publish latest service image
        with:
          github-push-secret: ${{secrets.GITHUB_TOKEN}}
          docker-image-name: dowhy-docs-generation
          docker-image-tag: latest # optional
          dockerfile-path: ./docs/Dockerfile # optional


CI Installation Check

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
name: CI Installation Check

on:
  push:
    branches: [main]
    paths:
      - '.github/workflows/ci-install.yml'
      - 'dowhy/**'
      - 'tests/**'
      - 'pyproject.toml'
      - 'poetry.lock'

  pull_request:
    branches: [main]
    paths:
      - '.github/workflows/ci-install.yml'
      - 'dowhy/**'
      - 'tests/**'
      - 'pyproject.toml'
      - 'poetry.lock'

jobs:
  build:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [3.8, 3.9, "3.10", "3.11"]

    steps:
    - uses: actions/checkout@v4

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install graphviz
      run: |
        sudo apt install graphviz libgraphviz-dev graphviz-dev pkg-config

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        echo "HEAD_REPO='$HEAD_REPO'"
        echo "HEAD_BRANCH='$HEAD_BRANCH'"
        echo "GITHUB_CONTEXT='$GITHUB_CONTEXT'"
        pip install git+https://github.com/${HEAD_REPO}@${HEAD_BRANCH:-main}
      env:
        GITHUB_CONTEXT: ${{ toJson(github) }}
        HEAD_REPO: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
        HEAD_BRANCH: ${{ github.event.pull_request.head.ref || github.head_ref || 'main' }}

    - name: Notify Discord
      if: failure() && github.ref == 'refs/heads/main'
      uses: th0th/notify-discord@v0.4.1
      env:
        DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK }}
        GITHUB_ACTOR: ${{ github.actor }}
        GITHUB_JOB_NAME: CI Install Check Failed on Main
        GITHUB_JOB_STATUS: ${{ job.status }}


CI

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
name: CI

on:
  push:
    branches: [main]
    paths:
      - '.github/workflows/ci.yml'
      - 'dowhy/**'
      - 'tests/**'
      - 'docs/source/example_notebooks/**'
      - 'pyproject.toml'
      - 'poetry.lock'
  pull_request:
    branches: [main]
    paths:
      - '.github/workflows/ci.yml'
      - 'dowhy/**'
      - 'tests/**'
      - 'docs/source/example_notebooks/**'
      - 'pyproject.toml'
      - 'poetry.lock'

jobs:
  build:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [3.8, 3.9, "3.10", "3.11"]
        poetry-version: [1.5.1]
        test-group: [1, 2, 3, 4, 5, 6]

    steps:
    - uses: actions/checkout@v4

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install Poetry ${{ matrix.poetry-version }}
      uses: abatilo/actions-poetry@v3.0.0
      with:
        poetry-version: ${{ matrix.poetry-version }}

    - name: Install graphviz
      run: |
        sudo apt install graphviz libgraphviz-dev graphviz-dev pkg-config

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        echo "installing poetry dependencies"
        poetry install -E plotting -E pygraphviz

    - name: Lint
      run: poetry run poe lint

    - name: Check Formatting
      run: poetry run poe format_check

    - name: Test
      run: poetry run poe test --splits 6 --group ${{ matrix.test-group }}

    - name: Install EconML dependency
      run: |
        poetry install -E plotting -E pygraphviz -E econml

    - name: Test EconML based functionality
      run: poetry run poe test_econml

    - name: Test README errors for PyPI
      run: |
          pip install poetry-dynamic-versioning
          pip install twine
          poetry-dynamic-versioning
          poetry build
          twine check dist/*

    - name: Notify Discord
      if: failure() && github.ref == 'refs/heads/main'
      uses: th0th/notify-discord@v0.4.1
      env:
        DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK }}
        GITHUB_ACTOR: ${{ github.actor }}
        GITHUB_JOB_NAME: CI Failed on Main
        GITHUB_JOB_STATUS: ${{ job.status }}


Close inactive issues

name: Close inactive issues
on:
  schedule:
    - cron: "30 1 * * *"

jobs:
  close-issues:
    runs-on: ubuntu-latest
    permissions:
      issues: write
      pull-requests: write
    steps:
      - uses: actions/stale@v9
        with:
          days-before-issue-stale: 14
          days-before-issue-close: 7
          stale-issue-label: "stale"
          stale-issue-message: "This issue is stale because it has been open for 14 days with no activity."
          close-issue-message: "This issue was closed because it has been inactive for 7 days since being marked as stale."
          # in case of issue, only process issues with any of the following labels:
          any-of-issue-labels: "question,help wanted,waiting for author"

          days-before-pr-stale: 60
          days-before-pr-close: 14
          stale-pr-label: "stale"
          stale-pr-message: "This PR is stale because it has been open for 60 days with no activity."
          close-pr-message: "This PR was closed because it has been inactive for 7 days since being marked as stale."
          repo-token: ${{ secrets.GITHUB_TOKEN }}


Build Docs (CI)

name: Build Docs (CI)

on:
  pull_request:
    branches: [main]
    paths:
      - ".github/workflows/docs-ci.yml"
      - "dowhy/**"
      - "docs/**"
      - "pyproject.toml"
      - "poetry.lock"

jobs:
  docs:
    runs-on: ubuntu-latest
    container:
      image: "ghcr.io/${{ github.repository_owner }}/dowhy-docs-generation:latest"
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Install Python Dependencies
        run: poetry install -E plotting -E pydot -E pygraphviz -E econml --with docs

      - run: git config --global --add safe.directory /__w/dowhy/dowhy

      - name: Build
        run: ./docs/generate_docs.sh


Build & Deploy Docs (For Release)

name: Build & Deploy Docs (For Release)

on:
  release:
    types: [created]
  workflow_dispatch:
    inputs:
      release_tag:
        description: 'Release tag'
        required: true
        type: string

env:
  # Pull the tag either from the release event if this workflow was triggered by a release
  # or from the input if this workflow was triggered manually
  release_tag: ${{ github.event_name == 'release' && github.event.release.tag_name || inputs.release_tag }}

jobs:
  docs:
    runs-on: ubuntu-latest
    container:
      image: "ghcr.io/${{ github.repository_owner }}/dowhy-docs-generation:latest"
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Install Python Dependencies
        run: poetry install -E plotting -E pydot -E pygraphviz -E econml --with docs

      - run: git config --global --add safe.directory /__w/dowhy/dowhy

      - name: Build
        run: ./docs/generate_docs.sh
        env:
          DOCS_VERSION: ${{ env.release_tag }}

      - name: Commit and push to gh-pages branch
        uses: peaceiris/actions-gh-pages@v4
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: ./dowhy-docs


Build & Deploy Docs

name: Build & Deploy Docs
on:
  push:
    branches:
      - main
    paths:
      - ".github/workflows/docs.yml"
      - "dowhy/**"
      - "docs/**"
      - "pyproject.toml"
      - "poetry.lock"
  workflow_run:
    workflows: [ "Build Docker image (docs)" ]
    types:
      - completed

jobs:
  docs:
    runs-on: ubuntu-latest
    container:
      image: "ghcr.io/${{ github.repository_owner }}/dowhy-docs-generation:latest"
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Install Python Dependencies
        run: poetry install -E plotting -E pydot -E pygraphviz -E econml --with docs

      - run: git config --global --add safe.directory /__w/dowhy/dowhy

      - name: Build
        run: ./docs/generate_docs.sh

      - name: Commit and push to gh-pages branch
        uses: peaceiris/actions-gh-pages@v4
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: ./dowhy-docs


Advanced Tests (nightly)

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions
name: Advanced Tests (nightly)

on:
  schedule:
    - cron: "0 0 * * *"

jobs:
  build:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [3.8, 3.9, "3.10", "3.11"]
        poetry-version: [1.5.1]

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Poetry ${{ matrix.poetry-version }}
        uses: abatilo/actions-poetry@v3.0.0
        with:
          poetry-version: ${{ matrix.poetry-version }}

      - name: Install graphviz
        run: |
          sudo apt install graphviz libgraphviz-dev graphviz-dev pkg-config

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          echo "installing poetry dependencies"
          poetry install -E plotting -E pydot -E pygraphviz -E econml --with docs

      - name: Run Advanced Tests
        run: poetry run poe test_advanced

      - name: Notify Discord
        if: github.ref == 'refs/heads/main'
        uses: th0th/notify-discord@v0.4.1
        env:
          DISCORD_WEBHOOK_URL: ${{ secrets.DISCORD_WEBHOOK }}
          GITHUB_ACTOR: ${{ github.actor }}
          GITHUB_JOB_NAME: Nightly Tests ${{ job.status == 'success' && 'Passed' || 'Failed' }}
          GITHUB_JOB_STATUS: ${{ job.status }}


Publish Package

# This workflows will upload a Python Package using Twine when a release is created
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries

name: Publish Package

on:
  release:
    types: [created]
  workflow_dispatch:

jobs:
  deploy:
    name: Upload release to PyPI
    runs-on: ubuntu-latest
    environment: release
    permissions:
      id-token: write
    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - name: Set up Python 3.9
      uses: actions/setup-python@v5
      with:
        python-version: '3.9'

    - name: Install Poetry
      uses: abatilo/actions-poetry@v3.0.0
      with:
        poetry-version: '1.5.1'

    - name: Install Poetry Dynamic Versioning Plugin
      run: pip install poetry-dynamic-versioning

    - name: Install dependencies
      run: poetry install

    - name: Build and publish
      run: |
        poetry-dynamic-versioning
        poetry build
    - name: Publish package distributions to PyPI
      uses: pypa/gh-action-pypi-publish@release/v1


Check coding conventions with flake8

name: Check coding conventions with flake8

on:
  push:
    branches:
      - '**'
    tags:
      - '**'
  pull_request:
    branches:
      - '**'

jobs:
  flake8:
    runs-on: ubuntu-latest

    steps:
      - name: Check out source repository
        uses: actions/checkout@v4
      - name: Set up Python environment
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: flake8 Lint
        uses: py-actions/flake8@v2


Publish source package and wheel

name: Publish source package and wheel

on:
  push:
    branches:
      - '**'
    tags:
      - '**'
  pull_request:
    branches:
      - '**'

jobs:
  build:
    runs-on: ubuntu-22.04

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        submodules: recursive

    - name: Install dependencies
      run: |
        sudo apt-get update -qy
        sudo apt-get install -y python3 python3-pip python3-venv
        python3 -m venv ../venv
        source ../venv/bin/activate
        python3 -m pip install flit pip-tools build
        # We need to install all dependencies before executing flit build,
        # because flit build loads the package to determine the version.
        pip-compile -q --resolver=backtracking pyproject.toml
        pip install -r requirements.txt
        rm requirements.txt

    - name: Build package
      run: |
        source ../venv/bin/activate
        python3 -m build

    - name: Build documentation
      run: |
        source ../venv/bin/activate
        python3 -m pip install .[docs]
        python3 -m pip install dist/*.whl
        cd docs
        sphinx-apidoc -o source/ ../ContactMechanics
        make html
        cd ..

    - name: Commit documentation changes
      run: |
        git clone https://github.com/ContactEngineering/ContactMechanics.git --branch gh-pages --single-branch gh-pages
        cp -r docs/_build/html/* gh-pages/
        cd gh-pages
        git config --local user.email "action@github.com"
        git config --local user.name "GitHub Action"
        git add .
        git commit -m "Update documentation" -a || true
        # The above command will fail if no changes were present, so we ignore
        # the return code.

    - name: Push documentation changes
      if: github.ref == 'refs/heads/master'
      uses: ad-m/github-push-action@master
      with:
        branch: gh-pages
        directory: gh-pages
        github_token: ${{ secrets.GITHUB_TOKEN }}

    - name: Publish package
      if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
      uses: pypa/gh-action-pypi-publish@master
      with:
        user: __token__
        password: ${{ secrets.PYPI_PASSWORD }}


Test code functionality

name: Test code functionality

on:
  push:
    branches:
      - '**'
    tags:
      - '**'
  pull_request:
    branches:
      - '**'
  schedule:
    - cron: "0 2 * * 5"

jobs:
  tests:
    runs-on: ubuntu-22.04
    timeout-minutes: 40

    strategy:
      matrix:
        mpi: ['yes', 'no']
        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']
        mpi4py-version: [3.1.5]
        netcdf4-python-version: [1.6.2]
        mufft-version: [0.91.2]

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        submodules: recursive

    - name: Set up python3 ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install dependencies and extension module
      run: |
        set -x
        sudo apt-get update -qy
        sudo apt-get install -y python3-dev python3-venv libfftw3-dev libopenblas-dev
        python3 -m venv venv
        source venv/bin/activate
        python3 -m pip install --upgrade pip
        python3 -m pip install flit numpy pip-tools setuptools wheel
        if [ "${{ matrix.mpi }}" == "yes" ]; then
          # We now need to recompile some modules to link with native MPI
          sudo apt-get install openmpi-bin libopenmpi-dev libfftw3-mpi-dev libhdf5-mpi-dev libpnetcdf-dev libnetcdf-pnetcdf-dev
          # Install mpi4py first (because netCDF4 depends on it)
          export NETCDF4_DIR=/usr/lib/x86_64-linux-gnu/netcdf/pnetcdf
          export CC=mpicc
          python3 -m pip install -v --no-binary mpi4py mpi4py==${{ matrix.mpi4py-version }}
          python3 -m pip install -v cython
          python3 -m pip install -v --no-build-isolation --no-binary netCDF4 netCDF4==${{ matrix.netcdf4-python-version }}
          python3 -m pip install -v --no-binary muFFT muFFT==${{ matrix.mufft-version }}
        else
          sudo apt-get install libhdf5-dev libnetcdf-dev
        fi
        python3 -m pip install -v .[test]
        # Check that NetCDF install was not overriden
        python3 .check_netcdf_capabilities.py ${{ matrix.mpi }}
        python3 .check_mufft_capabilities.py ${{ matrix.mpi }}

    - name: Test with pytest
      run: |
        source venv/bin/activate
        if [ "${{ matrix.mpi }}" == "yes" ]; then
          python3 run-tests.py --no-build --verbose --full-trace
        else
          python3 -m pytest --verbose --full-trace -s
        fi


Test examples run without failure

name: Test examples run without failure

on:
  push:
    branches:
      - '**'
    tags:
      - '**'
  pull_request:
    branches:
      - '**'
  schedule:
    - cron: "0 2 * * 5"

jobs:
  tests:
    runs-on: ubuntu-22.04
    timeout-minutes: 40

    strategy:
      matrix:
        python-version: ['3.8', '3.9', '3.10', '3.11', '3.12']

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        submodules: recursive

    - name: Set up python3 ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install dependencies and extension module
      run: |
        set -x
        sudo apt-get update -qy
        sudo apt-get install -y python3-dev python3-venv libfftw3-dev libopenblas-dev
        python3 -m venv venv
        source venv/bin/activate
        python3 -m pip install --upgrade pip
        python3 -m pip install flit numpy pip-tools setuptools wheel
        sudo apt-get install libhdf5-dev libnetcdf-dev
        python3 -m pip install -v .[test]
        # Check that NetCDF install was not overriden
        python3 .check_netcdf_capabilities.py no
        python3 .check_mufft_capabilities.py no

    - name: Test example files
      run: |
        source venv/bin/activate
        python3 -m pip install jupytext
        python3 -m pip list
        cd examples
        for f in *.ipynb
        do
            jupytext --check  pytest --warn-only $f
        done

        for f in *.py
        do
            jupytext --check pytest --warn-only $f
        done


Build Docs

name: Build Docs

# Controls when the workflow will run
on:
  # Triggers the workflow on push and pull request events but only for the main branch
  push:
    branches: [main]
  pull_request:
    branches: [main, refactoring]

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  build_docs:
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.9"
      - name: Install full package
        run: |
          python -m venv venv
          . ./venv/bin/activate
          pip install --upgrade pip wheel poetry==1.5.1 lightfm==1.17
          poetry cache clear pypi --all
          ./poetry_wrapper.sh --experimental install --all-extras
      - name: Build docs
        run: |
          . ./venv/bin/activate
          make -C docs clean html


Deploy Docs to Pages

name: Deploy Docs to Pages

on:
  push:
    branches: [release]
    tags:
      - 'v*.*.*'

  workflow_dispatch:

permissions:
  contents: read
  pages: write
  id-token: write

concurrency:
  group: "pages"
  cancel-in-progress: false

jobs:
  deploy_docs:
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"
      - name: Install full package
        run: |
          python -m venv venv
          . ./venv/bin/activate
          pip install --upgrade pip wheel poetry==1.5.1 lightfm==1.17
          poetry cache clear pypi --all
          ./poetry_wrapper.sh --experimental install --all-extras
      - name: Build docs
        run: |
          . ./venv/bin/activate
          make -C docs clean html
      - name: Setup Pages
        uses: actions/configure-pages@v4
      - name: Upload artifact
        uses: actions/upload-pages-artifact@v2
        with:
          path: 'docs/_build/html/'
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v3


CI Pipeline

name: CI Pipeline

# Controls when the workflow will run
on:
  # Triggers the workflow on push and pull request events but only for the main branch
  push:
    branches: [main]
  pull_request:
    branches: [main, refactoring]

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  static_tests:
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"
      - name: Install full package
        run: |
          python -m venv venv
          . ./venv/bin/activate
          pip install --upgrade pip wheel poetry==1.5.1 lightfm==1.17
          poetry cache clear pypi --all
          ./poetry_wrapper.sh --experimental install --all-extras
      - name: ruff
        run: |
          . ./venv/bin/activate
          ./poetry_wrapper.sh --generate
          ruff check .
      - name: black
        run: |
          . ./venv/bin/activate
          ./poetry_wrapper.sh --generate
          black --check --diff -- .
      - name: poetry-check
        run: |
          . ./venv/bin/activate
          ./poetry_wrapper.sh check
          ./poetry_wrapper.sh --experimental check
      - name: toml-sort
        run: |
          . ./venv/bin/activate
          ./poetry_wrapper.sh --generate
          toml-sort --check pyproject.toml
          ./poetry_wrapper.sh --experimental --generate
          toml-sort --check pyproject.toml

  pytest_core:
    needs: static_tests
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.8", "3.9", "3.10"]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install core package
        run: |
          python -m venv venv
          . ./venv/bin/activate
          pip install --upgrade pip wheel poetry==1.5.1 lightfm==1.17
          poetry cache clear pypi --all
          ./poetry_wrapper.sh install
      - name: pytest
        run: |
          . ./venv/bin/activate
          pytest -m core tests/ --ignore=tests/experimental
          mv .coverage .coverage_core_${{ matrix.python-version }}
      - name: Upload coverage
        uses: actions/upload-artifact@v4
        with:
          name: .coverage_core_${{ matrix.python-version }}
          path: .coverage_core_${{ matrix.python-version }}

  pytest_torch:
    needs: static_tests
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.8", "3.9", "3.10"]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install torch package
        run: |
          python -m venv venv
          . ./venv/bin/activate
          pip install --upgrade pip wheel poetry==1.5.1 lightfm==1.17
          poetry cache clear pypi --all
          ./poetry_wrapper.sh install -E torch
      - name: pytest
        run: |
          . ./venv/bin/activate
          pytest -m "not spark and not experimental" tests/ --ignore=tests/experimental
          mv .coverage .coverage_torch_${{ matrix.python-version }}
      - name: Upload coverage
        uses: actions/upload-artifact@v4
        with:
          name: .coverage_torch_${{ matrix.python-version }}
          path: .coverage_torch_${{ matrix.python-version }}

  pytest_spark:
    needs: static_tests
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.8", "3.9", "3.10"]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install spark package
        run: |
          python -m venv venv
          . ./venv/bin/activate
          pip install --upgrade pip wheel poetry==1.5.1 lightfm==1.17
          poetry cache clear pypi --all
          ./poetry_wrapper.sh install -E spark
      - name: pytest
        run: |
          . ./venv/bin/activate
          pytest -m "not torch and not experimental" tests/ --ignore=tests/experimental
          mv .coverage .coverage_spark_${{ matrix.python-version }}
      - name: Upload coverage
        uses: actions/upload-artifact@v4
        with:
          name: .coverage_spark_${{ matrix.python-version }}
          path: .coverage_spark_${{ matrix.python-version }}

  pytest_spark_torch:
    needs: static_tests
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.8", "3.9", "3.10"]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install full package
        run: |
          python -m venv venv
          . ./venv/bin/activate
          pip install --upgrade pip wheel poetry==1.5.1 lightfm==1.17
          poetry cache clear pypi --all
          ./poetry_wrapper.sh install --all-extras
      - name: pytest
        run: |
          . ./venv/bin/activate
          pytest -m "not experimental" --ignore=replay/experimental --ignore=tests/experimental
          mv .coverage .coverage_all_${{ matrix.python-version }}
      - name: Upload coverage
        uses: actions/upload-artifact@v4
        with:
          name: .coverage_all_${{ matrix.python-version }}
          path: .coverage_all_${{ matrix.python-version }}

  pytest_experimental:
    needs: static_tests
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.8", "3.9", "3.10"]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install experimental package
        run: |
          python -m venv venv
          . ./venv/bin/activate
          pip install --upgrade pip wheel poetry==1.5.1 lightfm==1.17
          poetry cache clear pypi --all
          ./poetry_wrapper.sh --experimental install --all-extras
      - name: pytest
        run: |
          . ./venv/bin/activate
          pytest -m "experimental"
          mv .coverage .coverage_experimental_${{ matrix.python-version }}
      - name: Upload coverage
        uses: actions/upload-artifact@v4
        with:
          name: .coverage_experimental_${{ matrix.python-version }}
          path: .coverage_experimental_${{ matrix.python-version }}

  merge_coverage:
    needs: [pytest_core, pytest_torch, pytest_spark, pytest_spark_torch, pytest_experimental]
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.8", "3.9", "3.10"]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install only dev package
        run: |
          python -m venv venv
          . ./venv/bin/activate
          pip install --upgrade pip wheel poetry==1.5.1 poetry-dynamic-versioning lightfm==1.17
          poetry cache clear pypi --all
          ./poetry_wrapper.sh install --only dev
      - name: Download coverage reports
        uses: actions/download-artifact@v4
        with:
          pattern: .coverage_*
          merge-multiple: true
      - name: Combine coverage
        run: |
          . ./venv/bin/activate
          coverage combine .coverage_core_${{ matrix.python-version }} .coverage_torch_${{ matrix.python-version }} .coverage_spark_${{ matrix.python-version }} .coverage_all_${{ matrix.python-version }} .coverage_experimental_${{ matrix.python-version }}
          coverage report --fail-under=100
          coverage xml

  build_production:
    needs: merge_coverage
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"
      - name: Install production package
        run: |
          python -m venv venv
          . ./venv/bin/activate
          pip install --upgrade pip wheel poetry==1.5.1 poetry-dynamic-versioning
          poetry cache clear pypi --all
          ./poetry_wrapper.sh install --all-extras
      - name: Build
        run: |
          . ./venv/bin/activate
          export PACKAGE_SUFFIX=.dev
          ./poetry_wrapper.sh build

  build_experimental:
    needs: merge_coverage
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"
      - name: Install full package
        run: |
          python -m venv venv
          . ./venv/bin/activate
          pip install --upgrade pip wheel poetry==1.5.1 poetry-dynamic-versioning lightfm==1.17
          poetry cache clear pypi --all
          ./poetry_wrapper.sh --experimental install --all-extras
      - name: Build
        run: |
          . ./venv/bin/activate
          export PACKAGE_SUFFIX=.preview
          ./poetry_wrapper.sh --experimental build


Publish to PyPI

name: Publish to PyPI

on:
  push:
    branches: [release]
    tags:
      - 'v*.*.*'

jobs:
  publish_experimental:
    runs-on: ubuntu-20.04
    permissions:
        id-token: write
    environment:
        name: release
        url: https://pypi.org/p/replay-rec

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Install experimental package
        run: |
          python -m venv venv
          . ./venv/bin/activate
          pip install --upgrade pip wheel poetry==1.5.1 poetry-dynamic-versioning lightfm==1.17

          poetry cache clear pypi --all
          ./poetry_wrapper.sh --experimental install --all-extras

      - name: Build Experimental
        run: |
          . ./venv/bin/activate
          export PACKAGE_SUFFIX=.preview
          ./poetry_wrapper.sh --experimental build

      - name: Publish Experimental
        uses: pypa/gh-action-pypi-publish@release/v1

  publish_production:
    runs-on: ubuntu-20.04
    permissions:
        id-token: write
    environment:
        name: release
        url: https://pypi.org/p/replay-rec

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Install production package
        run: |
          python -m venv venv
          . ./venv/bin/activate
          pip install --upgrade pip wheel poetry==1.5.1 poetry-dynamic-versioning

          poetry cache clear pypi --all
          ./poetry_wrapper.sh install --all-extras

      - name: Build Production
        run: |
          . ./venv/bin/activate
          ./poetry_wrapper.sh build

      - name: Publish Production
        uses: pypa/gh-action-pypi-publish@release/v1


Publish to test PyPI

name: Publish to test PyPI

on:
  workflow_dispatch:

jobs:

  publish_test:
    runs-on: ubuntu-20.04
    permissions:
        id-token: write
    environment:
        name: testpypi
        url: https://test.pypi.org/p/replay-rec

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"

      - name: Install full package
        run: |
          python -m venv venv
          . ./venv/bin/activate
          pip install --upgrade pip wheel poetry==1.5.1 poetry-dynamic-versioning lightfm==1.17

          poetry cache clear pypi --all
          ./poetry_wrapper.sh --experimental install --all-extras

      - name: Build
        run: |
          . ./venv/bin/activate
          export PACKAGE_SUFFIX=.preview
          echo $PACKAGE_SUFFIX
          ./poetry_wrapper.sh --experimental build

      - name: Publish
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          repository-url: https://test.pypi.org/legacy/


build

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: build

on:
  push:
    paths-ignore:
      - ".dev_scripts/**"
      - ".github/**.md"
      - "demo/**"
      - "docker/**"
      - "tools/**"

  pull_request:
    paths-ignore:
      - ".dev_scripts/**"
      - ".github/**.md"
      - "demo/**"
      - "docker/**"
      - "tools/**"
      - "docs/**"
      - "docs_zh-CN/**"

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build:
    env:
      FORCE_CUDA: 1
      CUDA_ARCH: ${{matrix.cuda_arch}}
    runs-on: ubuntu-18.04
    container:
      image: pytorch/pytorch:1.6.0-cuda10.1-cudnn7-devel

    strategy:
      matrix:
        python-version: [3.6, 3.7]
        torch: [1.5.0+cu101, 1.6.0+cu101, 1.7.0+cu101, 1.8.0+cu101]
        include:
          - torch: 1.5.0+cu101
            torch_version: torch1.5
            torchvision: 0.6.0+cu101
            mmcv_link: "torch1.5.0"
            cuda_arch: "7.0"
          - torch: 1.6.0+cu101
            torch_version: torch1.6
            mmcv_link: "torch1.6.0"
            torchvision: 0.7.0+cu101
            cuda_arch: "7.0"
          - torch: 1.7.0+cu101
            torch_version: torch1.7
            mmcv_link: "torch1.7.0"
            torchvision: 0.8.1+cu101
            cuda_arch: "7.0"
          - torch: 1.8.0+cu101
            torch_version: torch1.8
            mmcv_link: "torch1.8.0"
            torchvision: 0.9.0+cu101
            cuda_arch: "7.0"

    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Fetch GPG keys
        run: |
          apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu1804/x86_64/3bf863cc.pub
          apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/machine-learning/repos/ubuntu1804/x86_64/7fa2af80.pub
      - name: Install system dependencies
        run: |
          apt-get update && apt-get install -y ffmpeg libsm6 git ninja-build libglib2.0-0 libsm6 libxrender-dev python${{matrix.python-version}}-dev
          apt-get clean
          rm -rf /var/lib/apt/lists/*
      - name: Install PyTorch
        run: python -m pip install numpy==1.19.5 torch==${{matrix.torch}} torchvision==${{matrix.torchvision}} -f https://download.pytorch.org/whl/torch_stable.html
      - name: Install mmdet3d dependencies
        run: |
          python -m pip install mmcv-full==1.6.0 -f https://download.openmmlab.com/mmcv/dist/cu101/${{matrix.torch_version}}/index.html
          python -m pip install mmdet
          python -m pip install mmsegmentation
          python -m pip install -r requirements.txt
      - name: Build and install
        run: |
          rm -rf .eggs
          python setup.py check -m -s
          TORCH_CUDA_ARCH_LIST=${CUDA_ARCH} python setup.py build_ext --inplace
      - name: Run unittests and generate coverage report
        run: |
          coverage run --branch --source mmdet3d -m pytest tests/
          coverage xml
          coverage report -m
      # Only upload coverage report for python3.7 && pytorch1.5
      - name: Upload coverage to Codecov
        if: ${{matrix.torch == '1.5.0+cu101' && matrix.python-version == '3.7'}}
        uses: codecov/codecov-action@v1.0.10
        with:
          file: ./coverage.xml
          flags: unittests
          env_vars: OS,PYTHON
          name: codecov-umbrella
          fail_ci_if_error: false

  build_windows:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [windows-2022]
        python: [3.8]
        platform: [cpu]
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python }}
      - name: Upgrade pip
        run: python -m pip install pip --upgrade --user
      - name: Install PyTorch
        # As a complement to Linux CI, we test on PyTorch LTS version
        run: pip install torch==1.8.2+${{ matrix.platform }} torchvision==0.9.2+${{ matrix.platform }} -f https://download.pytorch.org/whl/lts/1.8/torch_lts.html
      - name: Install mmdet3d dependencies
        run: |
          pip install mmcv-full -f https://download.openmmlab.com/mmcv/dist/cpu/torch1.8/index.html --only-binary mmcv-full
          python -m pip install mmdet
          python -m pip install mmsegmentation
          python -m pip install -r requirements/build.txt -r requirements/runtime.txt -r requirements/tests.txt
      - name: Build and install
        run: pip install -e .
      - name: Run unittests and generate coverage report
        run: coverage run --branch --source mmdet3d -m pytest tests/
      - name: Generate coverage report
        run: |
          coverage xml
          coverage report -m
      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v2
        with:
          file: ./coverage.xml
          flags: unittests
          env_vars: OS,PYTHON
          name: codecov-umbrella
          fail_ci_if_error: false


deploy

name: deploy

on: push

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build-n-publish:
    runs-on: ubuntu-18.04
    if: startsWith(github.event.ref, 'refs/tags')
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python 3.7
        uses: actions/setup-python@v2
        with:
          python-version: 3.7
      - name: Install torch
        run: pip install torch
      - name: Build MMDet3D
        run: python setup.py sdist
      - name: Publish distribution to PyPI
        run: |
          pip install twine
          twine upload dist/* -u __token__ -p ${{ secrets.pypi_password }}


lint

name: lint

on: [push, pull_request]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  lint:
    runs-on: ubuntu-18.04
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python 3.7
        uses: actions/setup-python@v1
        with:
          python-version: 3.7
      - name: Install linting dependencies
        run: |
          python -m pip install --upgrade pip
          pip install flake8==3.8.3 isort==5.10.1 yapf==v0.30.0 interrogate
      - name: Lint with flake8
        run: flake8 .
      - name: Lint with isort
        run: isort --recursive --check-only --diff mmdet3d/ tests/ examples/
      - name: Format python codes with yapf
        run: yapf -r -d mmdet3d/ tests/ examples/
      - name: Check docstring
        run: interrogate -v --ignore-init-method --ignore-module --ignore-nested-functions --exclude mmdet3d/ops --ignore-regex "__repr__" --fail-under 95 mmdet3d


test-mim

name: test-mim

on:
  push:
    paths:
      - 'model-index.yml'
      - 'configs/**'

  pull_request:
    paths:
      - 'model-index.yml'
      - 'configs/**'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build_cpu:
    runs-on: ubuntu-18.04
    strategy:
      matrix:
        python-version: [3.7]
        torch: [1.8.0]
        include:
          - torch: 1.8.0
            torch_version: torch1.8
            torchvision: 0.9.0
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Upgrade pip
        run: pip install pip --upgrade
      - name: Install Pillow
        run: pip install Pillow==6.2.2
        if: ${{matrix.torchvision == '0.4.2'}}
      - name: Install PyTorch
        run: pip install torch==${{matrix.torch}}+cpu torchvision==${{matrix.torchvision}}+cpu -f https://download.pytorch.org/whl/torch_stable.html
      - name: Install openmim
        run: pip install openmim
      - name: Build and install
        run: rm -rf .eggs && mim install -e .
      - name: test commands of mim
        run: mim search mmdet3d


Snowpark pandas Changedoc Check

name: Snowpark pandas Changedoc Check

on:
  pull_request:
    types: [opened, synchronize, labeled, unlabeled]
    branches:
      - main
    paths:
      - 'src/snowflake/snowpark/modin/**'

jobs:
  check_pandas_change_doc:
    runs-on: ubuntu-latest
    if: ${{!contains(github.event.pull_request.labels.*.name, 'NO-PANDAS-CHANGEDOC-UPDATES')}}
    steps:
    - name: Checkout
      uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - name: Ensure Snowpark pandas docs is updated
      run: git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -q "docs/source/modin"


Changelog Check

name: Changelog Check

on:
  pull_request:
    types: [opened, synchronize, labeled, unlabeled]
    branches:
      - main

jobs:
  check_change_log:
    runs-on: ubuntu-latest
    if: ${{!contains(github.event.pull_request.labels.*.name, 'NO-CHANGELOG-UPDATES')}}
    steps:
    - name: Checkout
      uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - name: Ensure CHANGELOG.md is updated
      run: git diff --name-only --diff-filter=ACMRT ${{ github.event.pull_request.base.sha }} ${{ github.sha }} | grep -wq "CHANGELOG.md"


PR Checker

name: PR Checker

on:
  schedule:
    - cron: "0 0 * * *" # Runs 0:0 UTC on every day
  workflow_dispatch:
    inputs:
      logLevel:
        default: warning
        description: "Log level"
        required: true
      tags:
        description: "Check PRs"

permissions:
  contents: read
  pull-requests: read

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout Code
      uses: actions/checkout@v4
    - name: Setup Python 3.10
      uses: actions/setup-python@v3
      with:
        python-version: "3.10"

    - name: Dependencies
      run: |
        python3 -m pip install --upgrade pip
        pip3 install requests
    - name: Run script
      env:
        TOKEN: ${{secrets.GITHUB_TOKEN}}
      run:
        python3 ./scripts/longprs.py


CLA Assistant

name: "CLA Assistant"
on:
  issue_comment:
    types: [created]
  pull_request_target:
    types: [opened,closed,synchronize]

jobs:
  CLAssistant:
    runs-on: ubuntu-latest
    permissions:
      actions: write
      contents: write
      pull-requests: write
      statuses: write
    steps:
      - name: "CLA Assistant"
        if: (github.event.comment.body == 'recheck' || github.event.comment.body == 'I have read the CLA Document and I hereby sign the CLA') || github.event_name == 'pull_request_target'
        uses: contributor-assistant/github-action/@master
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          PERSONAL_ACCESS_TOKEN : ${{ secrets.CLA_BOT_TOKEN }}
        with:
          path-to-signatures: 'signatures/version1.json'
          path-to-document: 'https://github.com/snowflakedb/CLA/blob/main/README.md'
          branch: 'main'
          allowlist: 'dependabot[bot],github-actions,sfc-gh-snyk-sca-sa'
          remote-organization-name: 'snowflakedb'
          remote-repository-name: 'cla-db'


name: Daily Snowpark pandas jupyter notebook test

on:
    schedule:
      - cron: "0 16 * * *" # Runs 16:00 UTC on every day
    workflow_dispatch:
        inputs:
            logLevel:
                default: warning
                description: "Log level"
                required: true

jobs:
    lint:
        name: Check linting
        runs-on: ubuntu-latest
        steps:
          - uses: actions/checkout@v4
          - name: Set up Python
            uses: actions/setup-python@v4
            with:
              python-version: '3.9'
          - name: Display Python version
            run: python -c "import sys; import os; print(\"\n\".join(os.environ[\"PATH\"].split(os.pathsep))); print(sys.version); print(sys.executable);"
          - name: Upgrade setuptools and pip
            run: python -m pip install -U setuptools pip
          - name: Install tox
            run: python -m pip install tox
          - name: Run fix_lint
            run: python -m tox -e fix_lint

    build:
        needs: lint
        name: Build Wheel File
        runs-on: ubuntu-latest
        steps:
            - name: Checkout Code
              uses: actions/checkout@v4
            - name: Setup Python
              uses: actions/setup-python@v4
              with:
                python-version: 3.9
            - name: Upgrade setuptools and pip
              run: python -m pip install -U setuptools pip wheel
            - name: Generate wheel
              run: python -m pip wheel -v -w dist --no-deps .
            - name: Show wheels generated
              run: ls -lh dist
            - uses: actions/upload-artifact@v1
              with:
                name: wheel
                path: dist/
    test:
        name: Test modin-${{ matrix.os.download_name }}-${{ matrix.python-version }}-${{ matrix.cloud-provider }}
        needs: build
        runs-on: ${{ matrix.os.image_name }}
        strategy:
            fail-fast: false
            matrix:
                os:
                  - image_name: ubuntu-latest
                    download_name: linux
                  - image_name: macos-latest
                    download_name: macos
                  - image_name: windows-latest
                    download_name: windows
                python-version: ["3.9", "3.10", "3.11"]
                # FINANCIAL__ECONOMIC_ESSENTIALS database unavailable in gcp, azure
                cloud-provider: [aws]
        steps:
            - name: Checkout Code
              uses: actions/checkout@v4
            - name: Set up Python
              uses: actions/setup-python@v4
              with:
                    python-version: ${{ matrix.python-version }}
            - name: Display Python version
              run: python -c "import sys; print(sys.version)"
            - name: Decrypt parameters.py
              shell: bash
              run: .github/scripts/decrypt_parameters.sh
              env:
                  PARAMETER_PASSWORD: ${{ secrets.PARAMETER_PASSWORD }}
                  CLOUD_PROVIDER: ${{ matrix.cloud-provider }}
            - name: Download wheel(s)
              uses: actions/download-artifact@v3
              with:
                name: wheel
                path: dist
            - name: Show wheels downloaded
              run: ls -lh dist
              shell: bash
            - name: Upgrade setuptools, pip and wheel
              run: python -m pip install -U setuptools pip wheel
            - name: Install project
              run: python -m pip install -e ".[modin-development]"
            - name: Install test requirements
              run: python -m pip install -r tests/notebooks/test_requirements.txt
            - name: Run notebook tests
              run: |
                python -c "import sys; sys.path.append('../')"
                python -m pytest -ra --nbmake --nbmake-timeout=1000 tests/notebooks/modin
              env:
                PYTHON_VERSION: ${{ matrix.python-version }}
                cloud_provider: ${{ matrix.cloud-provider }}
                PYTEST_ADDOPTS: --color=yes --tb=short
                # Specify SNOWFLAKE_IS_PYTHON_RUNTIME_TEST: 1 when adding >= python3.11 with no server-side support
                # For example, see https://github.com/snowflakedb/snowpark-python/pull/681
              shell: bash