pre-commit

# Copyright 2023, NVIDIA CORPORATION & AFFILIATES. All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
#  * Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
#  * Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#  * Neither the name of NVIDIA CORPORATION nor the names of its
#    contributors may be used to endorse or promote products derived
#    from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE COPYRIGHT OWNER OR
# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

name: pre-commit

on:
  pull_request:

jobs:
  pre-commit:
    runs-on: ubuntu-22.04
    steps:
    - uses: actions/checkout@v3
    - uses: actions/setup-python@v3
    - uses: pre-commit/action@v3.0.0


CI

name: CI

on:
  pull_request:
  push:
    branches:
      - develop
      - main

jobs:
  main:
    strategy:
      fail-fast: false
      matrix:
        include:
          - name: "Test: Python 3.11"
            python: "3.11"
            tox: py311
          - name: "Test: Python 3.12"
            python: "3.12"
            tox: py312
            coverage: true
          - name: "Lint: pyright"
            python: "3.12"
            tox: pyright
          - name: "Lint: ruff lint"
            python: "3.12"
            tox: ruff-lint
          - name: "Lint: ruff format"
            python: "3.12"
            tox: ruff-format
          - name: "Docs"
            python: "3.12"
            tox: docs

    name: ${{ matrix.name }}
    runs-on: ubuntu-22.04
    container: ghcr.io/mopidy/ci:latest

    steps:
      - uses: actions/checkout@v4
      - name: Fix home dir permissions to enable pip caching
        run: chown -R root /github/home
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
          cache: pip
      - run: python -m pip install tox
      - run: python -m tox -e ${{ matrix.tox }}
        if: ${{ ! matrix.coverage }}
      - run: python -m tox -e ${{ matrix.tox }} -- --cov-report=xml
        if: ${{ matrix.coverage }}
      - uses: codecov/codecov-action@v4
        if: ${{ matrix.coverage }}
        with:
          token: ${{ secrets.CODECOV_TOKEN }}


Release

name: Release

on:
  release:
    types: [published]

jobs:
  release:
    runs-on: ubuntu-22.04

    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: "Install dependencies"
        run: python3 -m pip install build
      - name: "Build package"
        run: python3 -m build
      - uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.PYPI_TOKEN }}


Ubuntu devel

name: "Ubuntu devel"

on:
  schedule:
    # Every Friday at 18:20 UTC, so we can fix the issue over the weekend
    - cron: "20 18 * * 5"
  workflow_dispatch:

jobs:
  main:
    name: "Test: Python 3.12"
    runs-on: ubuntu-22.04

    # The container should be automatically updated from time to time.
    container: ubuntu:devel

    steps:
      - name: Install dependencies
        run: |
          apt-get update
          DEBIAN_FRONTEND=noninteractive apt-get install -y \
            git \
            python3 \
            tox \
            gstreamer1.0-plugins-bad \
            gstreamer1.0-plugins-good \
            gstreamer1.0-plugins-ugly \
            python3-gst-1.0 \
            libcairo2-dev \
            libgirepository1.0-dev \
            libglib2.0-dev
      - uses: actions/checkout@v4
      - run: tox -e py312


Add label automatically

name: "Add label automatically"
on:
- pull_request_target

jobs:
  triage:
    permissions:
      contents: read
      pull-requests: write
    runs-on: ubuntu-latest
    steps:
    - uses: actions/labeler@v4
      with:
        repo-token: "${{ secrets.GITHUB_TOKEN }}"


Upload Python Package

# This workflows will upload a Python Package using Twine when a release is created
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries

name: Upload Python Package

on:
  release:
    types: [published]

jobs:
  deploy_with_bdist_wheel:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [windows-latest, macos-11]
        # FIXME:  macos-latest will raise error now.
        # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
        python-version: [3.7, 3.8]

    steps:
    - uses: actions/checkout@v2
    #  This is because on macos systems you can install pyqlib using
    # `pip install pyqlib` installs, it does not recognize the
    # `pyqlib-<version>-cp38-cp38-macosx_11_0_x86_64.whl` and `pyqlib-<veresion>-cp38-cp37m-macosx_11_0_x86_64.whl`.
    # So we limit the version of python, in order to generate a version of qlib that is usable for macos: `pyqlib-<veresion>-cp38-cp37m
    # `pyqlib-<version>-cp38-cp38-macosx_10_15_x86_64.whl` and `pyqlib-<veresion>-cp38-cp37m-macosx_10_15_x86_64.whl`.
    # Python 3.7.16, 3.8.16  can build macosx_10_15.  But  Python 3.7.17, 3.8.17  can build macosx_11_0
    - name: Set up Python ${{ matrix.python-version }}
      if: matrix.os == 'macos-11' && matrix.python-version == '3.7'
      uses: actions/setup-python@v2
      with:
        python-version: "3.7.16"
    - name: Set up Python ${{ matrix.python-version }}
      if: matrix.os == 'macos-11' && matrix.python-version == '3.8'
      uses: actions/setup-python@v2
      with:
        python-version: "3.8.16"
    - name: Set up Python ${{ matrix.python-version }}
      if: matrix.os != 'macos-11'
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install setuptools wheel twine
    - name: Build wheel on ${{ matrix.os }}
      run: |
        pip install numpy
        pip install cython
        python setup.py bdist_wheel
    - name: Build and publish
      env:
        TWINE_USERNAME: __token__
        TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
      run: |
        twine upload dist/*

  deploy_with_manylinux:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
    - name: Build wheel on Linux
      uses: RalfG/python-wheels-manylinux-build@v0.3.1-manylinux2010_x86_64
      with:
        # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
        python-versions: 'cp37-cp37m cp38-cp38'
        build-requirements: 'numpy cython'
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: 3.7
    - name: Install dependencies
      run: |
        pip install twine
    - name: Build and publish
      env:
        TWINE_USERNAME: __token__
        TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}
      run: |
        twine upload dist/pyqlib-*-manylinux*.whl


Release Drafter

name: Release Drafter

on:
  push:
    # branches to consider in the event; optional, defaults to all
    branches:
      - main

permissions:
  contents: read

jobs:
  update_release_draft:
    permissions:
      contents: write
      pull-requests: read
    runs-on: ubuntu-latest
    steps:
      # Drafts your next Release notes as Pull Requests are merged into "master"
      - uses: release-drafter/release-drafter@v5.11.0
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


Mark stale issues and pull requests

name: Mark stale issues and pull requests

on:
  schedule:
  - cron: "0 0/3 * * *"

jobs:
  stale:

    runs-on: ubuntu-latest

    steps:
    - uses: actions/stale@v3
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
        stale-issue-message: 'This issue is stale because it has been open for three months with no activity. Remove the stale label or comment on the issue otherwise this will be closed in 5 days'
        stale-pr-message: 'This PR is stale because it has been open for a year with no activity. Remove the stale label or comment on the PR otherwise this will be closed in 5 days'
        stale-issue-label: 'stale'
        stale-pr-label: 'stale'
        days-before-stale: 90
        days-before-pr-stale: 365
        days-before-close: 5
        operations-per-run: 100
        exempt-issue-labels: 'bug,enhancement'
        remove-stale-when-updated: true


Test qlib from pip

name: Test qlib from pip

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

jobs:
  build:
    timeout-minutes: 120

    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        # Since macos-latest changed from 12.7.4 to 14.4.1,
        # the minimum python version that matches a 14.4.1 version of macos is 3.10,
        # so we limit the macos version to macos-12.
        os: [windows-latest, ubuntu-20.04, ubuntu-22.04, macos-11, macos-12]
        # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
        python-version: [3.7, 3.8]

    steps:
    - name: Test qlib from pip
      uses: actions/checkout@v3

    # Since version 3.7 of python for MacOS is installed in CI, version 3.7.17, this version causes "_bz not found error".
    # So we make the version number of python 3.7 for MacOS more specific.
    # refs: https://github.com/actions/setup-python/issues/682
    - name: Set up Python ${{ matrix.python-version }}
      if: (matrix.os == 'macos-latest' && matrix.python-version == '3.7') || (matrix.os == 'macos-11' && matrix.python-version == '3.7')
      uses: actions/setup-python@v4
      with:
        python-version: "3.7.16"

    - name: Set up Python ${{ matrix.python-version }}
      if: (matrix.os != 'macos-latest' || matrix.python-version != '3.7') && (matrix.os != 'macos-11' || matrix.python-version != '3.7')
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Update pip to the latest version
      run: |
        python -m pip install --upgrade pip

    - name: Qlib installation test
      run: |
        # 2024-05-30 scs has released a new version: 3.2.4.post2,
        # This will cause the CI to fail, so we have limited the version of scs for now.
        python -m pip install "scs<=3.2.4"
        python -m pip install pyqlib

    - name: Install Lightgbm for MacOS
      if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }}
      run: |
        /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)"
        HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm
        # FIX MacOS error: Segmentation fault
        # reference: https://github.com/microsoft/LightGBM/issues/4229
        wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb
        brew unlink libomp
        brew install libomp.rb

    - name: Downloads dependencies data
      run: |
        cd ..
        python -m qlib.run.get_data qlib_data --target_dir ~/.qlib/qlib_data/cn_data --region cn
        cd qlib

    - name: Test workflow by config
      # On macos-11 system, it will lead to "Segmentation fault: 11" error,
      # which may be caused by the excessive memory overhead of macos-11 system, so we disable macos-11 temporarily here.
      if: ${{ matrix.os != 'macos-11' }}
      run: |
        qrun examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml


Test qlib from source

name: Test qlib from source

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

jobs:
  build:
    timeout-minutes: 180
    # we may retry for 3 times for `Unit tests with Pytest`

    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        # Since macos-latest changed from 12.7.4 to 14.4.1,
        # the minimum python version that matches a 14.4.1 version of macos is 3.10,
        # so we limit the macos version to macos-12.
        os: [windows-latest, ubuntu-20.04, ubuntu-22.04, macos-11, macos-12]
        # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
        python-version: [3.7, 3.8]

    steps:
    - name: Test qlib from source
      uses: actions/checkout@v3

    # Since version 3.7 of python for MacOS is installed in CI, version 3.7.17, this version causes "_bz not found error".
    # So we make the version number of python 3.7 for MacOS more specific.
    # refs: https://github.com/actions/setup-python/issues/682
    - name: Set up Python ${{ matrix.python-version }}
      if: (matrix.os == 'macos-latest' && matrix.python-version == '3.7') || (matrix.os == 'macos-11' && matrix.python-version == '3.7')
      uses: actions/setup-python@v4
      with:
        python-version: "3.7.16"

    - name: Set up Python ${{ matrix.python-version }}
      if: (matrix.os != 'macos-latest' || matrix.python-version != '3.7') && (matrix.os != 'macos-11' || matrix.python-version != '3.7')
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Update pip to the latest version
      run: |
        python -m pip install --upgrade pip

    - name: Installing pytorch for macos
      if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }}
      run: |
        python -m pip install torch torchvision torchaudio

    - name: Installing pytorch for ubuntu
      if: ${{ matrix.os == 'ubuntu-20.04' || matrix.os == 'ubuntu-22.04' }}
      run: |
        python -m pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu

    - name: Installing pytorch for windows
      if: ${{ matrix.os == 'windows-latest' }}
      run: |
        python -m pip install torch torchvision torchaudio

    - name: Set up Python tools
      run: |
        python -m pip install --upgrade cython
        python -m pip install -e .[dev]

    - name: Lint with Black
      # Python 3.7 will use a black with low level. So we use python with higher version for black check
      if: (matrix.python-version != '3.7')
      run: |
        pip install -U black  # follow the latest version of black, previous Qlib dependency will downgrade black
        black . -l 120 --check --diff

    - name: Make html with sphinx
      # Since read the docs builds on ubuntu 22.04, we only need to test that the build passes on ubuntu 22.04.
      if: ${{ matrix.os == 'ubuntu-22.04' }}
      run: |
        cd docs
        sphinx-build -W --keep-going -b html . _build
        cd ..

    # Check Qlib with pylint
    # TODO: These problems we will solve in the future. Important among them are: W0221, W0223, W0237, E1102
      # C0103: invalid-name
      # C0209: consider-using-f-string
      # R0402: consider-using-from-import
      # R1705: no-else-return
      # R1710: inconsistent-return-statements
      # R1725: super-with-arguments
      # R1735: use-dict-literal
      # W0102: dangerous-default-value
      # W0212: protected-access
      # W0221: arguments-differ
      # W0223: abstract-method
      # W0231: super-init-not-called
      # W0237: arguments-renamed
      # W0612: unused-variable
      # W0621: redefined-outer-name
      # W0622: redefined-builtin
      # FIXME: specify exception type
      # W0703: broad-except
      # W1309: f-string-without-interpolation
      # E1102: not-callable
      # E1136: unsubscriptable-object
    # References for parameters: https://github.com/PyCQA/pylint/issues/4577#issuecomment-1000245962
    # We use sys.setrecursionlimit(2000) to make the recursion depth larger to ensure that pylint works properly (the default recursion depth is 1000).
    - name: Check Qlib with pylint
      run: |
        pylint --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136 --const-rgx='[a-z_][a-z0-9_]{2,30}$' qlib --init-hook "import astroid; astroid.context.InferenceContext.max_inferred = 500; import sys; sys.setrecursionlimit(2000)"
        pylint --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0246,W0612,W0621,W0622,W0703,W1309,E1102,E1136 --const-rgx='[a-z_][a-z0-9_]{2,30}$' scripts --init-hook "import astroid; astroid.context.InferenceContext.max_inferred = 500; import sys; sys.setrecursionlimit(2000)"

    # The following flake8 error codes were ignored:
      # E501 line too long
        # Description: We have used black to limit the length of each line to 120.
      # F541 f-string is missing placeholders
        # Description: The same thing is done when using pylint for detection.
      # E266 too many leading '#' for block comment
        # Description: To make the code more readable, a lot of "#" is used.
        # This error code appears centrally in:
          # qlib/backtest/executor.py
          # qlib/data/ops.py
          # qlib/utils/__init__.py
      # E402 module level import not at top of file
        # Description: There are times when module level import is not available at the top of the file.
      # W503 line break before binary operator
        # Description: Since black formats the length of each line of code, it has to perform a line break when a line of arithmetic is too long.
      # E731 do not assign a lambda expression, use a def
        # Description: Restricts the use of lambda expressions, but at some point lambda expressions are required.
      # E203 whitespace before ':'
        # Description: If there is whitespace before ":", it cannot pass the black check.
    - name: Check Qlib with flake8
      run: |
        flake8 --ignore=E501,F541,E266,E402,W503,E731,E203 --per-file-ignores="__init__.py:F401,F403" qlib

    # https://github.com/python/mypy/issues/10600
    - name: Check Qlib with mypy
      run: |
        mypy qlib --install-types --non-interactive || true
        mypy qlib --verbose

    - name: Check Qlib ipynb with nbqa
      run: |
        nbqa black . -l 120 --check --diff
        nbqa pylint . --disable=C0104,C0114,C0115,C0116,C0301,C0302,C0411,C0413,C1802,R0401,R0801,R0902,R0903,R0911,R0912,R0913,R0914,R0915,R1720,W0105,W0123,W0201,W0511,W0613,W1113,W1514,E0401,E1121,C0103,C0209,R0402,R1705,R1710,R1725,R1735,W0102,W0212,W0221,W0223,W0231,W0237,W0612,W0621,W0622,W0703,W1309,E1102,E1136,W0719,W0104,W0404,C0412,W0611,C0410 --const-rgx='[a-z_][a-z0-9_]{2,30}$'

    - name: Test data downloads
      run: |
        python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn
        python scripts/get_data.py download_data --file_name rl_data.zip --target_dir tests/.data/rl

    - name: Install Lightgbm for MacOS
      if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }}
      run: |
        /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)"
        HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm
        # FIX MacOS error: Segmentation fault
        # reference: https://github.com/microsoft/LightGBM/issues/4229
        wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb
        brew unlink libomp
        brew install libomp.rb

    # Run after data downloads
    - name: Check Qlib ipynb with nbconvert
      # Running the nbconvert check on a macos-11 system results in a "Kernel died" error, so we've temporarily disabled macos-11 here.
      if: ${{ matrix.os != 'macos-11' }}
      run: |
        # add more ipynb files in future
        jupyter nbconvert --to notebook --execute examples/workflow_by_code.ipynb

    - name: Test workflow by config (install from source)
      # On macos-11 system, it will lead to "Segmentation fault: 11" error,
      # which may be caused by the excessive memory overhead of macos-11 system, so we disable macos-11 temporarily here.
      if: ${{ matrix.os != 'macos-11' }}
      run: |
        python -m pip install numba
        python qlib/workflow/cli.py examples/benchmarks/LightGBM/workflow_config_lightgbm_Alpha158.yaml

    - name: Unit tests with Pytest
      uses: nick-fields/retry@v2
      with:
        timeout_minutes: 60
        max_attempts: 3
        command: |
          cd tests
          python -m pytest . -m "not slow" --durations=0


Test qlib from source slow

name: Test qlib from source slow

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]

jobs:
  build:
    timeout-minutes: 720
    # we may retry for 3 times for `Unit tests with Pytest`

    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        # Since macos-latest changed from 12.7.4 to 14.4.1,
        # the minimum python version that matches a 14.4.1 version of macos is 3.10,
        # so we limit the macos version to macos-12.
        os: [windows-latest, ubuntu-20.04, ubuntu-22.04, macos-11, macos-12]
        # not supporting 3.6 due to annotations is not supported https://stackoverflow.com/a/52890129
        python-version: [3.7, 3.8]

    steps:
    - name: Test qlib from source slow
      uses: actions/checkout@v3

    # Since version 3.7 of python for MacOS is installed in CI, version 3.7.17, this version causes "_bz not found error".
    # So we make the version number of python 3.7 for MacOS more specific.
    # refs: https://github.com/actions/setup-python/issues/682
    - name: Set up Python ${{ matrix.python-version }}
      if: (matrix.os == 'macos-latest' && matrix.python-version == '3.7') || (matrix.os == 'macos-11' && matrix.python-version == '3.7')
      uses: actions/setup-python@v4
      with:
        python-version: "3.7.16"

    - name: Set up Python ${{ matrix.python-version }}
      if: (matrix.os != 'macos-latest' || matrix.python-version != '3.7') && (matrix.os != 'macos-11' || matrix.python-version != '3.7')
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Set up Python tools
      run: |
        python -m pip install --upgrade pip
        pip install --upgrade cython numpy
        pip install -e .[dev]

    - name: Downloads dependencies data
      run: |
        python scripts/get_data.py qlib_data --name qlib_data_simple --target_dir ~/.qlib/qlib_data/cn_data --interval 1d --region cn

    - name: Install Lightgbm for MacOS
      if: ${{ matrix.os == 'macos-11' || matrix.os == 'macos-latest' }}
      run: |
        /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Microsoft/qlib/main/.github/brew_install.sh)"
        HOMEBREW_NO_AUTO_UPDATE=1 brew install lightgbm
        # FIX MacOS error: Segmentation fault
        # reference: https://github.com/microsoft/LightGBM/issues/4229
        wget https://raw.githubusercontent.com/Homebrew/homebrew-core/fb8323f2b170bd4ae97e1bac9bf3e2983af3fdb0/Formula/libomp.rb
        brew unlink libomp
        brew install libomp.rb

    - name: Unit tests with Pytest
      uses: nick-fields/retry@v2
      with:
        timeout_minutes: 240
        max_attempts: 3
        command: |
          cd tests
          python -m pytest . -m "slow" --durations=0


Run TPU tests

name: Run TPU tests
on:
  push:
    branches:
      - master
      - "*.*.*"
    paths:
      - "ignite/**"
      - "tests/ignite/**"
      - "tests/run_tpu_tests.sh"
      - "tests/run_code_style.sh"
      - "requirements-dev.txt"
      - ".github/workflows/tpu-tests.yml"
  pull_request:
    paths:
      - "ignite/**"
      - "tests/ignite/**"
      - "tests/run_tpu_tests.sh"
      - "tests/run_code_style.sh"
      - "requirements-dev.txt"
      - ".github/workflows/tpu-tests.yml"
  workflow_dispatch:

concurrency:
  # <workflow_name>-<branch_name>-<true || commit_sha (if branch is protected)>
  group: tpu-tests-${{ github.ref_name }}-${{ !(github.ref_protected) || github.sha }}
  cancel-in-progress: true

jobs:
  tpu-tests:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        xla-version: [nightly]

    steps:
      - uses: actions/checkout@v4
      - name: Set up Python 3.9
        uses: actions/setup-python@v4
        with:
          python-version: "3.9"
          architecture: "x64"

      - name: Get year & week number
        id: get-date
        run: echo "date=$(/bin/date "+%Y-%U")" >> $GITHUB_OUTPUT
        shell: bash -l {0}

      - name: Get pip cache dir
        id: pip-cache
        run: |
          pip3 install -U pip
          echo "pip_cache=$(pip cache dir)" >> $GITHUB_OUTPUT
        shell: bash -l {0}

      - uses: actions/cache@v3
        with:
          path: |
            ${{ steps.pip-cache.outputs.pip_cache }}
          key: ${{ steps.get-date.outputs.date }}-pytorch-${{ runner.os }}-${{ matrix.xla-version }}-${{ hashFiles('requirements-dev.txt') }}
          restore-keys: |
            ${{ steps.get-date.outputs.date }}-pytorch-${{ runner.os }}-${{ matrix.xla-version }}-

      - name: Install Torch XLA and others
        run: |

          ## Install mkl (alternative approach to https://github.com/pytorch/xla/blob/b0ba29f98a695671972d4a4cc07441014dba2892/.kokoro/common.sh#L31-L32)
          sudo apt-get update && sudo apt-get install -y libopenblas-dev libomp5
          pip install mkl==2021.4.0

          ## Install torch & xla and torchvision
          pip install --pre  https://storage.googleapis.com/tpu-pytorch/wheels/colab/torch-nightly-cp39-cp39-linux_x86_64.whl
          pip install --pre  https://storage.googleapis.com/tpu-pytorch/wheels/colab/torch_xla-nightly-cp39-cp39-linux_x86_64.whl
          pip install --pre  https://storage.googleapis.com/tpu-pytorch/wheels/colab/torchvision-nightly-cp39-cp39-linux_x86_64.whl

          # Check installation
          python -c "import torch"

          ## Install test deps and Ignite
          pip install -r requirements-dev.txt
          python setup.py install

      # Download MNIST: https://github.com/pytorch/ignite/issues/1737
      # to "/tmp" for tpu tests
      - name: Download MNIST
        uses: pytorch-ignite/download-mnist-github-action@master
        with:
          target_dir: /tmp

      - name: Run Tests
        uses: nick-fields/retry@v3
        with:
          max_attempts: 5
          timeout_minutes: 25
          shell: bash
          command: |
            python -c "import torch_xla; print('torch xla version:', torch_xla.__version__)"
            bash tests/run_tpu_tests.sh
          new_command_on_retry: USE_LAST_FAILED=1 bash tests/run_tpu_tests.sh
        env:
          LD_LIBRARY_PATH: ${{ env.LD_LIBRARY_PATH }}:${{ env.Python_ROOT_DIR }}/lib
          XRT_DEVICE_MAP: "CPU:0;/job:localservice/replica:0/task:0/device:XLA_CPU:0"
          XRT_WORKERS: "localservice:0;grpc://localhost:40934"

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v3
        with:
          file: ./coverage.xml
          flags: tpu
          fail_ci_if_error: false


PyTorch version tests

name: PyTorch version tests

on:
  # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#onschedule
  schedule:
    # Run at 00:00 UTC Every Day
    - cron: "0 0 * * *"
  workflow_dispatch:

jobs:
  build:
    runs-on: ubuntu-latest
    timeout-minutes: 85
    strategy:
      max-parallel: 5
      fail-fast: false
      matrix:
        python-version: [3.8, 3.9, "3.10"]
        pytorch-version:
          [2.2.2, 2.1.2, 2.0.1, 1.13.1, 1.12.1, 1.11.0, 1.10.0, 1.8.1, 1.5.1]
        exclude:
          - pytorch-version: 1.5.1
            python-version: 3.9
          - pytorch-version: 1.5.1
            python-version: "3.10"

          # disabling python 3.9 support with PyTorch 1.7.1 and 1.8.1, to stop repeated pytorch-version test fail.
          # https://github.com/pytorch/ignite/issues/2383
          - pytorch-version: 1.8.1
            python-version: 3.9
          - pytorch-version: 1.8.1
            python-version: "3.10"

          - pytorch-version: 1.10.0
            python-version: "3.10"

          - pytorch-version: 1.11.0
            python-version: "3.10"

    steps:
      - uses: actions/checkout@v4

      - name: Get year & week number
        id: get-date
        run: echo "date=$(/bin/date "+%Y-%U")" >> $GITHUB_OUTPUT
        shell: bash -l {0}

      - name: Get pip cache dir
        id: pip-cache
        run: |
          python3 -m pip install -U pip
          echo "pip_cache=$(python3 -m pip cache dir)" >> $GITHUB_OUTPUT
        shell: bash -l {0}

      - uses: actions/cache@v3
        with:
          path: |
            ~/conda_pkgs_dir
            ${{ steps.pip-cache.outputs.pip_cache }}
          key: ${{ steps.get-date.outputs.date }}-pytorch-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.pytorch-version }}-${{ hashFiles('requirements-dev.txt') }}
          restore-keys: |
            ${{ steps.get-date.outputs.date }}-pytorch-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.pytorch-version }}-

      - name: Setup Miniconda
        uses: conda-incubator/setup-miniconda@v2
        with:
          miniconda-version: "latest"
          python-version: ${{ matrix.python-version }}
          use-only-tar-bz2: true # IMPORTANT: This needs to be set for caching to work properly!

      - name: Install dependencies
        shell: bash -l {0}
        run: |
          conda install pytorch=${{ matrix.pytorch-version }} torchvision cpuonly python=${{ matrix.python-version }} -c pytorch
          pip install -r requirements-dev.txt
          python setup.py install

          # pytorch>=1.9.0,<1.11.0 is using "from setuptools import distutils; distutils.version.LooseVersion" anti-pattern
          # which raises the error: AttributeError: module 'distutils' has no attribute 'version' for setuptools>59
          bad_pth_version=$(python -c "import torch; print('.'.join(torch.__version__.split('.')[:2]) in ['1.9', '1.10'])")
          if [ "${bad_pth_version}" == "True" ]; then
            pip install --upgrade "setuptools<59"
            python -c "from setuptools import distutils; distutils.version.LooseVersion"
          fi

      - name: Download MNIST
        uses: pytorch-ignite/download-mnist-github-action@master
        with:
          target_dir: /tmp

      - name: Run Tests
        uses: nick-fields/retry@v3
        with:
          max_attempts: 5
          timeout_minutes: 15
          shell: bash
          command: bash -l tests/run_cpu_tests.sh "not test_time_profilers"
          new_command_on_retry: USE_LAST_FAILED=1 bash -l tests/run_cpu_tests.sh "not test_time_profilers"

  create-issue:
    runs-on: ubuntu-latest
    # https://docs.github.com/en/actions/reference/context-and-expression-syntax-for-github-actions#needs-context
    needs: build
    if: always() && needs.build.result == 'failure'
    steps:
      - uses: actions/checkout@v4
      - uses: JasonEtco/create-an-issue@v2
        name: Create issue if pytorch version tests failed
        with:
          filename: .github/failed_schedule_issue_template.md
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


Run Horovod tests

name: Run Horovod tests
on:
  push:
    branches:
      - master
      - "*.*.*"
    paths:
      - "ignite/**"
      - "tests/ignite/**"
      - "tests/run_cpu_tests.sh"
      - ".github/workflows/hvd-tests.yml"
  pull_request:
    paths:
      - "ignite/**"
      - "tests/ignite/**"
      - "tests/run_cpu_tests.sh"
      - ".github/workflows/hvd-tests.yml"
  workflow_dispatch:

concurrency:
  # <workflow_name>-<branch_name>-<true || commit_sha (if branch is protected)>
  group: hvd-tests-${{ github.ref_name }}-${{ !(github.ref_protected) || github.sha }}
  cancel-in-progress: true

jobs:
  horovod-tests:
    runs-on: ubuntu-latest
    timeout-minutes: 60
    strategy:
      matrix:
        python-version: [3.8]
        pytorch-channel: [pytorch]

    steps:
      - uses: actions/checkout@v4

      - name: Get year & week number
        id: get-date
        run: echo "date=$(/bin/date "+%Y-%U")" >> $GITHUB_OUTPUT
        shell: bash -l {0}

      - name: Get pip cache dir
        id: pip-cache
        run: |
          python3 -m pip install -U pip
          echo "pip_cache=$(python3 -m pip cache dir)" >> $GITHUB_OUTPUT
        shell: bash -l {0}

      - uses: actions/cache@v3
        with:
          path: |
            ~/conda_pkgs_dir
            ${{ steps.pip-cache.outputs.pip_cache }}
          key: ${{ steps.get-date.outputs.date }}-horovod-${{ hashFiles('requirements-dev.txt') }}

      - uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        shell: bash -l {0}
        run: |

          #install other dependencies
          pip install torch torchvision -f https://download.pytorch.org/whl/cpu/torch_stable.html
          pip install -r requirements-dev.txt
          pip install horovod
          python setup.py install

      # Download MNIST: https://github.com/pytorch/ignite/issues/1737
      # to "/tmp" for cpu tests
      - name: Download MNIST
        uses: pytorch-ignite/download-mnist-github-action@master
        with:
          target_dir: /tmp

      - name: Run Tests
        uses: nick-fields/retry@v3
        with:
          max_attempts: 5
          timeout_minutes: 15
          shell: bash
          command: bash tests/run_cpu_tests.sh
          new_command_on_retry: USE_LAST_FAILED=1 bash tests/run_cpu_tests.sh

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v3
        with:
          file: ./coverage.xml
          flags: hvd-cpu
          fail_ci_if_error: false


Run unit tests

name: Run unit tests
on:
  push:
    branches:
      - master
      - "*.*.*"
    paths:
      - "ignite/**"
      - "tests/ignite/**"
      - "tests/run_cpu_tests.sh"
      - "tests/run_code_style.sh"
      - "examples/**.py"
      - "requirements-dev.txt"
      - ".github/workflows/unit-tests.yml"
  pull_request:
    paths:
      - "ignite/**"
      - "tests/ignite/**"
      - "tests/run_cpu_tests.sh"
      - "tests/run_code_style.sh"
      - "examples/**.py"
      - "requirements-dev.txt"
      - ".github/workflows/unit-tests.yml"
  workflow_dispatch:

concurrency:
  # <workflow_name>-<branch_name>-<true || commit_sha (if branch is protected)>
  group: unit-tests-${{ github.ref_name }}-${{ !(github.ref_protected) || github.sha }}
  cancel-in-progress: true

jobs:
  cpu-tests:
    runs-on: ${{ matrix.os }}
    timeout-minutes: 85
    defaults:
      run:
        shell: bash
    strategy:
      max-parallel: 10
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
        pytorch-channel: [pytorch, pytorch-nightly]
        include:
          # includes a single build on windows
          - os: windows-latest
            pytorch-channel: pytorch
            python-version: 3.9
            skip-distrib-tests: 1
          # includes a single build on macosx
          - os: macos-latest
            pytorch-channel: pytorch
            python-version: 3.9
            skip-distrib-tests: 1

    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Get year & week number
        id: get-date
        run: |
          echo "date=$(/bin/date "+%Y-%U")" >> $GITHUB_OUTPUT

      - name: Get pip cache dir
        id: pip-cache
        run: |
          pip install -U pip || python -m pip install -U pip
          echo "pip_cache=$(pip cache dir)" >> $GITHUB_OUTPUT

      - uses: actions/cache@v3
        with:
          path: |
            ${{ steps.pip-cache.outputs.pip_cache }}
          key: ${{ steps.get-date.outputs.date }}-pytorch-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.pytorch-channel }}-${{ hashFiles('requirements-dev.txt') }}
          restore-keys: |
            ${{ steps.get-date.outputs.date }}-pytorch-${{ runner.os }}-${{ matrix.python-version }}-${{ matrix.pytorch-channel }}-

      - run: pip install pip wheel setuptools -Uqq

      - name: Install PyTorch
        if: ${{ matrix.pytorch-channel == 'pytorch' }}
        run: pip install torch torchvision -f https://download.pytorch.org/whl/cpu/torch_stable.html

      - name: Install PyTorch (nightly)
        if: ${{ matrix.pytorch-channel == 'pytorch-nightly' }}
        run: pip install torch torchvision -f https://download.pytorch.org/whl/nightly/cpu/torch_nightly.html --pre

      - name: Install dependencies
        run: |
          pip install -r requirements-dev.txt
          python setup.py install
          pip list

      - name: Check code formatting
        run: |
          bash ./tests/run_code_style.sh install
          bash ./tests/run_code_style.sh lint

      - name: Run Mypy
        # https://github.com/pytorch/ignite/pull/2780
        #
        if: ${{ matrix.os == 'ubuntu-latest' && matrix.pytorch-channel == 'pytorch-nightly'}}
        run: |
          bash ./tests/run_code_style.sh mypy

      # Download MNIST: https://github.com/pytorch/ignite/issues/1737
      # to "/tmp" for unit tests
      - name: Download MNIST
        uses: pytorch-ignite/download-mnist-github-action@master
        with:
          target_dir: /tmp

      # Copy MNIST to "." for the examples
      - name: Copy MNIST
        run: |
          cp -R /tmp/MNIST .

      - name: Run Tests
        uses: nick-fields/retry@v3
        with:
          max_attempts: 5
          timeout_minutes: 15
          shell: bash
          command: SKIP_DISTRIB_TESTS=${{ matrix.skip-distrib-tests }} bash tests/run_cpu_tests.sh
          new_command_on_retry: USE_LAST_FAILED=1 SKIP_DISTRIB_TESTS=${{ matrix.skip-distrib-tests }} bash tests/run_cpu_tests.sh

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v3
        with:
          file: ./coverage.xml
          flags: cpu
          fail_ci_if_error: false

      - name: Run MNIST Examples
        run: |
          # MNIST
          # 1) mnist.py
          python examples/mnist/mnist.py --epochs=1

      - name: Run MNIST with loggers Examples
        if: ${{ matrix.os == 'ubuntu-latest' }}
        run: |
          # 2) mnist_with_visdom.py
          python -c "from visdom.server.build import download_scripts; download_scripts()" # download scripts : https://github.com/facebookresearch/visdom/blob/master/py/server.py#L929
          python -m visdom.server &
          sleep 10
          python examples/mnist/mnist_with_visdom.py --epochs=1
          kill %1
          # 3.1) mnist_with_tensorboard.py with tbX
          python examples/mnist/mnist_with_tensorboard.py --epochs=1
          # 3.2) mnist_with_tensorboard.py with native torch tb
          pip uninstall -y tensorboardX
          python examples/mnist/mnist_with_tensorboard.py --epochs=1

      - name: Run MNIST Example With Crash
        if: ${{ matrix.os == 'ubuntu-latest' }}
        continue-on-error: true
        run: |
          # 4) mnist_save_resume_engine.py
          python examples/mnist/mnist_save_resume_engine.py --epochs=2 --crash_iteration 1100

      - name: Resume MNIST from previous crash
        if: ${{ matrix.os == 'ubuntu-latest' }}
        run: |
          python examples/mnist/mnist_save_resume_engine.py --epochs=2 --resume_from=/tmp/mnist_save_resume/checkpoint_1.pt

      - name: Run GAN example
        if: ${{ matrix.os == 'ubuntu-latest' }}
        run: |
          # DCGAN
          python examples/gan/dcgan.py --dataset fake --dataroot /tmp/fakedata --output-dir /tmp/outputs-dcgan --batch-size 2 --epochs 2  --workers 0

      - name: Run RL Examples
        if: ${{ matrix.os == 'ubuntu-latest' }}
        run: |
          # RL
          # 1) Actor-Critic
          python examples/reinforcement_learning/actor_critic.py --max-episodes=2
          # 2) Reinforce
          python examples/reinforcement_learning/reinforce.py --max-episodes=2

      - name: Run Neural Style Example
        if: ${{ matrix.os == 'ubuntu-latest' }}
        run: |
          #fast-neural-style
          #train
          mkdir -p ~/.cache/torch/checkpoints/ && wget "https://download.pytorch.org/models/vgg16-397923af.pth" -O ~/.cache/torch/checkpoints/vgg16-397923af.pth
          python examples/fast_neural_style/neural_style.py train --epochs 1 --cuda 0 --dataset test --dataroot . --image_size 32 --style_image examples/fast_neural_style/images/style_images/mosaic.jpg --style_size 32
      - name: Run SR Example
        if: ${{ matrix.os == 'ubuntu-latest' }}
        run: |
          # Super-Resolution
          python examples/super_resolution/main.py --upscale_factor 3 --crop_size 180 --batch_size 4 --test_batch_size 100 --n_epochs 1 --lr 0.001 --threads 2 --debug


Nightly Releases

name: Nightly Releases

on:
  # https://docs.github.com/en/free-pro-team@latest/actions/reference/workflow-syntax-for-github-actions#onschedule
  schedule:
    # Run at 00:00 UTC Every Day
    - cron: "0 0 * * *"

jobs:
  build-publish:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup Miniconda
        uses: conda-incubator/setup-miniconda@v2
        with:
          miniconda-version: "latest"
          python-version: "3.10"

      - name: Setup nightly version
        run: |
          sed -i "s/__version__ = \"\(.*\)\"/__version__ = \"\1.dev$(date -u +%Y%m%d)\"/g" ignite/__init__.py
          cat ignite/__init__.py

      - name: Install dependencies
        shell: bash -l {0}
        run: |
          conda install -y pytorch torchvision cpuonly -c pytorch-nightly
          pip install -r requirements-dev.txt

      - name: Build and Publish Conda binaries
        shell: bash -l {0}
        env:
          ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }}
          UPLOAD_USER: "pytorch-nightly"
        run: |
          chmod +x ./conda.recipe/build_and_upload.sh
          ./conda.recipe/build_and_upload.sh

      - name: Build and Publish PyPI binaries
        shell: bash -l {0}
        run: |
          # workaround to fix https://github.com/pytorch/ignite/issues/2373
          pip uninstall -y twine pkginfo
          pip install --upgrade --no-cache-dir twine 'pkginfo>=1.8.2'
          python setup.py sdist bdist_wheel
          twine --version
          twine check dist/*
          TWINE_USERNAME="${{ secrets.PYPI_USER }}" TWINE_PASSWORD="${{ secrets.PYPI_TOKEN }}" twine upload --verbose dist/*

      - uses: JasonEtco/create-an-issue@v2
        name: Create issue if nightly releases failed
        if: failure()
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          filename: .github/failed_schedule_issue_template.md


Build docs

name: Build docs

on:
  push:
    branches:
      - master
  pull_request:
    paths-ignore:
      - "tests/**"
      - "docker/**"
  release:
    types: [published]
  workflow_dispatch:

jobs:
  build-deploy:
    permissions:
      contents: write
    if: (github.ref == 'refs/heads/master' && github.event_name == 'push') || github.event_name == 'release'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v4
        with:
          python-version: "3.10"

      - run: sudo npm install katex -g
      - uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: pip-${{ hashFiles('requirements-dev.txt') }}-${{ hashFiles('docs/requirements.txt') }}

      - name: Install docs deps
        run: bash .github/workflows/install_docs_deps.sh

      - name: Build docs
        run: bash .github/workflows/build_docs.sh

      - name: Deploy docs
        uses: peaceiris/actions-gh-pages@v3
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: docs/build/html
          publish_branch: gh-pages
          commit_message: Deploy pytorch/ignite docs
          force_orphan: true

  linkcheck:
    if: github.event_name == 'pull_request' || github.event_name == 'push'
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v4
        with:
          python-version: "3.10"

      - uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: pip-${{ hashFiles('requirements-dev.txt') }}-${{ hashFiles('docs/requirements.txt') }}

      - name: Install docs deps
        run: bash .github/workflows/install_docs_deps.sh

      - name: make linkcheck
        working-directory: ./docs/
        run: make linkcheck

  doctest:
    if: github.event_name == 'pull_request' || github.event_name == 'push'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v4
        with:
          python-version: "3.10"

      - run: sudo npm install katex -g
      - uses: actions/cache@v3
        with:
          path: ~/.cache/pip
          key: pip-${{ hashFiles('requirements-dev.txt') }}-${{ hashFiles('docs/requirements.txt') }}

      - name: Install docs deps
        run: bash .github/workflows/install_docs_deps.sh

      - name: make doctest
        working-directory: ./docs/
        run: |
          make html
          make doctest
          make coverage


Build/Publish Docker Images

name: Build/Publish Docker Images

on:
  pull_request:
    paths:
      - docker/**
      - ".github/workflows/docker-build.yml"
  release:
    types: [published]
  workflow_dispatch:

concurrency:
  # <workflow_name>-<branch_name>-<true || commit_sha (if branch is protected)>
  group: docker-build-publish-${{ github.ref_name }}-${{ !(github.ref_protected) || github.sha }}
  cancel-in-progress: true

jobs:

  setup:
    name: Setup
    runs-on: ubuntu-latest
    outputs:
      modified: ${{ steps.set-modified.outputs.modified }}
      pth_version: ${{ steps.set-versions.outputs.pth_version }}
      hvd_version: ${{ steps.set-versions.outputs.hvd_version }}
      msdp_version: ${{ steps.set-versions.outputs.msdp_version }}
    steps:
      - uses: actions/checkout@v4
      - name: Changed Files Exporter
        if: github.event_name == 'pull_request'
        id: files
        uses: umani/changed-files@v4.1.0
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
      - name: Get a list of modified files
        if: github.event_name == 'pull_request'
        run: echo "modified=${{ steps.files.outputs.files_created }} ${{ steps.files.outputs.files_updated }}" >> $GITHUB_ENV
      - name: Set outputs
        id: set-modified
        run: echo "modified=${{ env.modified }}" >> $GITHUB_OUTPUT
      - name: Set versions
        id: set-versions
        working-directory: docker
        run: |
          echo "pth_version=$(python -c "import configparser; cfg=configparser.ConfigParser(); cfg.read('docker.cfg'); print(cfg.get('DEFAULT', 'build_docker_image_pytorch_version'))")" >> $GITHUB_OUTPUT
          echo "hvd_version=$(python -c "import configparser; cfg=configparser.ConfigParser(); cfg.read('docker.cfg'); print(cfg.get('DEFAULT', 'build_docker_image_hvd_version'))")" >> $GITHUB_OUTPUT

  build-hvd:
    name: Build all Horovod flavoured PyTorch-Ignite images
    needs: setup
    if: github.event_name != 'pull_request' || (contains(needs.setup.outputs.modified, 'hvd/') || contains(needs.setup.outputs.modified, 'docker.cfg'))

    # Cherry-picked from https://github.com/pytorch/test-infra/blob/main/.github/workflows/linux_job.yml
    env:
      REPOSITORY: ${{ github.repository }}
      PR_NUMBER: ${{ github.event.pull_request.number }}
    runs-on: linux.4xlarge
    steps:
      - name: Clean workspace
        run: |
          echo "::group::Cleanup debug output"
          sudo rm -rfv "${GITHUB_WORKSPACE}"
          mkdir -p "${GITHUB_WORKSPACE}"
          echo "::endgroup::"

      - name: Checkout repository (pytorch/test-infra)
        uses: actions/checkout@v3
        with:
          # Support the use case where we need to checkout someone's fork
          repository: pytorch/test-infra
          path: test-infra

      - name: Setup Linux
        uses: ./test-infra/.github/actions/setup-linux

      - name: Checkout repository (${{ github.repository }})
        uses: actions/checkout@v3
        with:
          # Support the use case where we need to checkout someone's fork
          repository: ${{ github.repository }}
          ref: ${{ github.ref }}
          path: ${{ github.repository }}
          fetch-depth: 1

      - name: Build docker images
        working-directory: ${{ github.repository }}/docker
        run: |

          export PTH_VERSION=${{ needs.setup.outputs.pth_version }}
          export HVD_VERSION=${{ needs.setup.outputs.hvd_version }}

          bash build.sh hvd hvd-base
          bash build.sh hvd hvd-vision
          bash build.sh hvd hvd-nlp

      - name: Publish docker images
        if: github.event_name == 'workflow_dispatch' || github.event_name == 'release'
        env:
          DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
          DOCKER_USER: ${{ secrets.DOCKER_USER }}
        working-directory: ${{ github.repository }}/docker
        run: |
          bash push_all.sh hvd-base
          bash push_all.sh hvd-vision
          bash push_all.sh hvd-nlp

  build-hvd-apex:
    name: Build all Horovod + Apex flavoured PyTorch-Ignite images
    needs: setup
    if: github.event_name != 'pull_request' || (contains(needs.setup.outputs.modified, 'hvd/') || contains(needs.setup.outputs.modified, 'docker.cfg'))

    # Cherry-picked from https://github.com/pytorch/test-infra/blob/main/.github/workflows/linux_job.yml
    env:
      REPOSITORY: ${{ github.repository }}
      PR_NUMBER: ${{ github.event.pull_request.number }}
    runs-on: linux.12xlarge
    steps:
      - name: Clean workspace
        run: |
          echo "::group::Cleanup debug output"
          sudo rm -rfv "${GITHUB_WORKSPACE}"
          mkdir -p "${GITHUB_WORKSPACE}"
          echo "::endgroup::"

      - name: Checkout repository (pytorch/test-infra)
        uses: actions/checkout@v3
        with:
          # Support the use case where we need to checkout someone's fork
          repository: pytorch/test-infra
          path: test-infra

      - name: Setup Linux
        uses: ./test-infra/.github/actions/setup-linux

      - name: Checkout repository (${{ github.repository }})
        uses: actions/checkout@v3
        with:
          # Support the use case where we need to checkout someone's fork
          repository: ${{ github.repository }}
          ref: ${{ github.ref }}
          path: ${{ github.repository }}
          fetch-depth: 1

      - name: Build docker images
        working-directory: ${{ github.repository }}/docker
        run: |

          export PTH_VERSION=${{ needs.setup.outputs.pth_version }}
          export HVD_VERSION=${{ needs.setup.outputs.hvd_version }}

          bash build.sh hvd hvd-apex
          bash build.sh hvd hvd-apex-vision
          bash build.sh hvd hvd-apex-nlp

      - name: Publish docker images
        if: github.event_name == 'workflow_dispatch' || github.event_name == 'release'
        env:
          DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
          DOCKER_USER: ${{ secrets.DOCKER_USER }}
        working-directory: ${{ github.repository }}/docker
        run: |
          bash push_all.sh hvd-apex
          bash push_all.sh hvd-apex-vision
          bash push_all.sh hvd-apex-nlp

  build-main:
    name: Build all PyTorch-Ignite images
    needs: setup
    if: github.event_name != 'pull_request' || (contains(needs.setup.outputs.modified, 'main/') || contains(needs.setup.outputs.modified, 'docker.cfg'))

    # Cherry-picked from https://github.com/pytorch/test-infra/blob/main/.github/workflows/linux_job.yml
    env:
      REPOSITORY: ${{ github.repository }}
      PR_NUMBER: ${{ github.event.pull_request.number }}
    runs-on: linux.4xlarge
    steps:
      - name: Clean workspace
        run: |
          echo "::group::Cleanup debug output"
          sudo rm -rfv "${GITHUB_WORKSPACE}"
          mkdir -p "${GITHUB_WORKSPACE}"
          echo "::endgroup::"

      - name: Checkout repository (pytorch/test-infra)
        uses: actions/checkout@v3
        with:
          # Support the use case where we need to checkout someone's fork
          repository: pytorch/test-infra
          path: test-infra

      - name: Setup Linux
        uses: ./test-infra/.github/actions/setup-linux

      - name: Checkout repository (${{ github.repository }})
        uses: actions/checkout@v3
        with:
          # Support the use case where we need to checkout someone's fork
          repository: ${{ github.repository }}
          ref: ${{ github.ref }}
          path: ${{ github.repository }}
          fetch-depth: 1

      - name: Build docker images
        working-directory: ${{ github.repository }}/docker
        run: |

          export PTH_VERSION=${{ needs.setup.outputs.pth_version }}
          export HVD_VERSION=${{ needs.setup.outputs.hvd_version }}

          bash build.sh main base
          bash build.sh main vision
          bash build.sh main nlp

      - name: Publish docker images
        if: github.event_name == 'workflow_dispatch' || github.event_name == 'release'
        env:
          DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
          DOCKER_USER: ${{ secrets.DOCKER_USER }}
        working-directory: ${{ github.repository }}/docker
        run: |
          bash push_all.sh base
          bash push_all.sh vision
          bash push_all.sh nlp

  build-main-apex:
    name: Build all PyTorch-Ignite images with Apex
    needs: setup
    if: github.event_name != 'pull_request' || (contains(needs.setup.outputs.modified, 'main/') || contains(needs.setup.outputs.modified, 'docker.cfg'))

    # Cherry-picked from https://github.com/pytorch/test-infra/blob/main/.github/workflows/linux_job.yml
    env:
      REPOSITORY: ${{ github.repository }}
      PR_NUMBER: ${{ github.event.pull_request.number }}
    runs-on: linux.12xlarge
    steps:
      - name: Clean workspace
        run: |
          echo "::group::Cleanup debug output"
          sudo rm -rfv "${GITHUB_WORKSPACE}"
          mkdir -p "${GITHUB_WORKSPACE}"
          echo "::endgroup::"

      - name: Checkout repository (pytorch/test-infra)
        uses: actions/checkout@v3
        with:
          # Support the use case where we need to checkout someone's fork
          repository: pytorch/test-infra
          path: test-infra

      - name: Setup Linux
        uses: ./test-infra/.github/actions/setup-linux

      - name: Checkout repository (${{ github.repository }})
        uses: actions/checkout@v3
        with:
          # Support the use case where we need to checkout someone's fork
          repository: ${{ github.repository }}
          ref: ${{ github.ref }}
          path: ${{ github.repository }}
          fetch-depth: 1

      - name: Build docker images
        working-directory: ${{ github.repository }}/docker
        run: |

          export PTH_VERSION=${{ needs.setup.outputs.pth_version }}
          export HVD_VERSION=${{ needs.setup.outputs.hvd_version }}

          bash build.sh main apex
          bash build.sh main apex-vision
          bash build.sh main apex-nlp

      - name: Publish docker images
        if: github.event_name == 'workflow_dispatch' || github.event_name == 'release'
        env:
          DOCKER_TOKEN: ${{ secrets.DOCKER_TOKEN }}
          DOCKER_USER: ${{ secrets.DOCKER_USER }}
        working-directory: ${{ github.repository }}/docker
        run: |
          bash push_all.sh apex
          bash push_all.sh apex-vision
          bash push_all.sh apex-nlp


Triage

name: Triage

on:
  pull_request_target:
#     types: [opened]
#   issues:
#     types: [opened]

jobs:
  triage:
    permissions:
      contents: read
      pull-requests: write
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Pull Request Labeler
        uses: actions/labeler@v5
        with:
          configuration-path: .github/pr-labeler-config.yml
          repo-token: "${{ secrets.GITHUB_TOKEN }}"

# Turned off due to unexpected behavior on issue opening+labeling? https://github.com/pytorch/ignite/issues/1836
#       - name: Welcome
#         uses: actions/first-interaction@v1
#         with:
#           issue-message: "**Thank you for opening your First Issue!**\n\nWe appreciate a lot user's feedback on what we are doing!\n\nIf you'd like to contribute to the project, please check out our [Contributing Guide](https://github.com/pytorch/ignite/blob/master/CONTRIBUTING.md)."
#           pr-message: "**Thank you for opening your First Pull Request!**\nWe appreciate a lot community contributions as pull requests!\n\nIf you would like to get more details on the project development, please take a look at our [Contributing Guide](https://github.com/pytorch/ignite/blob/master/CONTRIBUTING.md)."
#           repo-token: ${{ secrets.GITHUB_TOKEN }}


Anaconda Stable Releases

name: Anaconda Stable Releases

on:
  release:
    types: [published]

jobs:
  conda-build-publish:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup Miniconda
        uses: conda-incubator/setup-miniconda@v2
        with:
          miniconda-version: "latest"
          python-version: "3.10"

      - name: Install dependencies
        shell: bash -l {0}
        run: |
          conda install -y pytorch torchvision cpuonly -c pytorch
          python setup.py install

      - name: Build and Publish Conda binaries
        shell: bash -l {0}
        env:
          ANACONDA_TOKEN: ${{ secrets.ANACONDA_TOKEN }}
          UPLOAD_USER: "pytorch"
        run: |
          chmod +x ./conda.recipe/build_and_upload.sh
          ./conda.recipe/build_and_upload.sh


PyPI Stable Releases

name: PyPI Stable Releases

on:
  release:
    types: [published]

jobs:
  build-publish:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Setup Miniconda
        uses: conda-incubator/setup-miniconda@v2
        with:
          miniconda-version: "latest"
          python-version: "3.10"

      - name: Install dependencies
        shell: bash -l {0}
        run: |
          conda install -y pytorch torchvision cpuonly -c pytorch
          pip install -r requirements-dev.txt

      - name: Build and Publish PyPI binaries
        shell: bash -l {0}
        run: |
          # workaround to fix https://github.com/pytorch/ignite/issues/2373
          pip uninstall -y twine pkginfo
          pip install --upgrade --no-cache-dir twine 'pkginfo>=1.8.2'
          python setup.py sdist bdist_wheel
          twine --version
          twine check dist/*
          TWINE_USERNAME="${{ secrets.PYPI_USER }}" TWINE_PASSWORD="${{ secrets.PYPI_TOKEN }}" twine upload --verbose dist/*


Run unit tests on GPUs

name: Run unit tests on GPUs
on:
  push:
    paths:
      - "ignite/**"
      - "tests/ignite/**"
      - "tests/run_gpu_tests.sh"
      - "tests/run_code_style.sh"
      - "examples/**.py"
      - "requirements-dev.txt"
      - ".github/workflows/gpu-tests.yml"
  workflow_dispatch:

concurrency:
  # <workflow_name>-<branch_name>-<true || commit_sha (if branch is protected)>
  group: gpu-tests-${{ github.ref_name }}-${{ !(github.ref_protected) || github.sha }}
  cancel-in-progress: true

# Cherry-picked from https://github.com/pytorch/test-infra/blob/main/.github/workflows/linux_job.yml

jobs:
  gpu-tests:
    strategy:
      matrix:
        pytorch-channel: [pytorch, pytorch-nightly]
      fail-fast: false
    env:
      DOCKER_IMAGE: "pytorch/conda-builder:cuda12.1"
      REPOSITORY: ${{ github.repository }}
      PR_NUMBER: ${{ github.event.pull_request.number }}
    runs-on: linux.8xlarge.nvidia.gpu
    timeout-minutes: 85

    steps:
      - name: Clean workspace
        run: |
          echo "::group::Cleanup debug output"
          sudo rm -rfv "${GITHUB_WORKSPACE}"
          mkdir -p "${GITHUB_WORKSPACE}"
          echo "::endgroup::"

      - name: Checkout repository (pytorch/test-infra)
        uses: actions/checkout@v3
        with:
          # Support the use case where we need to checkout someone's fork
          repository: pytorch/test-infra
          path: test-infra

      - name: Setup Linux
        uses: ./test-infra/.github/actions/setup-linux

      - name: Pull docker image
        uses: ./test-infra/.github/actions/pull-docker-image
        with:
          docker-image: ${{ env.DOCKER_IMAGE }}

      - name: Checkout repository (${{ github.repository }})
        uses: actions/checkout@v3
        with:
          # Support the use case where we need to checkout someone's fork
          repository: ${{ github.repository }}
          ref: ${{ github.ref }}
          path: ${{ github.repository }}
          fetch-depth: 1

      - name: Start Pytorch container
        working-directory: ${{ github.repository }}
        run: |
          docker run --name pthd --gpus=all --rm \
            --cap-add=SYS_PTRACE \
            --detach \
            --ipc=host \
            --security-opt seccomp=unconfined \
            --shm-size=2g \
            --tty \
            --ulimit stack=10485760:83886080 \
            -v $PWD:/work \
            -w /work \
            ${DOCKER_IMAGE}

          script=$(cat << EOF

            set -xe

            nvidia-smi
            ls -alh

            conda --version
            python --version

          EOF
          )
          docker exec -t pthd /bin/bash -c "${script}"

      - name: Install PyTorch and dependencies
        continue-on-error: false
        run: |

          script=$(cat << EOF

          set -xe

          # Install PyTorch
          if [ "${{ matrix.pytorch-channel }}" == "pytorch" ]; then
            pip install --upgrade torch torchvision --index-url https://download.pytorch.org/whl/cu121
          else
            pip install --upgrade --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu121
          fi

          python -c "import torch; print(torch.__version__, ', CUDA is available: ', torch.cuda.is_available()); exit(not torch.cuda.is_available())"
          pip list

          # Install dependencies
          pip install -r requirements-dev.txt
          pip install -e .

          EOF
          )

          docker exec -t pthd /bin/bash -c "${script}"

      - name: Run GPU Unit Tests
        continue-on-error: false
        uses: nick-fields/retry@v2.9.0
        with:
          max_attempts: 5
          timeout_minutes: 25
          shell: bash
          command: docker exec -t pthd /bin/bash -xec 'bash tests/run_gpu_tests.sh 2'
          new_command_on_retry: docker exec -e USE_LAST_FAILED=1 -t pthd /bin/bash -xec 'bash tests/run_gpu_tests.sh 2'

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v3
        with:
          file: ${{ github.repository }}/coverage.xml
          flags: gpu-2
          fail_ci_if_error: false

      - name: Run examples in container
        continue-on-error: false
        run: |
          SCRIPT=$(cat << EOF

          set -xe

          # Install additional example dependencies
          pip install fire

          # Check training on cifar10, run without backend
          ## initial run
          CI=1 python examples/cifar10/main.py run --checkpoint_every=200 --stop_iteration=500
          ## resume
          CI=1 python examples/cifar10/main.py run --checkpoint_every=200 --num_epochs=7 --resume-from=/tmp/output-cifar10/resnet18_backend-None-1_stop-on-500/training_checkpoint_400.pt

          # Check training on cifar10, run with NCCL backend using torchrun
          ## initial run
          CI=1 torchrun --nproc_per_node=2 examples/cifar10/main.py run --backend=nccl --checkpoint_every=200 --stop_iteration=500
          ## resume
          CI=1 torchrun --nproc_per_node=2 examples/cifar10/main.py run --backend=nccl --checkpoint_every=200 --num_epochs=7 --resume-from=/tmp/output-cifar10/resnet18_backend-nccl-2_stop-on-500/training_checkpoint_400.pt

          # Check training on cifar10, run with NCCL backend using spawn
          ## initial run
          CI=1 python -u examples/cifar10/main.py run --backend=nccl --nproc_per_node=2 --checkpoint_every=200 --stop_iteration=500
          ## resume
          CI=1 python -u examples/cifar10/main.py run --backend=nccl --nproc_per_node=2 --checkpoint_every=200 --num_epochs=7 --resume-from=/tmp/output-cifar10/resnet18_backend-nccl-2_stop-on-500/training_checkpoint_400.pt

          EOF
          )

          docker exec -t pthd /bin/bash -c "${script}"

      - name: Teardown Linux
        if: ${{ always() }}
        uses: ./test-infra/.github/actions/teardown-linux


Run HVD-specific unit tests on GPUs

name: Run HVD-specific unit tests on GPUs
on:
  push:
    paths:
      - "ignite/**"
      - "tests/ignite/**"
      - "tests/run_gpu_tests.sh"
      - "tests/run_code_style.sh"
      - "examples/**.py"
      - "requirements-dev.txt"
      - ".github/workflows/gpu-hvd-tests.yml"
  workflow_dispatch:

concurrency:
  # <workflow_name>-<branch_name>-<true || commit_sha (if branch is protected)>
  group: gpu-hvd-tests-${{ github.ref_name }}-${{ !(github.ref_protected) || github.sha }}
  cancel-in-progress: true

# Cherry-picked from https://github.com/pytorch/test-infra/blob/main/.github/workflows/linux_job.yml

jobs:
  gpu-hvd-tests:
    strategy:
      matrix:
        pytorch-channel: [pytorch]
      fail-fast: false
    env:
      DOCKER_IMAGE: "pytorch/conda-builder:cuda12.1"
      REPOSITORY: ${{ github.repository }}
      PR_NUMBER: ${{ github.event.pull_request.number }}
    runs-on: linux.8xlarge.nvidia.gpu
    timeout-minutes: 60

    steps:
      - name: Clean workspace
        run: |
          echo "::group::Cleanup debug output"
          sudo rm -rfv "${GITHUB_WORKSPACE}"
          mkdir -p "${GITHUB_WORKSPACE}"
          echo "::endgroup::"

      - name: Checkout repository (pytorch/test-infra)
        uses: actions/checkout@v3
        with:
          # Support the use case where we need to checkout someone's fork
          repository: pytorch/test-infra
          path: test-infra

      - name: Setup Linux
        uses: ./test-infra/.github/actions/setup-linux

      - name: Pull docker image
        uses: ./test-infra/.github/actions/pull-docker-image
        with:
          docker-image: ${{ env.DOCKER_IMAGE }}

      - name: Checkout repository (${{ github.repository }})
        uses: actions/checkout@v3
        with:
          # Support the use case where we need to checkout someone's fork
          repository: ${{ github.repository }}
          ref: ${{ github.ref }}
          path: ${{ github.repository }}
          fetch-depth: 1

      - name: Start Pytorch container
        working-directory: ${{ github.repository }}
        run: |
          docker run --name pthd --gpus=all --rm \
            --cap-add=SYS_PTRACE \
            --detach \
            --ipc=host \
            --security-opt seccomp=unconfined \
            --shm-size=2g \
            --tty \
            --ulimit stack=10485760:83886080 \
            -v $PWD:/work \
            -w /work \
            ${DOCKER_IMAGE}

          script=$(cat << EOF

            set -xe

            nvidia-smi
            ls -alh

            conda --version
            python --version

          EOF
          )
          docker exec -t pthd /bin/bash -c "${script}"

      - name: Install PyTorch and dependencies
        continue-on-error: false
        run: |

          script=$(cat << EOF

          set -xe

          # Install PyTorch
          if [ "${{ matrix.pytorch-channel }}" == "pytorch" ]; then
            pip install --upgrade torch torchvision --index-url https://download.pytorch.org/whl/cu121
          else
            pip install --upgrade --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cu121
          fi

          python -c "import torch; print(torch.__version__, ', CUDA is available: ', torch.cuda.is_available()); exit(not torch.cuda.is_available())"
          pip list

          # Install dependencies
          pip install -r requirements-dev.txt
          pip install -e .

          EOF
          )

          docker exec -t pthd /bin/bash -c "${script}"

      - name: Install Horovod with NCCL GPU ops
        run: |
          script=$(cat << EOF

          set -xe

          # Can't build Horovod with recent pytorch due to pytorch required C++17 standard
          # and horovod is still using C++14
          # HOROVOD_GPU_OPERATIONS=NCCL HOROVOD_WITH_PYTORCH=1 pip install horovod[pytorch]
          # Using a similar hack as described here:
          # https://github.com/horovod/horovod/issues/3941#issuecomment-1732505345
          git clone --recursive https://github.com/horovod/horovod.git /horovod
          cd /horovod
          sed -i "s/CMAKE_CXX_STANDARD 14/CMAKE_CXX_STANDARD 17/g" CMakeLists.txt
          sed -i "s/CMAKE_CXX_STANDARD 14/CMAKE_CXX_STANDARD 17/g" horovod/torch/CMakeLists.txt
          HOROVOD_GPU_OPERATIONS=NCCL HOROVOD_WITH_PYTORCH=1 python setup.py install

          horovodrun --check-build
          pip list

          EOF
          )

          docker exec -t pthd /bin/bash -c "${script}"

      - name: Run GPU and CPU Unit HVD Tests
        run: |

          script=$(cat << EOF

          set -xe

          bash tests/run_gpu_tests.sh 2 hvd
          CUDA_VISIBLE_DEVICES="" pytest --cov ignite --cov-append --cov-report term-missing --cov-report xml -vvv tests/ignite -m distributed -k hvd

          EOF
          )

          docker exec -t pthd /bin/bash -c "${script}"

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v3
        with:
          file: ${{ github.repository }}/coverage.xml
          flags: gpu-2
          fail_ci_if_error: false

      - name: Run examples in container
        continue-on-error: false
        run: |
          SCRIPT=$(cat << EOF

          set -xe

          # Install additional example dependencies
          pip install fire

          # Check training on CIFAR10, run with horovod backend using horovodrun
          # initial run
          CI=1 horovodrun -np 2 python -u examples/cifar10/main.py run --backend=horovod --checkpoint_every=200 --stop_iteration=500
          # resume
          CI=1 horovodrun -np 2 python examples/cifar10/main.py run --checkpoint_every=200 --num_epochs=7 --resume-from=/tmp/output-cifar10/resnet18_backend-horovod-2_stop-on-500/training_checkpoint_400.pt

          # Check training on CIFAR10 using spawn
          # initial run
          CI=1 python -u examples/cifar10/main.py run --backend=horovod --nproc_per_node=2 --checkpoint_every=200 --stop_iteration=500
          # resume
          CI=1 python -u examples/cifar10/main.py run --backend=horovod --nproc_per_node=2 --checkpoint_every=200 --num_epochs=7 --resume-from=/tmp/output-cifar10/resnet18_backend-horovod-2_stop-on-500/training_checkpoint_400.pt

          EOF
          )

          docker exec -t pthd /bin/bash -c "${script}"

      - name: Teardown Linux
        if: ${{ always() }}
        uses: ./test-infra/.github/actions/teardown-linux


Discuss "help-wanted" issue on Discord

name: Discuss "help-wanted" issue on Discord

on:
  issues:
    types:
      - labeled
  workflow_dispatch:
    inputs:
      issue_number:
        description: 'Issue number'
        required: true

permissions:
  issues: write

jobs:
  discord:
    runs-on: ubuntu-latest
    steps:
      - name: "Discuss on Discord-Issues"
        if: ${{ github.event.label.name == 'help wanted' }}
        uses: EndBug/discuss-on-discord@v1.1.0
        with:
          discord_bot_token: ${{ secrets.DISCORD_BOT_TOKEN }}
          destination: ${{ secrets.DISCORD_BOT_DESTINATION }}
          issue_number: ${{ github.event.inputs.issue_number || github.event.issue.number }}
          issue_comment: Hey 👋, I've just created a [thread]($THREAD_LINK$) for this issue on [PyTorch-Ignite Discord](https://pytorch-ignite.ai/chat) where you can quickly talk to the community on the topic.
          discord_message: New issue created in `${{ github.repository }}`:<https://github.com/${{ github.repository }}/issues/${{ github.event.issue.number }}>


Run unit tests on M1

name: Run unit tests on M1
on:
  push:
    branches:
      - master
      - "*.*.*"
    paths:
      - "ignite/**"
      - "tests/ignite/**"
      - "tests/run_code_style.sh"
      - "examples/**.py"
      - "requirements-dev.txt"
      - ".github/workflows/mps-tests.yml"
  pull_request:
    paths:
      - "ignite/**"
      - "tests/ignite/**"
      - "tests/run_code_style.sh"
      - "examples/**.py"
      - "requirements-dev.txt"
      - ".github/workflows/mps-tests.yml"
  workflow_dispatch:

concurrency:
  # <workflow_name>-<branch_name>-<true || commit_sha (if branch is protected)>
  group: mps-tests-${{ github.ref_name }}-${{ !(github.ref_protected) || github.sha }}
  cancel-in-progress: true

# Cherry-picked from
# - https://github.com/pytorch/vision/blob/main/.github/workflows/tests.yml
# - https://github.com/pytorch/test-infra/blob/main/.github/workflows/macos_job.yml

jobs:
  mps-tests:
    strategy:
      matrix:
        python-version: [3.8]
        pytorch-channel: ["pytorch"]
        skip-distrib-tests: [1]
      fail-fast: false
    runs-on: ["macos-m1-stable"]
    timeout-minutes: 60

    steps:
      - name: Clean workspace
        run: |
          echo "::group::Cleanup debug output"
          sudo rm -rfv "${GITHUB_WORKSPACE}"
          mkdir -p "${GITHUB_WORKSPACE}"
          echo "::endgroup::"

      - name: Checkout repository (pytorch/test-infra)
        uses: actions/checkout@v3
        with:
          # Support the use case where we need to checkout someone's fork
          repository: pytorch/test-infra
          path: test-infra

      - name: Checkout repository (${{ github.repository }})
        uses: actions/checkout@v3
        with:
          # Support the use case where we need to checkout someone's fork
          repository: ${{ github.repository }}
          ref: ${{ github.ref }}
          path: ${{ github.repository }}
          fetch-depth: 1

      - name: Setup miniconda
        uses: ./test-infra/.github/actions/setup-miniconda
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install PyTorch
        if: ${{ matrix.pytorch-channel == 'pytorch' }}
        shell: bash -l {0}
        run: |
          conda shell.bash hook
          conda activate $CONDA_ENV
          pip install torch torchvision

      - name: Install PyTorch (nightly)
        if: ${{ matrix.pytorch-channel == 'pytorch-nightly' }}
        shell: bash -l {0}
        run: |
          conda shell.bash hook
          conda activate $CONDA_ENV
          pip install --pre torch torchvision --index-url https://download.pytorch.org/whl/nightly/cpu

      - name: Install dependencies
        shell: bash -l {0}
        working-directory: ${{ github.repository }}
        run: |
          conda activate $CONDA_ENV
          # TODO: We add set -xe to explicitly fail the CI if one of the commands is failing.
          # Somehow the step is passing even if a subcommand failed
          set -xe
          pip install -r requirements-dev.txt
          echo "1 returned code: $?"
          pip install -e .
          echo "2 returned code: $?"
          pip list
          echo "3 returned code: $?"

      # Download MNIST: https://github.com/pytorch/ignite/issues/1737
      # to "/tmp" for unit tests
      - name: Download MNIST
        uses: pytorch-ignite/download-mnist-github-action@master
        with:
          target_dir: /tmp

      # Copy MNIST to "." for the examples
      - name: Copy MNIST
        run: |
          cp -R /tmp/MNIST .

      - name: Run Tests
        shell: bash -l {0}
        working-directory: ${{ github.repository }}
        run: |
          conda activate $CONDA_ENV
          SKIP_DISTRIB_TESTS=${{ matrix.skip-distrib-tests }} bash tests/run_cpu_tests.sh

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v3
        with:
          file: ${{ github.repository }}/coverage.xml
          flags: mps
          fail_ci_if_error: false

      - name: Run MNIST Examples
        shell: bash -l {0}
        working-directory: ${{ github.repository }}
        run: |
          conda activate $CONDA_ENV
          python examples/mnist/mnist.py --epochs=1


Discuss "help-wanted" PR on Discord

name: Discuss "help-wanted" PR on Discord

on:
  pull_request:
    types:
      - labeled
  workflow_dispatch:
    inputs:
      pull_request_number:
        description: 'Pull request number'
        required: true

permissions:
  pull-requests: write

jobs:
  discord:
    runs-on: ubuntu-latest
    steps:
      - name: "Discuss on Discord-PR (Non-maintainer only)"
        if: ${{ github.event.label.name == 'help wanted' }}
        uses: EndBug/discuss-on-discord@v1.1.0
        with:
          discord_bot_token: ${{ secrets.DISCORD_BOT_TOKEN }}
          destination: ${{ secrets.DISCORD_BOT_DESTINATION }}
          issue_number: ${{ github.event.inputs.pull_request_number || github.event.pull_request.number }}
          issue_comment: Hey 👋, I've just created a [thread]($THREAD_LINK$) for this pull request on [PyTorch-Ignite Discord](https://pytorch-ignite.ai/chat) where you can quickly talk to the community on the topic.
          discord_message: New PR created in `${{ github.repository }}`:<https://github.com/${{ github.repository }}/pull/${{ github.event.pull_request_number || github.event.pull_request.number }}>


Format python code

name: Format python code

on:
  push:
    paths:
      - "**.py"
      - "setup.cfg"
      - "requirements-dev.txt"
      - "pyproject.toml"
      - "tests/run_code_style.sh"
      - ".github/workflows/code-style.yml"
      - "!assets/**"
      - "!docker/**"
      - "!docs/**"
      - "!conda.recipe"

jobs:
  code-style:
    runs-on: ubuntu-latest
    steps:
      - if: github.event_name == 'push'
        uses: actions/checkout@v4
      - uses: actions/setup-python@v4
        with:
          python-version: "3.8"
      - run: |
          bash ./tests/run_code_style.sh install
          bash ./tests/run_code_style.sh fmt

      - name: Commit and push changes
        uses: stefanzweifel/git-auto-commit-action@v4
        with:
          commit_message: "autopep8 fix"
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


Backport

name: Backport
on:
  pull_request_target:
    types:
      - closed
      - labeled

jobs:
  backport:
    name: Backport
    runs-on: ubuntu-latest
    # Only react to merged PRs for security reasons.
    # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target.
    if: >
      github.event.pull_request.merged
      && (
        github.event.action == 'closed'
        || (
          github.event.action == 'labeled'
          && contains(github.event.label.name, 'backport')
        )
      )
    steps:
      - uses: tibdex/backport@9565281eda0731b1d20c4025c43339fb0a23812e # v2.0.4
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}


CI

---
name: CI

on: [push, pull_request]

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v3
      - name: Set up Python 3.x
        uses: actions/setup-python@v4
        with:
          python-version: "3.11"
      - name: Install dependencies
        run: |
          python3 -m pip install nox
      - name: Lint the code
        run: nox -s lint

  test-linux:
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.7", "3.8", "3.9", "3.10", "3.11", "3.12"]
        nox-session: [""]
        runs-on: ["ubuntu-latest"]

    runs-on: ${{ matrix.runs-on }}
    name: test-${{ matrix.python-version }}
    continue-on-error: false
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v3
      - name: Set Up Python - ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install Dependencies
        run: |
          python -m pip install nox
      - name: Run Tests
        shell: bash
        run: .buildkite/run-nox.sh
        env:
          PYTHON_VERSION: ${{ matrix.python-version }}
          NOX_SESSION: ${{ matrix.nox-session }}


docs-preview

---
name: docs-preview

on:
  pull_request_target:
    types: [opened]

permissions:
  pull-requests: write

jobs:
  doc-preview-pr:
    runs-on: ubuntu-latest
    steps:
      - uses: elastic/docs/.github/actions/docs-preview@master
        with:
          github-token: ${{ secrets.GITHUB_TOKEN }}
          repo: ${{ github.event.repository.name }}
          preview-path: 'guide/en/elasticsearch/client/python-api/index.html'
          pr: ${{ github.event.pull_request.number }}


issues

name: issues
on:
  schedule:
    - cron: '30 1 * * *'

jobs:
  stale:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/stale@v9
        with:
          stale-issue-label: stale
          stale-issue-message: 'This issue has been automatically marked as stale because it has not had recent activity. It will be closed if no further activity occurs. Thank you for your contributions.'
          days-before-stale: 120
          days-before-close: 14
          days-before-pr-stale: -1
          days-before-pr-close: -1


tests

name: tests

on: [push, pull_request]

jobs:

  docs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: 3.8
      - name: Install OS packages
        run: |
          sudo apt-get update
          sudo apt-get install libopus-dev libvpx-dev
      - name: Build documentation
        run: |
          pip install . -r requirements/doc.txt
          make -C docs html SPHINXOPTS=-W

  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: 3.8
      - name: Install packages
        run: pip install check-manifest mypy ruff
      - name: Run linters
        run: |
          ruff check .
          ruff format --check --diff .
          check-manifest

  test:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
        python:
          - '3.12'
          - '3.11'
          - '3.10'
          - '3.9'
          - '3.8'
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
      - name: Install OS packages and disable firewall
        if: matrix.os == 'macos-latest'
        run: |
          brew update
          brew install opus libvpx
          sudo /usr/libexec/ApplicationFirewall/socketfilterfw --setglobalstate off
      - name: Install OS packages
        if: matrix.os == 'ubuntu-latest'
        run: |
          sudo apt-get update
          sudo apt-get install libopus-dev libvpx-dev
      - name: Run tests
        run: |
          python -m pip install -U pip setuptools wheel
          pip install .[dev]
          coverage run -m unittest discover -v
          coverage xml
        shell: bash
      - name: Upload coverage report
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}

  package-source:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: 3.8
      - name: Build source package
        run: |
          pip install -U build
          python -m build --sdist
      - name: Upload source package
        uses: actions/upload-artifact@v4
        with:
          name: dist-source
          path: dist/

  package-wheel:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: macos-14
            arch: arm64
          - os: macos-13
            arch: x86_64
          - os: ubuntu-latest
            arch: aarch64
          - os: ubuntu-latest
            arch: i686
          - os: ubuntu-latest
            arch: x86_64
          - os: windows-latest
            arch: AMD64
          - os: windows-latest
            arch: x86
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: 3.8
      - name: Install QEMU
        if: matrix.os == 'ubuntu-latest'
        uses: docker/setup-qemu-action@v3
      - name: Build wheels
        env:
          CIBW_ARCHS: ${{ matrix.arch }}
          CIBW_BEFORE_BUILD: python scripts/fetch-vendor.py /tmp/vendor
          CIBW_BEFORE_BUILD_WINDOWS: python scripts\fetch-vendor.py C:\cibw\vendor
          CIBW_ENVIRONMENT: CFLAGS=-I/tmp/vendor/include LDFLAGS=-L/tmp/vendor/lib
          CIBW_ENVIRONMENT_WINDOWS: INCLUDE=C:\\cibw\\vendor\\include LIB=C:\\cibw\\vendor\\lib
          CIBW_SKIP: '*-musllinux*'
          CIBW_TEST_COMMAND: python -c "import aiortc"
          # There are no binary wheels for cryptography on these platforms.
          CIBW_TEST_SKIP: "*-{manylinux_i686,win32} pp*"
        run: |
          pip install cibuildwheel
          cibuildwheel --output-dir dist
        shell: bash
      - name: Upload wheels
        uses: actions/upload-artifact@v4
        with:
          name: dist-wheel-${{ matrix.os }}-${{ matrix.arch }}
          path: dist/

  publish:
    runs-on: ubuntu-latest
    needs: [lint, test, package-source, package-wheel]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          merge-multiple: true
          path: dist/
      - name: Publish to PyPI
        if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags/')
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          user: __token__
          password: ${{ secrets.PYPI_TOKEN }}


build

---
name: build
on: [push, pull_request]
permissions:
  contents: read

jobs:
  std_tests:
    runs-on: ubuntu-latest
    strategy:
      max-parallel: 4
      matrix:
        python-version: [3.8, 3.9, 3.10.9, 3.11, 3.12.0]

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Setup Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install -e .
          pip install -r requirements.txt
          pip install -r requirements-dev.txt

      - name: Run black
        run: |
          black --check .

      - name: Run linter
        run: |
          pylama .

      - name: Run type checker
        run: |
          mypy -p napalm --config-file mypy.ini

      - name: Run Tests
        run: |
          py.test --cov=napalm --cov-report term-missing -vs --pylama

  build_docs:
    needs: std_tests
    runs-on: ubuntu-latest

    strategy:
      matrix:
        python-version: [3.11]

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4

      - name: Setup Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install -e .
          python -m pip install -r docs/requirements.txt
          pip install -r requirements-dev.txt
          pip install -r requirements.txt

      - name: Doctests
        run: |
          make doctest


Publish Python Package

name: Publish Python Package
on:
  push:
    # Sequence of patterns matched against refs/tags
    tags:
      - '*'

permissions:
  contents: read

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.x'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install setuptools wheel
    - name: Build
      run: |
        python setup.py sdist bdist_wheel
    - name: Publish
      uses: pypa/gh-action-pypi-publish@master
      with:
        user: __token__
        password: ${{ secrets.PYPI_TOKEN }}