Push Release

name: Push Release

on:
  push:
    branches:
      - release/v[0-9]+.[0-9]+.[0-9]+
  workflow_dispatch:
    inputs:
      ref_name:
        type: string

jobs:
  pr:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          ref: develop
          fetch-depth: 0
      - run: |
          git config user.name 'voxel51-bot'
          git config user.email 'bot@voxel51.com'
          git checkout merge/${{ inputs.ref_name || github.ref_name }} \
          || git checkout -b merge/${{ inputs.ref_name || github.ref_name }}
          git push -u origin merge/${{ inputs.ref_name || github.ref_name }}
          git checkout develop
          git pull origin merge/${{ inputs.ref_name || github.ref_name }} --no-rebase
          git pull origin ${{ inputs.ref_name || github.ref_name }} --no-rebase
      - uses: peter-evans/create-pull-request@v6
        with:
          author: voxel51-bot <bot@voxel51.com>
          token: ${{ secrets.FIFTYONE_GITHUB_TOKEN }}
          base: develop
          body: Merge `${{ inputs.ref_name || github.ref_name }}` to `develop`
          branch: merge/${{ inputs.ref_name || github.ref_name }}
          title: Merge `${{ inputs.ref_name || github.ref_name }}` to `develop`


Test

name: Test

on: workflow_call

jobs:
  test-app:
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Cache
        id: node-cache
        uses: actions/cache@v3
        with:
          path: |
            app/node_modules
            app/.yarn/cache
          key: node-modules-${{ hashFiles('app/yarn.lock') }}
      - name: Install
        if: steps.node-cache.outputs.cache-hit != 'true'
        run: cd app && yarn install
      - name: Run
        run: cd app && yarn test --coverage && mv ./coverage/coverage-final.json ../
      - name: Upload
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./coverage-final.json
          flags: app

  test-python:
    runs-on: ${{ matrix.os }}
    env:
      FIFTYONE_DO_NOT_TRACK: true
    strategy:
      fail-fast: false
      matrix:
        os:
          - ubuntu-latest-m
          - windows-latest
        python:
          - "3.8"
          - "3.10"
          - "3.11"
        exclude:
          - os: windows-latest
            python: "3.8"
          - os: windows-latest
            python: "3.10"
          - os: windows-latest
            python: "3.11"
    defaults:
      run:
        shell: bash
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup
        uses: actions/setup-python@v5
        id: pip-cache
        with:
          python-version: ${{ matrix.python }}
          cache: "pip"
          cache-dependency-path: |
            requirements/common.txt
            requirements/github.txt
            requirements/test.txt
      - name: Install requirements
        if: steps.pip-cache.outputs.cache-hit != true
        run: |
          pip install -r requirements/github.txt
      - name: Install fiftyone
        run: |
          pip install .
      - name: Configure
        id: test_config
        run: |
          python tests/utils/setup_config.py
          python tests/utils/github_actions_flags.py
      - name: FFmpeg
        uses: FedericoCarboni/setup-ffmpeg@v2
      # Important: use pytest_wrapper.py instead of pytest directly to ensure
      # that services shut down cleanly and do not conflict with steps later in
      # this workflow
      - name: Run tests
        run: |
          python tests/utils/pytest_wrapper.py tests/ --verbose \
            --cov --cov-report xml \
            --ignore tests/benchmarking/ \
            --ignore tests/isolated/ \
            --ignore tests/utils/ \
            --ignore tests/intensive/ \
            --ignore tests/no_wrapper
      - name: Upload
        if: ${{ !startsWith(matrix.os, 'windows') && matrix.python == '3.11' }}
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./coverage.xml
          flags: python
      - name: Run no wrapper tests
        run: |
          pytest tests/no_wrapper --verbose
      # Intended to run even if the tests above failed
      - name: Run isolated tests
        if: success() || failure()
        run: |
          find tests/isolated/ -name '*.py' -print0 | xargs -0 --verbose -n1 python tests/utils/pytest_wrapper.py --verbose


release

---
# --------------------( LICENSE                            )--------------------
# Copyright (c) 2014-2024 Beartype authors.
# See "LICENSE" for further details.
#
# --------------------( SYNOPSIS                           )--------------------
# GitHub-specific continuous deployment (CD) configuration, enabling automated
# publication of both source tarballs and binary wheels in various popular
# formats to both GitHub itself and PyPI on each push of a tag to the "master"
# branch of this repository.
#
# --------------------( SEE ALSO                           )--------------------
# * https://blog.chezo.uno/how-to-release-python-package-from-github-actions-d5a1d8edba6e
#   Well-authored blog post strongly inspiring this configuration.
# * https://packaging.python.org/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows
#   Official PyPA workflow also inspiring this configuration.

# ....................{ TODO                               }....................
#FIXME: This workflow, while technically working, now emits this warning:
#    Trusted Publishers allows publishing packages to PyPI from automated
#    environments like GitHub Actions without needing to use username/password
#    combinations or API tokens to authenticate with PyPI. Read more:
#    https://docs.pypi.org/trusted-publishers
#
#Seems quite reasonable. Let's dig a bit deeper into this when time permits.

# ....................{ METADATA                           }....................
# Non-human-readable (i.e., machine-readable) label associated with this
# GitHub Actions workflow.
name: release

# ....................{ TRIGGER                            }....................
# Confine deployment to only new tags satisfying a release-specific format.
on:
  push:
    # Sequence of glob expressions matched against "refs/tags" pushed to the
    # branches above.
    tags:
      - 'v*'   # Match "v"-prefixed tags (e.g., "v6.9.6").

# ....................{ MAIN                               }....................
jobs:
  # ...................{ GITHUB                             }...................
  # Job publishing a human-readable changelog and codebase tarballs to GitHub
  # for this release.
  release:
    name: "Create tagged release on GitHub"
    runs-on: ubuntu-latest

    steps:
      - name: "Checking out repository..."
        uses: 'actions/checkout@v4'
      - name: "Publishing GitHub release..."
        uses: 'ncipollo/release-action@v1'
        with:
          name: "Beartype ${{ github.ref }}"
          body: ${{ github.event.commits[0].message }}
          token: ${{ secrets.GITHUB_TOKEN }}

  # ...................{ PYPI                               }...................
  # Job publishing both a static distribution (sdist) and binary wheel to PyPI
  # for this release.
  pypi:
    name: "Publish tagged release to PyPI"
    runs-on: ubuntu-latest

    # GitHub Environment associated with this job.
    #
    # Note that PyPI strongly recommends use of a GitHub Environment as an
    # additional security precaution for GitHub repositories that grant push
    # access to different users with differing permissions. So, basically *ALL*
    # GitHub repositories is what we are saying. See also:
    # * "Publishing to PyPI with a Trusted Publisher."
    #   https://docs.pypi.org/trusted-publishers
    # * "Using environments for deployment."
    #   https://docs.github.com/en/actions/deployment/targeting-different-environments/using-environments-for-deployment#creating-an-environment
    environment:
      # Arbitrary name of this environment.
      name: bear_den

      # Arbitrary URL associated with this environment.
      url: https://pypi.org/project/beartype

    # Enable the "write" permission as required for PyPI Trusted Publishing.
    # See also:
    #     https://github.com/pypa/gh-action-pypi-publish
    permissions:
      id-token: write  # IMPORTANT: this permission is mandatory for trusted publishing

    # Perform this job *ONLY* if the prior job succeeded.
    needs: release

    steps:
      - name: "Checking out repository..."
        uses: 'actions/checkout@v4'
      - name: "Installing latest stable Python 3.x..."
        uses: 'actions/setup-python@v5'
        with:
          python-version: '3.x'

      # See "python_test.yml" for further details.
      - name: 'Upgrading packager dependencies...'
        run: |
          set -xe
          python -VV
          python -m site
          python -m pip --quiet install --upgrade pip hatch wheel

      #FIXME: Pretty sure that "twine" is no longer required. *shrug*
      # - name: 'Installing package dependencies...'
      #   run: |
      #     python -m pip --quiet install --upgrade twine

      - name: "Creating source tarball and binary wheel..."
        run: |
          set -xe
          hatch build -t sdist -t wheel
      - name: "Publishing PyPI release from tag..."
        uses: 'pypa/gh-action-pypi-publish@release/v1'
        #FIXME: This should no longer be required. See also:
        #    https://github.com/pypa/gh-action-pypi-publish
        # with:
        #   user: __token__
        #   password: ${{ secrets.PYPI_API_TOKEN }}

      #FIXME: Enable after integrating this "pypi" job into the "release" job
      #above. In theory, this should be trivial. Let's test this later, eh?
      # - name: "Publishing GitHub release assets from tag..."
      #   uses: actions/upload-release-asset@v1.0.1
      #   env:
      #     GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      #   with:
      #     upload_url: ${{ steps.create_release.outputs.upload_url }} # This pulls from the CREATE RELEASE step above, referencing it's ID to get its outputs object, which include a `upload_url`. See this blog post for more info: https://jasonet.co/posts/new-features-of-github-actions/#passing-data-to-future-steps
      #     asset_path: "dist/beartype-${{ github.ref }}.zip"
      #     asset_name: "beartype-${{ github.ref }}.zip"
      #     asset_content_type: application/zip


test

---
# --------------------( LICENSE                            )--------------------
# Copyright (c) 2014-2024 Beartype authors.
# See "LICENSE" for further details.
#
# --------------------( SYNOPSIS                           )--------------------
# GitHub-specific continuous integration (CI) configuration, enabling the usual
# GitHub Actions workflow for pure-Python packages exercised by "tox".
#
# --------------------( SEE ALSO                           )--------------------
# * https://hynek.me/articles/python-github-actions
#   Well-authored blog post strongly inspiring this configuration.

# ....................{ TODO                               }....................
#FIXME: [CACHING] Add support for caching "pip" downloads across runs.
#Currently, unresolved issues in GitHub Actions prevents sane caching of "pip"
#downloads. Naturally, horrifying hacks circumventing these issues do exist but
#are presumably worse than these issues. See also this pertinent comment:
#    https://github.com/actions/cache/issues/342#issuecomment-673371329
#FIXME: [CACHING] Huzzah! The "setup-python" action itself now supports
#out-of-the-box caching. Note that doing so will require a minor of
#configuration on our part -- but nothing *TOO* tremendous, hopefully. See:
#    https://github.com/actions/setup-python#caching-packages-dependencies

# ....................{ METADATA                           }....................
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# WARNING: Changes to this name *MUST* be manually synchronized with:
# * The "|GitHub Actions badge|" image URL in our top-level "README.rst".
#!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
# Non-human-readable (i.e., machine-readable) label associated with this
# GitHub Actions workflow.
name: test

# ....................{ TRIGGER                            }....................
# Confine testing to only...
#
# Note that "**" matches all (possibly deeply "/"-nested) branches. See also:
# * https://docs.github.com/en/actions/reference/workflow-syntax-for-github-actions#filter-pattern-cheat-sheet
#   GitHub-specific glob syntax for matching branches and tags below.
on:
  # Pushes to the main branch. Pushes to other branches are assumed to be
  # developer-specific and thus already tested locally by that developer.
  push:
    branches:
      - main
  # Pull requests against the main branch. Pull requests against other branches
  # should, ideally, *NEVER* occur; if and when they do, we ignore them.
  pull_request:
    branches:
      - main  # '**'

# ....................{ VARIABLES                          }....................
# List of private environment variables specific to this configuration and
# globally set for *ALL* jobs declared below. To avoid conflict with
# third-party processes, prefix the name of each variable by "_".
env:
  #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
  # CAUTION: Optional in-flight test-time dependencies (e.g., NumPy, mypy) are
  # intentionally listed in the "beartype.meta.LIBS_TESTTIME_OPTIONAL" global
  # rather than below. "tox" isolates both the package being tested and its
  # dependency tree to virtual environments. Listing in-flight dependencies
  # here would install those dependencies outside those virtual environments,
  # thus reducing to a pointless, expensive, and failure-prone noop.
  #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
  # Whitespace-delimited list of the names of all Python packages to be
  # installed by "pip" below. Although this package has *NO* mandatory runtime
  # dependencies, fully exercising all tests necessitates these pre-flight
  # test-time dependencies. These include:
  # * "tox", the only mandatory test-time dependency.
  _PIP_PACKAGE_NAMES: |
    tox

# ....................{ MAIN                               }....................
jobs:
  # ...................{ TESTS                              }...................
  # Job iteratively exercising our test suite against all Python interpreters
  # supported by this package (and also measuring the coverage of that suite).
  tests:
    # ..................{ MATRIX                             }..................
    strategy:
      matrix:
        # List of all platform-specific Docker images to test against,
        # including:
        # * The latest Long-Term Service (LTS) release of Ubuntu Linux, still
        #   the most popular Linux distro and thus a sane baseline.
        # * The latest *whatever* release of Microsoft Windows. Although Linux
        #   and macOS are both POSIX-compliant and thus crudely comparable from
        #   the low-level CLI perspective, Windows is POSIX-noncompliant and
        #   thus heavily divergent from both macOS and Linux.
        # * The latest *whatever* release of Apple macOS. We don't particularly
        #   need to exercise tests on macOS, given the platform's patent
        #   POSIX-compliant low-level similarities to Linux, but... what the
        #   heck. Why not? Since this is the lowest priority, we defer macOS
        #   testing until last.

        #FIXME: Re-enable "macos-latest" support *AFTER* the "setup-python"
        #GitHub Action resolves this currently outstanding issue:
        #    https://github.com/actions/setup-python/issues/852
        # platform: [ubuntu-latest, windows-latest, macos-latest]
        platform: [ubuntu-latest, windows-latest]

        #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!
        # WARNING: Changes to this section *MUST* be manually synchronized with:
        # * The "envlist" setting of the "[tox]" subsection in "tox.ini".
        # * The "include" setting below.
        #!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!

        # List of all "tox" environments (defined by the "envlist" setting of
        # the "[tox]" subsection in "tox.ini") to be tested, which the
        # ${TOXENV} environment variable declared below exposes to "tox".
        tox-env:
          - py38-coverage
          - py39-coverage
          - py310-coverage
          - py311-coverage
          - py312-coverage
          #FIXME: Uncomment after we resolve tests currently broken under *ANY*
          #PyPy version. All tests used to pass under PyPy 3.7 and 3.8, but were
          #recently broken by work generalizing @beartype to decorate builtin
          #method descriptors (e.g., @property, @classmethod, @staticmethod).
          # - pypy37-coverage
          # - pypy38-coverage
          # - pypy39-coverage

        # Map each "tox" environment name listed in the "tox-env" list above to
        # the corresponding "python-version" string supported by the
        # "actions/setup-python" GitHub Action run below.
        #
        # Note that:
        # * Python version specifiers *MUST* be quoted: e.g.,
        #     # Do this.
        #     python-version: "3.10"
        #     # Do *NOT* do this.
        #     python-version: 3.10
        #   Why? Because YAML sensibly treats an unquoted literal satisfying
        #   floating-point syntax as a floating-point number and thus silently
        #   truncates *ALL* ignorable zeroes suffixing that number (e.g.,
        #   truncating 3.10 to 3.1). That then results in non-human-readable CI
        #   errors, as discussed upstream at:
        #     https://github.com/actions/setup-python/issues/160#issuecomment-724485470
        # * Python pre-releases may be selected with a space-delimited range
        #   embedded in a single quoted version specifier. For example,
        #   selecting the Python 3.11 pre-release reduces to:
        #     python-version: "3.11.0-alpha - 3.11.0"
        include:
          - tox-env: py38-coverage
            python-version: "3.8"
          - tox-env: py39-coverage
            python-version: "3.9"
          - tox-env: py310-coverage
            python-version: "3.10"
          - tox-env: py311-coverage
            python-version: "3.11"
          - tox-env: py312-coverage
            python-version: "3.12"
          #- tox-env: pypy37-coverage
          #  python-version: "pypy-3.7"
          #- tox-env: pypy38-coverage
          #  python-version: "pypy-3.8"
          #FIXME: Uncomment after PyPy 3.9 goes live.
          # - tox-env: pypy39-coverage
          #   python-version: "pypy-3.9"

    # ..................{ SETTINGS                           }..................
    # Arbitrary human-readable description.
    name: "[${{ matrix.platform }}] Python ${{ matrix.python-version }} CI"

    # Name of the current Docker image to run tests under.
    runs-on: "${{ matrix.platform }}"

    # Time in minutes to wait on the command pipeline run below to exit
    # *BEFORE* sending a non-graceful termination request (i.e., "SIGTERM"
    # under POSIX-compliant systems).
    timeout-minutes: 10

    # ..................{ VARIABLES                          }..................
    # External shell environment variables exposed to commands run below.
    env:
      # Prevent "pip" from wasting precious continuous integration (CI) minutes
      # deciding whether it should be upgrading. We're *NOT* upgrading you,
      # "pip". Accept this and let us test faster.
      PIP_NO_PIP_VERSION_CHECK: 1

      # Map from the current item of the "tox-env" list defined above to the
      # ${TOXENV} environment variable recognized by "tox".
      TOXENV: "${{ matrix.tox-env }}"

    # ..................{ PROCESS                            }..................
    steps:
      # ..................{ SETUP                            }..................
      - name: 'Checking out repository...'
        uses: 'actions/checkout@v4'
      - name: "Installing Python ${{ matrix.python-version }}..."
        uses: 'actions/setup-python@v5'
        with:
          python-version: "${{ matrix.python-version }}"
      - name: 'Displaying Python metadata...'
        run: |
          python3 -VV
          python3 -m site

      # ..................{ INSTALL                          }..................
      # Note that:
      # * This command *MUST* be platform-agnostic by running under both:
      #   * POSIX-compliant platforms (e.g., Linux, macOS).
      #   * POSIX-noncompliant platforms (e.g., Windows).
      #   In particular, commands that assume a POSIX-compliant shell (e.g.,
      #   Bash) *MUST* be avoided.
      # * Packaging dependencies (e.g., "pip") are upgraded *BEFORE* all
      #   remaining dependencies (e.g., "tox").
      - name: 'Upgrading packager dependencies...'
        run: |
          python3 -m pip --quiet install --upgrade pip setuptools wheel
      - name: 'Installing package dependencies...'
        run: |
          python3 -m pip --quiet install --upgrade ${{ env._PIP_PACKAGE_NAMES }}

      # ..................{ TYPE-CHECK                       }..................
      # Type-check this package *BEFORE* testing this package. Why? Because
      # "tox" internally creates a copy of this package residing at a temporary
      # "build/lib/{package_name}" subdirectory. When type-checking this package
      # *AFTER* testing this package, the existence of that subdirectory
      # confuses mypy into believing that multiple duplicate copies of this
      # package exist, which induces mypy to emit false positives, which induces
      # this entire job and thus run to fail. In short, mypy.

      # Type-check this package with "pyright". See also:
      #     https://github.com/jakebailey/pyright-action
      - name: 'Type-checking package with "pyright"...'
        uses: 'jakebailey/pyright-action@v2'
        with:
          python-version: "${{ matrix.python-version }}"

      #FIXME: Revisit @jpetrucciani's otherwise stellar GitHub Action
      #"jpetrucciani/mypy-check" once the issue tracker there settles a bit.
      #See also these open issue currently blocking our usage here:
      #    https://github.com/jpetrucciani/mypy-check/issues/31
      #    https://github.com/jpetrucciani/mypy-check/issues/30
      # Type-check this package with "mypy".
      - name: 'Type-checking package with "mypy:"...'
        run: |
          # Manually install "mypy" in the standard way.
          python3 -m pip --quiet install mypy
          # Log this "mypy" version for debuggability.
          mypy --version
          # Run this "mypy" instance against our main package.
          mypy ./beartype/

      # ..................{ TEST                             }..................
      - name: 'Testing package with "tox"...'
        # Run the subsequent script as a Bash script. Although POSIX-compliant
        # platforms (e.g., Linux, macOS) sensibly default to Bash, Windows
        # insanely defaults to a Windows-specific shell (e.g., PowerShell).
        shell: bash
        run: |
          # If the current platform is macOS, export a "tox"-specific
          # environment variable circumventing "pip" installation issues by
          # instructing "tox" to reinstall already installed Python packages.
          # By default, "tox" avoids doing so for efficiency. This is required
          # to specifically circumvent installation of NumPy under macOS. As
          # discussed at numpy/numpy#15947, macOS bundles a patently broken
          # BLAS replacement called "Accelerate" causing NumPy to raise
          # exceptions on importation resembling:
          #     RuntimeError: Polyfit sanity test emitted a warning, most
          #     likely due to using a buggy Accelerate backend. If you compiled
          #     yourself, more information is available at
          #     https://numpy.org/doc/stable/user/building.html#accelerated-blas-lapack-libraries
          #     Otherwise report this to the vendor that provided NumPy.
          #     RankWarning: Polyfit may be poorly conditioned
          #
          # The kludge leveraged here is the canonical solution. See also:
          #     https://github.com/numpy/numpy/issues/15947#issuecomment-745428684
          #
          # Ideally, we would instead isolate setting this environment variable
          # in a prior step with sane GitHub Actions syntax: e.g.,
          #     if: ${{ matrix.platform }} == 'macos-latest'
          #     env:
          #       _TOX_PIP_INSTALL_OPTIONS: '--force-reinstall'
          #
          # Sadly, the "env:" map only locally exports the environment
          # variables it declares to the current step. Thanks, GitHub Actions.
          if [[ ${{ matrix.platform }} == 'macos-latest' ]]; then
              export _TOX_PIP_INSTALL_OPTIONS='--force-reinstall'
              echo "Massaging macOS dependencies with \"pip install ${_TOX_PIP_INSTALL_OPTIONS}\"..."
          fi
          # Dismantled, this is:
          # * "--skip-missing-interpreters=false" disables the corresponding
          #   "skip_missing_interpreters = true" setting globally enabled by
          #   our top-level "tox.ini" configuration, forcing CI failures for
          #   unavailable Python environments. See also:
          #       https://github.com/tox-dev/tox/issues/903
          python3 -m tox --skip-missing-interpreters=false

      # ..................{ COVERAGE                         }..................
      - name: 'Publishing test coverage to Codecov...'
        uses: 'codecov/codecov-action@v4'
        with:
          name: "${{ matrix.platform }}-${{ matrix.python-version }}"


Wheel Builder

name: Wheel Builder

on:
  push:
    tags:
      - '*'

jobs:
  build_wheels:
    name: Push new release to PyPI
    runs-on: ubuntu-latest
    environment:
      name: pypi
      url: https://pypi.org/p/mpf
    permissions:
      id-token: write

    steps:
      - name: Checkout MPF
        uses: actions/checkout@v3

      - name: Setup python
        uses: actions/setup-python@v4
        with:
          python-version: 3.11

      - name: Install MPF
        run: |
          pip install --upgrade pip setuptools wheel build
          pip install -e .

      - name: Run tests
        run: python -m unittest discover -s mpf/tests

      - name: Build wheel
        run: python -m build

      - uses: actions/upload-artifact@v3
        with:
          path: ./dist/*.*

      - name: Publish to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1


Lint all Python files

name: Lint all Python files

on: [push, pull_request]

jobs:
  prospector:
    name: Run Prospector on all Python files
    runs-on: ubuntu-latest

    steps:
      - name: Checkout repository
        uses: actions/checkout@v2

      - name: Setup python
        uses: actions/setup-python@v4
        with:
          python-version: 3.11

      - name: Install MPF
        run: |
          pip install --upgrade pip setuptools wheel build prospector
          pip install -e .

      - name: Run Prospector
        run: prospector


Test on all platforms

name: Test on all platforms

on: [push, pull_request]

jobs:
  tests:
    name: Test Python ${{ matrix.python-version }} on ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        include:
        - os: windows-latest
          python-version: 3.8
        - os: windows-latest
          python-version: 3.9
        - os: windows-latest
          python-version: "3.10"
        - os: windows-latest
          python-version: 3.11
        - os: ubuntu-latest
          python-version: 3.8
        - os: ubuntu-latest
          python-version: 3.9
        - os: ubuntu-latest
          python-version: "3.10"
        - os: ubuntu-latest
          python-version: 3.11
        - os: macos-latest
          python-version: 3.8
        - os: macos-latest
          python-version: 3.9
        - os: macos-latest
          python-version: "3.10"
        - os: macos-latest
          python-version: 3.11

    steps:
      - name: Checkout MPF
        uses: actions/checkout@v3

      - name: Setup python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install MPF
        run: |
          pip install --upgrade pip setuptools wheel build coveralls
          pip install -e .

      - name: Run tests
      # Oct 15 2023, "coverage" randomly is not recognized as valid on windows
      # even though you can see it being installed via pip (from coveralls)?!?
        shell: bash  # Need bash on windows to use the if statement
        run: |
          if [ "${{ runner.os }}" == "Windows" ]; then
            python3 -m unittest discover -s mpf/tests
          else
            python3 -m coverage run -m unittest discover -s mpf/tests
          fi

      - name: Upload coverage data to coveralls.io
        if: matrix.os != 'windows-latest'
        run: coveralls --service=github
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          COVERALLS_FLAG_NAME: ${{ matrix.python-version }}-${{ matrix.os }}
          COVERALLS_PARALLEL: true

  coveralls:
    name: Indicate completion to coveralls.io
    needs: tests
    runs-on: ubuntu-latest
    container: python:3-slim
    steps:
    - name: Finished
      run: |
        pip3 install --upgrade coveralls
        coveralls --service=github --finish
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


ci

name: ci

on: [push, pull_request]

jobs:
  build:
    runs-on: ubuntu-20.04
    steps:
      # Checkout Repository
      - name: Checkout
        uses: actions/checkout@v4

      # Install Tools
      - name: Install Tools
        run: |
          sudo apt-get install wget build-essential

      - name: Set up Python 3.9
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"
          cache: "pip"
          cache-dependency-path: "setup.py"

      - name: Install Python dependencies
        run: |
          python3 -m pip install setuptools requests pexpect meson

      # Install (n)Migen / LiteX / Cores
      - name: Install LiteX
        run: |
          wget https://raw.githubusercontent.com/enjoy-digital/litex/master/litex_setup.py
          python3 litex_setup.py init install --user

      # Install Project
      - name: Install Project
        run: python3 setup.py develop --user

      # Test
      - name: Run Tests
        run: python3 setup.py test


Style formatting

name: Style formatting

on: [push, pull_request]

jobs:
  mypy:
    name: Style check with Black and iSort
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v1
      - uses: actions/setup-python@v1
        with:
          python-version: "3.8"
      - run: "python -m pip install black==22.3.0 isort==5.10.1"
        name: Install dependencies
      - run: "make stylediff"
        name: Run Style Checks


Flake8 Check

name: Flake8 Check

on:
  pull_request:
    branches: [ main ]

jobs:
  build:

    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python 3.11
      uses: actions/setup-python@v5
      with:
        python-version: '3.11'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install flake8
    - name: Check with flake8
      run: |
        flake8 .


Docker image build

name: 'Docker image build'

on:
  push:
    branches: main
  workflow_dispatch:

# cancel existing builds if a push is made
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  build:
    name: 'Image build'
    if: "!contains(github.event.head_commit.message, 'skip ci')"
    runs-on: ubuntu-latest
    steps:
      - name: Send job start webhook
        continue-on-error: true
        uses: appleboy/discord-action@3c62873803ab2f276e28cdde23972899fb26e3a5
        with:
          webhook_id: ${{ secrets.WEBHOOK_ID }}
          webhook_token: ${{ secrets.WEBHOOK_TOKEN }}
          message: |
            Starting build for commit [${{ github.sha }}](https://github.com/${{ github.repository }}/commit/${{ github.sha }}) by ${{ github.actor }}
            >>> ${{ github.event.head_commit.message }}
      - name: Checkout
        uses: actions/checkout@v4
      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
      - name: Install testing dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install flake8
      - name: Check with flake8
        run: |
          flake8 .
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Cache Docker layers
        uses: actions/cache@v3
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-buildx-
      - name: Login to GHCR
        uses: docker/login-action@v3
        with:
          # GitHub username and personal access token
          registry: ghcr.io
          username: ${{ secrets.CR_USERNAME }}
          password: ${{ secrets.CR_PAT }}
      - name: Build Docker image
        id: docker_build
        uses: docker/build-push-action@v3
        with:
          context: .
          file: ./Dockerfile
          platforms: linux/amd64
          push: true
          tags: ghcr.io/nh-server/kurisu:latest
          build-args: |
            COMMIT=${{ github.sha }}
            BRANCH=${{ github.ref }}
          cache-from: type=local,src=/tmp/.buildx-cache
          cache-to: type=local,dest=/tmp/.buildx-cache
      - name: Image digest
        run: echo ${{ steps.docker_build.outputs.digest }}
# https://stackoverflow.com/questions/60477061/github-actions-how-to-deploy-to-remote-server-using-ssh/60479844#60479844
      - name: Create SSH key
        run: |
          mkdir -p ~/.ssh/
          chmod 700 ~/.ssh/
          echo "$SSH_PRIVATE_KEY" > ~/.ssh/id_rsa
          chmod 600 ~/.ssh/id_rsa
          echo "$SSH_KNOWN_HOSTS" > ~/.ssh/known_hosts
        shell: bash
        env:
          SSH_PRIVATE_KEY: ${{ secrets.SSH_PRIVATE_KEY }}
          SSH_KNOWN_HOSTS: ${{ secrets.SSH_KNOWN_HOSTS }}
      - name: Send push webhook
        continue-on-error: true
        uses: appleboy/discord-action@3c62873803ab2f276e28cdde23972899fb26e3a5
        with:
          webhook_id: ${{ secrets.WEBHOOK_ID }}
          webhook_token: ${{ secrets.WEBHOOK_TOKEN }}
          message: Pulling on server
      - name: Pull on server
        run: docker-compose -f docker-compose-prod.yml pull
        env:
          DOCKER_HOST: ${{ secrets.SSH_HOST }}
          COMPOSE_PARAMIKO_SSH: '1'
      - name: Send up webhook
        continue-on-error: true
        uses: appleboy/discord-action@3c62873803ab2f276e28cdde23972899fb26e3a5
        with:
          webhook_id: ${{ secrets.WEBHOOK_ID }}
          webhook_token: ${{ secrets.WEBHOOK_TOKEN }}
          message: Running on server
      - name: Run on server
        run: docker compose -f docker-compose-prod.yml up -d
        env:
          DOCKER_HOST: ${{ secrets.SSH_HOST }}
          COMPOSE_PARAMIKO_SSH: '1'
      - name: Send webhook on failure
        if: ${{ failure() }}
        continue-on-error: true
        uses: appleboy/discord-action@3c62873803ab2f276e28cdde23972899fb26e3a5
        with:
          webhook_id: ${{ secrets.WEBHOOK_ID }}
          webhook_token: ${{ secrets.WEBHOOK_TOKEN }}
          message: |
            Failed to build [${{ github.sha }}](https://github.com/${{ github.repository }}/commit/${{ github.sha }})
      - name: Send webhook on cancellation
        if: ${{ cancelled() }}
        continue-on-error: true
        uses: appleboy/discord-action@3c62873803ab2f276e28cdde23972899fb26e3a5
        with:
          webhook_id: ${{ secrets.WEBHOOK_ID }}
          webhook_token: ${{ secrets.WEBHOOK_TOKEN }}
          message: |
            Cancelled build for [${{ github.sha }}](https://github.com/${{ github.repository }}/commit/${{ github.sha }})


Release

name: Release

on:
   push:
     tags:
       - 'v*'

jobs:
  formatting:
    name: Check Formatting
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: 3.8
    - name: Install Formatting
      run: |
        python -m pip install --upgrade pip
        pip install hacking isort
    - name: Check Formatting
      run: |
        flake8 .
        grep -rlIP '[^\x00-\x7F]' skrobot | grep '\.pyx\?' | sort > unicode.out; grep -rlI '# -\*- coding: \(utf-8\|latin-1\) -\*-' skrobot | grep '\.pyx\?' | sort > coding.out; comm -23 unicode.out coding.out > test_code.out; cat test_code.out;  test \! -s test_code.out
    - name: Check isort
      run: |
        isort -c .

  requirements-formatting:
    name: Check Requirements Formatting
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v3
    - name: Check Formatting
      run: |
        sort -C requirements.txt

  tests-3x:
    name: Run Unit Tests
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python-version: [3.6.15, 3.8.16, 3.10.10]
        os: [ubuntu-20.04]
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install APT On Linux
      run: |
        sudo apt-get update -qq -y
        sudo apt-get install -qq -y libspatialindex-dev freeglut3-dev libsuitesparse-dev libblas-dev liblapack-dev
    - name: Install Pytest
      run: |
        python -m pip install --upgrade pip setuptools wheel
        pip install Cython
        pip install pytest hacking
    - name: Install scikit-robot
      run: |
        pip cache purge
        pip install --no-cache-dir .[all] "numpy<2.0"
    - name: Run Pytest
      run: pytest -v tests

  tests-2x:
    name: Run Unit Tests
    runs-on: ubuntu-20.04
    container:
      image: python:2.7.18-buster
    steps:
    - uses: actions/checkout@v3
    - name: Install APT On Linux
      run: |
        apt-get update && apt-get install sudo cmake -y
        sudo apt-get update -qq -y
        sudo apt-get install -qq -y libspatialindex-dev freeglut3-dev libsuitesparse-dev libblas-dev liblapack-dev
    - name: Install Pytest
      run: |
        python -m pip install --upgrade pip setuptools wheel
        pip install Cython
        pip install pytest hacking
    - name: Install scikit-robot
      run: pip install --no-cache-dir .[all]
    - name: Run Pytest
      run: sudo pytest -v tests  # require sudo to access /tmp dir

  pypi:
    name: Release To PyPi
    needs: [formatting, requirements-formatting, tests-2x, tests-3x]
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.x'
    - name: Install publishing dependencies
      run: |
        python -m pip install --upgrade pip
        pip install setuptools wheel
    - name: Build
      run: |
        python setup.py sdist
        python setup.py bdist_wheel
    - name: Upload to PyPI
      uses: pypa/gh-action-pypi-publish@master
      with:
        password: ${{ secrets.PYPI_TOKEN }}


Run Tests

name: Run Tests

on:
  push:
    branches: [ main ]
  pull_request:
    branches: [ main ]
  schedule:
    - cron: '0, 0,12 * * *'

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

jobs:
  formatting:
    name: Check Formatting
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: 3.8
    - name: Install Formatting
      run: |
        python -m pip install --upgrade pip
        pip install hacking isort
    - name: Check Formatting
      run: |
        flake8 .
        grep -rlIP '[^\x00-\x7F]' skrobot | grep '\.pyx\?' | sort > unicode.out; grep -rlI '# -\*- coding: \(utf-8\|latin-1\) -\*-' skrobot | grep '\.pyx\?' | sort > coding.out; comm -23 unicode.out coding.out > test_code.out; cat test_code.out;  test \! -s test_code.out
    - name: Check isort
      run: |
        isort -c .

  requirements-formatting:
    name: Check Requirements Formatting
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v3
    - name: Check Formatting
      run: |
        sort -C requirements.txt

  tests-3x:
    name: Run Unit Tests
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        python-version: [3.6.15, 3.8.16, 3.10.10]
        os: [ubuntu-20.04]
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install APT On Linux
      run: |
        sudo apt-get update -qq -y
        sudo apt-get install -qq -y libspatialindex-dev freeglut3-dev libsuitesparse-dev libblas-dev liblapack-dev
    - name: Install Pytest
      run: |
        python -m pip install --upgrade pip setuptools wheel
        pip install Cython
        pip install pytest hacking
    - name: Install scikit-robot
      run: |
        pip cache purge
        pip install --no-cache-dir .[all] "numpy<2.0"
    - name: Run Pytest
      run: pytest -v tests

  tests-2x:
    name: Run Unit Tests
    runs-on: ubuntu-20.04
    container:
      image: python:2.7.18-buster
    steps:
    - uses: actions/checkout@v3
    - name: Install APT On Linux
      run: |
        apt-get update && apt-get install sudo cmake -y
        sudo apt-get update -qq -y
        sudo apt-get install -qq -y libspatialindex-dev freeglut3-dev libsuitesparse-dev libblas-dev liblapack-dev
    - name: Install Pytest
      run: |
        python -m pip install --upgrade pip setuptools wheel
        pip install Cython
        pip install pytest hacking
    - name: Install scikit-robot
      run: pip install --no-cache-dir .[all]
    - name: Run Pytest
      run: sudo pytest -v tests  # require sudo to access /tmp dir


Deploy

name: Deploy
on:
    push:
      paths-ignore:
        - "README.md"
        - "**/docs/**"
        - ".github/workflows/docs.yml"

    pull_request:
      branches:
        - master

    schedule:
        - cron: "0 11 * * *"

jobs:

    publish-pypi:
        name: Publish to PyPi
        if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
        runs-on: ubuntu-latest
        permissions:
          # IMPORTANT: this permission is mandatory for trusted publishing
          id-token: write
        steps:
            - name: Checkout source
              uses: actions/checkout@v4
            - name: Set up Python 3.9
              uses: actions/setup-python@v5
              with:
                  python-version: 3.9
            - name: Build package
              run: |
                  pip install wheel setuptools -U
                  python setup.py sdist
            - name: Publish
              uses: pypa/gh-action-pypi-publish@release/v1
              with:
                  skip-existing: true

    test-publish-pypi:
        name: Build & verify PyPi
        runs-on: ubuntu-latest

        steps:
            - uses: actions/checkout@v4
            - uses: actions/setup-python@v5
              with:
                  python-version: 3.9

            - name: Install twine
              run: |
                  python -m pip install twine
            - name: Build package
              run: |
                pip install wheel setuptools -U
                python setup.py sdist

            - name: List result
              run: |
                  ls -lh dist
            - name: Check long_description
              run: |
                  python -m twine check dist/*

    build-conda:
        name: Build in Conda
        if: startsWith(github.event.ref, 'refs/tags') == false
        runs-on: ${{ matrix.os }}
        defaults:
          run:
            shell: bash -l {0}
        strategy:
            fail-fast: false
            matrix:
                os: [ubuntu-latest, macos-latest, macos-12]
                python-version: [3.9]

        steps:
            - name: Checkout
              uses: actions/checkout@v4
              with:
                  fetch-depth: 0
            # - name: Set up Python
            #   uses: actions/setup-python@v5
            #   with:
            #       python-version: ${{ matrix.python-version }}
            # - name: Cache conda
            #   uses: actions/cache@v4
            #   env:
            #     # Increase this value to reset cache if etc/example-environment.yml has not changed
            #     CACHE_NUMBER: 0
            #   with:
            #       path: ~/conda_pkgs_dir
            #       key: conda-${{ matrix.os }}-python-${{ matrix.python-version }}-${{ hashFiles('ci/environment.yml') }}
            - name: Add conda ${{ matrix.python-version }} to system path
              uses: conda-incubator/setup-miniconda@v3
              with:
                  activate-environment: "test_env"
                  auto-activate-base: false
                  architecture: "x64"
                  miniforge-variant: Mambaforge
                  #miniforge-version: latest
                  python-version: ${{ matrix.python-version }}
                  auto-update-conda: true
                  environment-file: ci/environment.yml
                  use-only-tar-bz2: true
                  channel-priority: true
                  channels: conda-forge, xspecmodels, threeml

            - name: Init Env
              shell: bash -l {0}
              run: |

                  # Make sure we fail in case of error
                  if [[ ${{matrix.os}} == ubuntu-latest ]];
                  then

                      miniconda_os=Linux
                      compilers="gcc_linux-64 gxx_linux-64 gfortran_linux-64"
                  else  # osx

                      miniconda_os=MacOSX
                      compilers="clang_osx-64 clangxx_osx-64 gfortran_osx-64"

                      # On macOS we also need the conda libx11 libraries used to build xspec
                      # We also need to pin down ncurses, for now only on macos.
                      xorg="xorg-libx11"
                  fi

                  export PKG_VERSION=$(python -c "import versioneer;print(versioneer.get_version())")
                  echo "PKG_VERSION=$PKG_VERSION" >> $GITHUB_ENV

                  echo "HOME= ${HOME}"
                  echo "Conda installation path ${CONDA}"
                  echo "Building ${PKG_VERSION}"

                  #echo "Python version: ${{matrix.python-version}}"

                  conda install codecov pytest-cov
            - name: Conda list
              shell: bash -l {0}
              run: |
                  conda list
            - name: Build the Distribution
              shell: bash -l {0}
              run: |
                  # Build package

                  #conda install conda-build -n base
                  mamba install -c conda-forge boa

                  #conda build --python=${{matrix.python-version}} conda-dist/recipe
                  conda mambabuild --python=${{matrix.python-version}} conda-dist/recipe

                  #conda install --use-local astromodels
                  conda install -c ${CONDA}/envs/test_env/conda-bld/ astromodels
            - name: Test conda build
              shell: bash -l {0}
              run: |
                  cd astromodels/tests
                  echo "======>  importing XSPEC..."
                  python -c "import astromodels.xspec"
                  python -m pytest -vv --cov=astromodels --cov-report=xml
              env:
                  OMP_NUM_THREADS: 1
                  MKL_NUM_THREADS: 1
                  NUMEXPR_NUM_THREADS: 1
                  MPLBACKEND: "Agg"

    publish-conda:
        name: Publish to Conda
        if: github.event_name == 'push' && startsWith(github.event.ref, 'refs/tags')
        runs-on: ${{ matrix.os }}
        strategy:
            matrix:
                os: [ubuntu-latest, macos-latest]
                python-version: [3.9]

        steps:
            - name: Checkout
              uses: actions/checkout@v4
              with:
                  fetch-depth: 0
            - name: Cache conda
              uses: actions/cache@v4
              env:
                # Increase this value to reset cache if etc/example-environment.yml has not changed
                CACHE_NUMBER: 0
              with:
                  path: ~/conda_pkgs_dir
                  key: conda-${{ matrix.os }}-python-${{ matrix.python-version }}-${{ hashFiles('ci/environment.yml') }}
            - name: Add conda ${{ matrix.python-version }} to system path
              uses: conda-incubator/setup-miniconda@v3
              with:
                  activate-environment: "test_env"
                  auto-activate-base: false
                  architecture: "x64"
                  miniforge-variant: Mambaforge
                  #miniforge-version: latest
                  python-version: ${{ matrix.python-version }}
                  auto-update-conda: true
                  environment-file: ci/environment.yml
                  use-only-tar-bz2: true
                  channel-priority: true
                  channels: conda-forge, xspecmodels, threeml

            - name: Init Env
              shell: bash -l {0}
              run: |

                  # Make sure we fail in case of error
                  if [[ ${{matrix.os}} == ubuntu-latest ]];
                  then
                  miniconda_os=Linux
                  compilers="gcc_linux-64 gxx_linux-64 gfortran_linux-64"
                  else  # osx
                  miniconda_os=MacOSX
                  compilers="clang_osx-64 clangxx_osx-64 gfortran_osx-64"

                  # On macOS we also need the conda libx11 libraries used to build xspec
                  # We also need to pin down ncurses, for now only on macos.
                  xorg="xorg-libx11"
                  fi

                  export PKG_VERSION=$(python -c "import versioneer;print(versioneer.get_version())")
                  echo "PKG_VERSION=$PKG_VERSION" >> $GITHUB_ENV

                  echo "HOME= ${HOME}"
                  echo "Conda installation path ${CONDA}"
                  echo "Building ${PKG_VERSION}"
            - name: Build the Distribution
              shell: bash -l {0}
              run: |
                  # Build package

                  #conda install conda-build -n base
                  mamba install -c conda-forge boa

                  #conda build --python=${{matrix.python-version}} conda-dist/recipe
                  conda mambabuild --python=${{matrix.python-version}} conda-dist/recipe

                  #conda install --use-local astromodels
                  conda install -c ${CONDA}/envs/test_env/conda-bld/ astromodels
            - name: Publish to Conda
              shell: bash -l {0}
              run: |

                  conda install -c conda-forge anaconda-client
                  echo "Uploading ${{matrix.os}} IsDev: ${ISDEV} "

                  if [[ "${ISDEV}" == "true" ]]; then

                  LABEL="--label dev"

                  fi

                  if [[ ${{matrix.os}} == ubuntu-latest ]];
                  then

                  anaconda -v --show-traceback -t $UPLOAD_TOKEN upload -u threeml ${CONDA}/envs/test_env/conda-bld/linux-64/*.tar.bz2 --force $LABEL

                  else

                  anaconda -v --show-traceback -t $UPLOAD_TOKEN upload -u threeml ${CONDA}/envs/test_env/conda-bld/osx-64/*.tar.bz2 --force $LABEL

                  fi

              env:
                  UPLOAD_TOKEN: ${{secrets.CONDA_UPLOAD_TOKEN}}
                  CURRENT_OS: ${{matrix.os}}
                  ISDEV: ${{contains(github.ref, 'dev')}}


Docs

name: Docs
on: [push, release]

jobs:
  notebooks:
    name: "Build the notebooks for the docs"
    runs-on: macos-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
          fetch-depth: 0 # otherwise, you will failed to push refs to dest repo

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: 3.9
      #- name: Checkout
      #  uses: actions/checkout@v4
      - name: Init Environment
        #shell: bash -l {0}
        run: |

          brew update
          brew install --cask basictex

          eval "$(/usr/libexec/path_helper)"

      - name: Setup base pkgs
        #shell: bash -l {0}
        run: |
          brew install hdf5

          python -m pip install --upgrade pip wheel

          python -m pip install numpy numba tempita jupytext jupyterthemes jupyter_latex_envs papermill "matplotlib<3.9" astropy pandas tables healpy
          python -m pip install ebltable black cython pkgconfig h5py

      - name: Install the package
        #shell: bash -l {0}
        run: |

          #python setup.py develop
          pip3 install -e .

      - name: Execute the notebooks
        #shell: bash -l {0}
        run: |

          # Download an example file

          wget https://www.astropy.org/astropy-data/photometry/spitzer_example_image.fits

          mv spitzer_example_image.fits docs/md/

          jupytext --to ipynb --execute docs/md/*.md
          mv docs/md/*.ipynb docs/notebooks/
          ls docs/notebooks
        env:
          OMP_NUM_THREADS: 1
          MKL_NUM_THREADS: 1
          NUMEXPR_NUM_THREADS: 1
          MPLBACKEND: "Agg"
      - name: Create function gallery
        #shell: bash -l {0}
        run: |
          # move to the scripts directory
          cd scripts

          # now generate the function docs
          python generate_func_docs.py

          cd ..

          ls docs/notebooks

      - uses: actions/upload-artifact@v4
        with:
          name: notebooks-for-${{ github.sha }}
          path: docs/notebooks

      - name: Sleep for 10 min
        uses: juliangruber/sleep-action@v1
        with:
          time: 10m

      - name: Trigger RTDs build
        uses: dfm/rtds-action@main
        with:
          webhook_url: ${{ secrets.RTDS_WEBHOOK_URL }}
          webhook_token: ${{ secrets.RTDS_WEBHOOK_TOKEN }}
          commit_ref: ${{ github.ref }}

  api_doc:
    name: "Create the API stubs"
    runs-on: macos-latest
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
          fetch-depth: 0 # otherwise, you will failed to push refs to dest repo

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: 3.9

      - name: Build the API doc
        run: |

          brew install c-blosc
          brew install hdf5

          pip3 install tempita tables
          pip3 install cython blosc2 h5py
          pip3 install numpy scipy numba astropy pandas
          pip3 install wheel pkgconfig

          #python setup.py develop
          pip3 install -e .

          brew install sphinx-doc pandoc

          pip3 install mock recommonmark
          pip3 install sphinx-rtd-dark-mode
          pip3 install -U sphinx

          sphinx-apidoc -f -o docs/api/ astromodels

      - uses: actions/upload-artifact@v4
        with:
          name: api-stubs-for-${{ github.sha }}
          path: docs/api

  build_docs:
    name: "Build the Documentation"
    runs-on: macos-latest
    needs: [notebooks, api_doc]
    steps:

      - uses: actions/checkout@v4
        with:
          persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
          fetch-depth: 0 # otherwise, you will failed to push refs to dest repo

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: 3.9

      - name: Install package
        run: |

          brew install c-blosc
          brew install hdf5

          pip3 install tempita
          pip3 install cython blosc2 h5py
          pip3 install numpy scipy numba astropy

          brew install sphinx-doc pandoc

          pip3 install wheel pkgconfig
          pip3 install mock recommonmark
          pip3 install sphinx-rtd-dark-mode sphinx-math-dollar
          pip3 install -r docs/requirements.txt

          #python setup.py develop
          pip3 install -e .

          rm -rf docs/md/*

      - uses: actions/download-artifact@master
        with:
          name: notebooks-for-${{ github.sha }}
          path: docs/notebooks

      - uses: actions/download-artifact@master
        with:
          name: api-stubs-for-${{ github.sha }}
          path: docs/notebooks/api

      - name: Build and Commit
        uses: sphinx-notes/pages@v2
        with:
          documentation_path: docs
          sphinx_version: 5.1.1
          requirements_path: docs/requirements.txt

      - name: Push changes
        if: github.event_name == 'push' #&& startsWith(github.event.ref, 'refs/tags')
        uses: ad-m/github-push-action@master
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          branch: gh-pages


Close stale issues and PRs

name: Close stale issues and PRs
on:
  schedule:
  - cron: 30 1 * * *

jobs:
  stale:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/stale@v3
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}
        stale-issue-message: 'This issue has become stale. Is there an update? We will close in 15 days'
        stale-pr-message: 'This PR has become stale. We will close in 60 days'
        stale-issue-label: 'no-issue-activity'
        stale-pr-label: 'no-pr-activity'
        days-before-pr-stale: 90
        days-before-issue-stale: 30
        days-before-issue-close: 15
        days-before-pr-close: 60


CI

name: CI
on:
    push:
      paths-ignore:
        - "README.md"
        - "**/docs/**"
        - ".github/workflows/docs.yml"

    pull_request:
      branches:
        - master
        - dev

    schedule:
        - cron: "0 11 * * *"

jobs:
    skip_duplicate:
        runs-on: ubuntu-latest
        outputs:
            should_skip: ${{ steps.skip_check.outputs.should_skip }}
        steps:
            - id: skip_check
              uses: fkirc/skip-duplicate-actions@master
              with:
                  github_token: ${{ github.token }}
                  concurrent_skipping: never
                  skip_after_successful_duplicate: true
                  paths_ignore: '["**/README.md", "**/docs/**"]'
                  do_not_skip: '["pull_request",  "schedule"]'

    test-pip:
        name: Test with Pip install
        needs: skip_duplicate
        if: ${{ needs.skip_duplicate.outputs.should_skip == 'false' }}
        strategy:
            fail-fast: false
            matrix:
                python-version: ["3.9", "3.10", "3.11"]
                os: ["ubuntu-latest", "macos-latest", "macos-12"]
        runs-on: ${{ matrix.os }}
        steps:
            - name: Checkout
              uses: actions/checkout@v4
            - name: Set up Python ${{ matrix.python-version }}
              uses: actions/setup-python@v5
              with:
                  python-version: ${{ matrix.python-version }}
            - name: Install dependencies
              run: |
                  python -m pip install --upgrade pip wheel
                  pip install --upgrade flake8 coverage pytest-cov cython

                  if [[ ${{matrix.os}} == macos-latest ]];
                  then
                    brew update
                    brew install hdf5
                  fi

                  pip install -e .
            - name: Lint with flake8
              run: |
                  # stop the build if there are Python syntax errors or undefined names
                  flake8

            - name: Test pip on ${{ matrix.python-version }}
              run: |
                  python -m pytest -vv --cov=astromodels/ --cov-report=xml --junitxml=junit/test-results.xml
              env:
                  OMP_NUM_THREADS: 1
                  MKL_NUM_THREADS: 1
                  NUMEXPR_NUM_THREADS: 1
                  MPLBACKEND: "Agg"

            - name: Publish Unit Test Results Ubuntu
              uses: EnricoMi/publish-unit-test-result-action@v2
              if: runner.os == 'Linux'
              with:
                files: junit/test-*.xml
                comment_mode: off

            - name: Publish Unit Test Results MacOS
              uses: EnricoMi/publish-unit-test-result-action/composite@v2
              if: runner.os == 'macOS'
              with:
                files: junit/test-*.xml
                comment_mode: off

            - name: Upload coverage to Codecov
              uses: codecov/codecov-action@v4
              with:
                  file: ./coverage.xml
                  files: ./coverage1.xml,./coverage2.xml
                  directory: ./coverage/reports/

    test-conda:
        name: Test Conda version
        needs: skip_duplicate
        if: ${{ needs.skip_duplicate.outputs.should_skip == 'false' }}
        strategy:
            fail-fast: false
            matrix:
                os: ["ubuntu-latest", "macos-latest", "macos-12"]
                python-version: [3.9]
        runs-on: ${{ matrix.os }}
        steps:
            - name: Checkout
              uses: actions/checkout@v4
            # - name: Cache conda
            #   uses: actions/cache@v4
            #   env:
            #     # Increase this value to reset cache if etc/example-environment.yml has not changed
            #     CACHE_NUMBER: 0
            #   with:
            #       path: ~/conda_pkgs_dir
            #       key: conda-${{ matrix.os }}-python-${{ matrix.python-version }}-${{ hashFiles('ci/environment.yml') }}
            - name: Add conda ${{ matrix.python-version }} to system path
              uses: conda-incubator/setup-miniconda@v3
              with:
                  activate-environment: "test_env"
                  auto-activate-base: false
                  architecture: "x64"
                  #mamba-version: "*"
                  miniforge-version: latest
                  python-version: ${{ matrix.python-version }}
                  auto-update-conda: true
                  environment-file: ci/environment.yml
                  use-only-tar-bz2: true
                  channel-priority: strict
                  channels: conda-forge, xspecmodels

            - name: Init Env
              shell: bash -l {0}
              run: |
                  # Make sure we fail in case of error
                  if [[ ${{matrix.os}} == ubuntu-latest ]];
                  then
                  miniconda_os=Linux
                  compilers="gcc_linux-64 gxx_linux-64 gfortran_linux-64"
                  else  # osx
                  miniconda_os=MacOSX
                  compilers="clang_osx-64 clangxx_osx-64 gfortran_osx-64"

                  # On macOS we also need the conda libx11 libraries used to build xspec
                  # We also need to pin down ncurses, for now only on macos.
                  xorg="xorg-libx11"
                  fi

                  export PKG_VERSION=$(python -c "import versioneer;print(versioneer.get_version())")

                  mamba install ${PKG} codecov pytest-cov git flake8

            - name: Conda list
              shell: bash -l {0}
              run: |
                  conda list
            - name: install it
              shell: bash -l {0}
              run: |
                  pip install -e .
            - name: Lint with flake8
              shell: bash -l {0}
              run: |
                  # stop the build if there are Python syntax errors or undefined names
                  flake8

            - name: Test conda install
              shell: bash -l {0}
              run: |

                  echo "======>  importing XSPEC..."
                  python -c "import astromodels.xspec"
                  python -m pytest -vv --cov=astromodels --cov-report=xml

              env:
                  OMP_NUM_THREADS: 1
                  MKL_NUM_THREADS: 1
                  NUMEXPR_NUM_THREADS: 1
                  MPLBACKEND: "Agg"

            - name: Upload coverage to Codecov
              uses: codecov/codecov-action@v4
              with:
                  file: ./coverage.xml
                  files: ./coverage1.xml,./coverage2.xml
                  directory: ./coverage/reports/


Mark stale issues and pull requests

# This workflow warns and then closes issues and PRs that have had no activity for a specified amount of time.
#
# You can adjust the behavior by modifying this file.
# For more information, see:
# https://github.com/actions/stale
name: Mark stale issues and pull requests

on:
  schedule:
  - cron: '0 * * * *'

jobs:
  stale:

    runs-on: ubuntu-latest
    permissions:
      issues: write
      pull-requests: write

    steps:
    - uses: actions/stale@v5
      with:
        repo-token: ${{ secrets.GITHUB_TOKEN }}

        # STALE
        # Duration of inactivity before making Issues/PRs stale
        days-before-pr-stale: 15
        days-before-issue-stale: 60

        # Stale Comment
        stale-pr-message: >
            This Pull request has been automatically marked as STALE due to inactivity for 15 days
            and will be CLOSED on further inactivity on the PR for another 15 days.
        stale-issue-message: >
            This Issue has been marked as STALE due to inactivity for 60 days.
            and will be CLOSED after another 30 days on further inactivity.

        # stale label
        stale-pr-label: 'stale-warning-on-inactivity'
        stale-issue-label: 'stale-warning-on-inactivity'

        # remove stale label when updated.
        remove-stale-when-updated: true

        # CLOSE
        # Duration of inactivity before closing stale Issues/PRs
        days-before-pr-close: 15
        days-before-issue-close: 30

        # closing Comment on the staled Issues/PRs
        close-pr-message: >
            This pull request has been automatically CLOSED,
            because there has been no activity for 15 days after marking PR as STALE.
        close-issue-message: >
            This Issue has been automatically CLOSED,
            because there has been no activity for 30 days after marking issue as STALE.

        # close Issue/pr label
        close-pr-label: "closed-due-to-inactivity"
        close-issue-label: "closed-due-to-inactivity"

        # Change the order used to fetch the issues and pull requests
        ascending: true

        # labels to exempt from stale action
        exempt-pr-labels: "pinned,need-help"

        # Maximum number of operations per run
        operations-per-run: 300


tests

name: tests

on: [push, pull_request]

jobs:
  tox:
    runs-on: ubuntu-20.04
    strategy:
      max-parallel: 5
      matrix:
        python-version:
        - '3.9.16'
        - '3.8.15'
    steps:
    - uses: actions/checkout@v3
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: pip install tox
    - name: Test with tox
      run: tox -e py


PR review

name: "PR review"
on:
  pull_request_target:
    types: [opened, reopened, synchronize, edited, edited]

jobs:
  labeler:
    name: review
    permissions:
      contents: read
      pull-requests: write
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: false

    # Label PRs
    - uses: actions/labeler@v5.0.0
      with:
        dot: true
        configuration-path: .github/labeler.yaml

    # Check PR target branch
    - name: check branch
      uses: Vankka/pr-target-branch-action@def32ec9d93514138d6ac0132ee62e120a72aed5
      if: github.repository == 'commaai/openpilot'
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      with:
        target: /^(?!master$).*/
        exclude: /commaai:.*/
        change-to: ${{ github.base_ref }}
        already-exists-action: close_this
        already-exists-comment: "Your PR should be made against the `master` branch"

    # Welcome comment
    - name: comment
      uses: thollander/actions-comment-pull-request@fabd468d3a1a0b97feee5f6b9e499eab0dd903f6
      if: github.event.pull_request.head.repo.full_name != 'commaai/openpilot'
      with:
        message: |
            <!-- _(run_id **${{ github.run_id }}**)_ -->
            Thanks for contributing to openpilot! In order for us to review your PR as quickly as possible, check the following:
            * Convert your PR to a draft unless it's ready to review
            * Read the [contributing docs](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md)
            * Before marking as "ready for review", ensure:
              * the goal is clearly stated in the description
              * all the tests are passing
              * the change is [something we merge](https://github.com/commaai/openpilot/blob/master/docs/CONTRIBUTING.md#what-gets-merged)
              * include a route or your device' dongle ID if relevant
        comment_tag: run_id
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


badges

name: badges
on:
  schedule:
    - cron: '0 * * * *'
  workflow_dispatch:

env:
  BASE_IMAGE: openpilot-base
  DOCKER_REGISTRY: ghcr.io/commaai
  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $DOCKER_REGISTRY/$BASE_IMAGE:latest /bin/bash -c

jobs:
  badges:
    name: create badges
    runs-on: ubuntu-latest
    if: github.repository == 'commaai/openpilot'
    permissions:
      contents: write
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
    - name: Push badges
      run: |
        ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/ui/translations/create_badges.py"

        rm .gitattributes

        git checkout --orphan badges
        git rm -rf --cached .
        git config user.email "badge-researcher@comma.ai"
        git config user.name "Badge Researcher"

        git add translation_badge.svg
        git commit -m "Add/Update badges"
        git push -f origin HEAD


docs

name: docs

on:
  push:
    branches:
      - master
  pull_request:

concurrency:
  group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
  cancel-in-progress: true

env:
  BASE_IMAGE: openpilot-base

  BUILD: selfdrive/test/docker_build.sh base

  RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c

jobs:
  docs:
    name: build docs
    runs-on: ubuntu-latest
    timeout-minutes: 45
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
    - name: Build openpilot
      run: |
        ${{ env.RUN }} "scons -j$(nproc)"
    - name: Build docs
      run: |
        ${{ env.RUN }} "apt update && apt install -y doxygen && cd docs && make -j$(nproc) html"

    - uses: actions/checkout@v4
      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
      with:
        path: openpilot-docs
        ssh-key: ${{ secrets.OPENPILOT_DOCS_KEY }}
        repository: commaai/openpilot-docs
    - name: Push
      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
      run: |
        set -x

        source release/identity.sh

        cd openpilot-docs

        git checkout --orphan tmp
        git rm -rf .

        cp -r ../build/docs/html/ docs/
        cp -r ../docs/README.md .
        touch docs/.nojekyll
        echo -n docs.comma.ai > docs/CNAME
        git add -f .

        git commit -m "build docs"

        # docs live in different repo to not bloat openpilot's full clone size
        git push -f origin tmp:gh-pages


prebuilt

name: prebuilt
on:
  schedule:
    - cron: '0 * * * *'
  workflow_dispatch:

env:
  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
  BUILD: selfdrive/test/docker_build.sh prebuilt

jobs:
  build_prebuilt:
    name: build prebuilt
    runs-on: ubuntu-latest
    if: github.repository == 'commaai/openpilot'
    env:
      PUSH_IMAGE: true
    permissions:
      checks: read
      contents: read
      packages: write
    steps:
    - name: Wait for green check mark
      if: ${{ github.event_name != 'workflow_dispatch' }}
      uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc
      with:
        ref: master
        wait-interval: 30
        running-workflow-name: 'build prebuilt'
        repo-token: ${{ secrets.GITHUB_TOKEN }}
        check-regexp: ^((?!.*(build master-ci).*).)*$
    - uses: actions/checkout@v4
      with:
        submodules: true
    - run: git lfs pull
    - name: Build and Push docker image
      run: |
        $DOCKER_LOGIN
        eval "$BUILD"


release

name: release
on:
  schedule:
    - cron: '0 10 * * *'
  workflow_dispatch:

jobs:
  build_masterci:
    name: build master-ci
    env:
      TARGET_DIR: /tmp/openpilot
      ImageOS: ubuntu20
    container:
      image: ghcr.io/commaai/openpilot-base:latest
    runs-on: ubuntu-latest
    if: github.repository == 'commaai/openpilot'
    permissions:
      checks: read
      contents: write
    steps:
    - name: Install wait-on-check-action dependencies
      run: |
        sudo apt-get update
        sudo apt-get install -y libyaml-dev
    - name: Wait for green check mark
      if: ${{ github.event_name != 'workflow_dispatch' }}
      uses: lewagon/wait-on-check-action@ccfb013c15c8afb7bf2b7c028fb74dc5a068cccc
      with:
        ref: master
        wait-interval: 30
        running-workflow-name: 'build master-ci'
        repo-token: ${{ secrets.GITHUB_TOKEN }}
        check-regexp: ^((?!.*(build prebuilt).*).)*$
    - uses: actions/checkout@v4
      with:
        submodules: true
        fetch-depth: 0
    - name: Pull LFS
      run: |
        git config --global --add safe.directory '*'
        git lfs pull
    - name: Build master-ci
      run: |
        release/build_devel.sh
    - name: Run tests
      run: |
        export PYTHONPATH=$TARGET_DIR
        cd $TARGET_DIR
        scons -j$(nproc)
        pytest -n logical selfdrive/car/tests/test_car_interfaces.py
    - name: Push master-ci
      run: |
        unset TARGET_DIR
        BRANCH=master-ci release/build_devel.sh


repo maintenance

name: repo maintenance

on:
  schedule:
    - cron: "0 14 * * 1" # every Monday at 2am UTC (6am PST)
  workflow_dispatch:

jobs:
  bump_submodules:
    name: bump_submodules
    runs-on: ubuntu-latest
    container:
      image: ghcr.io/commaai/openpilot-base:latest
    if: github.repository == 'commaai/openpilot'
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - name: bump submodules
      run: |
        git config --global --add safe.directory '*'
        git -c submodule."tinygrad".update=none submodule update --remote
        git add .
    - name: Create Pull Request
      uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83
      with:
        author: Vehicle Researcher <user@comma.ai>
        token: ${{ secrets.ACTIONS_CREATE_PR_PAT }}
        commit-message: bump submodules
        title: '[bot] Bump submodules'
        branch: auto-bump-submodules
        base: master
        delete-branch: true
        body: 'Automatic PR from repo-maintenance -> bump_submodules'
        labels: bot
  package_updates:
    name: package_updates
    runs-on: ubuntu-latest
    container:
      image: ghcr.io/commaai/openpilot-base:latest
    if: github.repository == 'commaai/openpilot'
    steps:
    - uses: actions/checkout@v4
    - name: poetry lock
      run: |
        pip install poetry
        poetry lock
    - name: pre-commit autoupdate
      run: |
        git config --global --add safe.directory '*'
        pre-commit autoupdate
    - name: Create Pull Request
      uses: peter-evans/create-pull-request@9153d834b60caba6d51c9b9510b087acf9f33f83
      with:
        author: Vehicle Researcher <user@comma.ai>
        token: ${{ secrets.ACTIONS_CREATE_PR_PAT }}
        commit-message: Update Python packages and pre-commit hooks
        title: '[bot] Update Python packages and pre-commit hooks'
        branch: auto-package-updates
        base: master
        delete-branch: true
        body: 'Automatic PR from repo-maintenance -> package_updates'
        labels: bot


selfdrive

name: selfdrive

on:
  push:
    branches:
      - master
  pull_request:
  workflow_dispatch:

concurrency:
  group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
  cancel-in-progress: true

env:
  PYTHONWARNINGS: error
  BASE_IMAGE: openpilot-base
  AZURE_TOKEN: ${{ secrets.AZURE_COMMADATACI_OPENPILOTCI_TOKEN }}

  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}
  BUILD: selfdrive/test/docker_build.sh base

  RUN: docker run --shm-size 1G -v $PWD:/tmp/openpilot -w /tmp/openpilot -e CI=1 -e PRE_COMMIT_HOME=/tmp/pre-commit -e PYTHONWARNINGS=error -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/pre-commit:/tmp/pre-commit -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c

  PYTEST: pytest --continue-on-collection-errors --cov --cov-report=xml --cov-append --durations=0 --durations-min=5 --hypothesis-seed 0 -n logical

jobs:
  build_release:
    name: build release
    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
                   ((github.event_name != 'pull_request') ||
                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
    env:
      STRIPPED_DIR: /tmp/releasepilot
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - run: git lfs pull
    - name: Build devel
      timeout-minutes: 1
      run: TARGET_DIR=$STRIPPED_DIR release/build_devel.sh
    - uses: ./.github/workflows/setup-pre-commit
    - uses: ./.github/workflows/setup-with-retry
    - name: Check submodules
      if: github.ref == 'refs/heads/master' && github.repository == 'commaai/openpilot'
      timeout-minutes: 1
      run: release/check-submodules.sh
    - name: Build openpilot and run checks
      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
      run: |
        cd $STRIPPED_DIR
        ${{ env.RUN }} "python system/manager/build.py"
    - name: Run tests
      timeout-minutes: 3
      run: |
        cd $STRIPPED_DIR
        ${{ env.RUN }} "release/check-dirty.sh && \
                        MAX_EXAMPLES=5 $PYTEST -m 'not slow' selfdrive/car"
    - name: pre-commit
      timeout-minutes: 3
      run: |
        cd $GITHUB_WORKSPACE
        cp .pre-commit-config.yaml $STRIPPED_DIR
        cp pyproject.toml $STRIPPED_DIR
        cp poetry.lock $STRIPPED_DIR
        cd $STRIPPED_DIR
        ${{ env.RUN }} "unset PYTHONWARNINGS && SKIP=check-added-large-files,check-hooks-apply,check-useless-excludes pre-commit run --all && chmod -R 777 /tmp/pre-commit"

  build:
    strategy:
      matrix:
        arch: ${{ fromJson('["x86_64"]') }} # TODO: Re-add build test for aarch64 once we switched to ubuntu-2404
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - name: Setup docker push
      if: github.ref == 'refs/heads/master' && github.event_name != 'pull_request' && github.repository == 'commaai/openpilot'
      run: |
        echo "PUSH_IMAGE=true" >> "$GITHUB_ENV"
        : # (TODO: Re-add this once we test other archs) echo "TARGET_ARCHITECTURE=${{ matrix.arch }}" >> "$GITHUB_ENV"
        $DOCKER_LOGIN
    - uses: ./.github/workflows/setup-with-retry
      with:
        docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
    - uses: ./.github/workflows/compile-openpilot
      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 15 || 30) }} # allow more time when we missed the scons cache

  static_analysis:
    name: static analysis
    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
                   ((github.event_name != 'pull_request') ||
                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - uses: ./.github/workflows/setup-pre-commit
    - uses: ./.github/workflows/setup-with-retry
    - name: Build openpilot
      run: ${{ env.RUN }} "scons -j$(nproc)"
    - name: pre-commit
      timeout-minutes: 4
      run: ${{ env.RUN }} "unset PYTHONWARNINGS && pre-commit run --all && chmod -R 777 /tmp/pre-commit"

  unit_tests:
    name: unit tests
    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
                   ((github.event_name != 'pull_request') ||
                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
      with:
        docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
    - name: Build openpilot
      timeout-minutes: ${{ ((steps.restore-scons-cache.outputs.cache-hit == 'true') && 10 || 30) }} # allow more time when we missed the scons cache
      run: ${{ env.RUN }} "scons -j$(nproc)"
    - name: Run unit tests
      timeout-minutes: 15
      run: |
        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
                        $PYTEST --timeout 60 -m 'not slow' && \
                        ./selfdrive/ui/tests/create_test_translations.sh && \
                        QT_QPA_PLATFORM=offscreen ./selfdrive/ui/tests/test_translations && \
                        pytest ./selfdrive/ui/tests/test_translations.py"
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v4
      with:
        name: ${{ github.job }}
      env:
        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

  process_replay:
    name: process replay
    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
                   ((github.event_name != 'pull_request') ||
                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
      with:
        docker_hub_pat: ${{ secrets.DOCKER_HUB_PAT }}
    - name: Cache test routes
      id: dependency-cache
      uses: actions/cache@v4
      with:
        path: .ci_cache/comma_download_cache
        key: proc-replay-${{ hashFiles('.github/workflows/selfdrive_tests.yaml', 'selfdrive/test/process_replay/ref_commit', 'selfdrive/test/process_replay/test_regen.py') }}
    - name: Build openpilot
      run: |
        ${{ env.RUN }} "scons -j$(nproc)"
    - name: Run replay
      timeout-minutes: 30
      run: |
        ${{ env.RUN }} "coverage run selfdrive/test/process_replay/test_processes.py -j$(nproc) && \
                        chmod -R 777 /tmp/comma_download_cache && \
                        coverage combine && \
                        coverage xml"
    - name: Print diff
      id: print-diff
      if: always()
      run: cat selfdrive/test/process_replay/diff.txt
    - uses: actions/upload-artifact@v4
      if: always()
      continue-on-error: true
      with:
        name: process_replay_diff.txt
        path: selfdrive/test/process_replay/diff.txt
    - name: Upload reference logs
      if: ${{ failure() && steps.print-diff.outcome == 'success' && github.repository == 'commaai/openpilot' && env.AZURE_TOKEN != '' }}
      run: |
        ${{ env.RUN }} "unset PYTHONWARNINGS && AZURE_TOKEN='$AZURE_TOKEN' python selfdrive/test/process_replay/test_processes.py -j$(nproc) --upload-only"
    # PYTHONWARNINGS triggers a SyntaxError in onnxruntime
    - name: Run model replay with ONNX
      timeout-minutes: 4
      run: |
        ${{ env.RUN }} "unset PYTHONWARNINGS && \
                        ONNXCPU=1 NO_NAV=1 coverage run selfdrive/test/process_replay/model_replay.py && \
                        coverage combine && coverage xml"
    - name: Run regen
      timeout-minutes: 4
      run: |
        ${{ env.RUN }} "ONNXCPU=1 $PYTEST selfdrive/test/process_replay/test_regen.py && \
                        chmod -R 777 /tmp/comma_download_cache"
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v4
      with:
        name: ${{ github.job }}
      env:
        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

  test_cars:
    name: cars
    runs-on: ${{ ((github.repository == 'commaai/openpilot') &&
                   ((github.event_name != 'pull_request') ||
                    (github.event.pull_request.head.repo.full_name == 'commaai/openpilot'))) && 'namespace-profile-amd64-8x16' || 'ubuntu-latest' }}
    strategy:
      fail-fast: false
      matrix:
        job: [0, 1]
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
    - name: Cache test routes
      id: dependency-cache
      uses: ./.github/workflows/auto-cache
      with:
        path: .ci_cache/comma_download_cache
        key: car_models-${{ hashFiles('selfdrive/car/tests/test_models.py', 'selfdrive/car/tests/routes.py') }}-${{ matrix.job }}
    - name: Build openpilot
      run: ${{ env.RUN }} "scons -j$(nproc)"
    - name: Test car models
      timeout-minutes: 20
      run: |
        ${{ env.RUN }} "$PYTEST selfdrive/car/tests/test_models.py && \
                        chmod -R 777 /tmp/comma_download_cache"
      env:
        NUM_JOBS: 2
        JOB_ID: ${{ matrix.job }}
    - name: "Upload coverage to Codecov"
      uses: codecov/codecov-action@v4
      with:
        name: ${{ github.job }}-${{ matrix.job }}
      env:
        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

  car_docs_diff:
    name: PR comments
    runs-on: ubuntu-latest
    if: github.event_name == 'pull_request'
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
          ref: ${{ github.event.pull_request.base.ref }}
      - run: git lfs pull
      - uses: ./.github/workflows/setup-with-retry
      - name: Get base car info
        run: |
          ${{ env.RUN }} "scons -j$(nproc) && python selfdrive/debug/dump_car_docs.py --path /tmp/openpilot_cache/base_car_docs"
          sudo chown -R $USER:$USER ${{ github.workspace }}
      - uses: actions/checkout@v4
        with:
          submodules: true
          path: current
      - run: cd current && git lfs pull
      - name: Save car docs diff
        id: save_diff
        run: |
          cd current
          ${{ env.RUN }} "scons -j$(nproc)"
          output=$(${{ env.RUN }} "python selfdrive/debug/print_docs_diff.py --path /tmp/openpilot_cache/base_car_docs")
          output="${output//$'\n'/'%0A'}"
          echo "::set-output name=diff::$output"
      - name: Find comment
        if: ${{ env.AZURE_TOKEN != '' }}
        uses: peter-evans/find-comment@3eae4d37986fb5a8592848f6a574fdf654e61f9e
        id: fc
        with:
          issue-number: ${{ github.event.pull_request.number }}
          body-includes: This PR makes changes to
      - name: Update comment
        if: ${{ steps.save_diff.outputs.diff != '' && env.AZURE_TOKEN != '' }}
        uses: peter-evans/create-or-update-comment@71345be0265236311c031f5c7866368bd1eff043
        with:
          comment-id: ${{ steps.fc.outputs.comment-id }}
          issue-number: ${{ github.event.pull_request.number }}
          body: "${{ steps.save_diff.outputs.diff }}"
          edit-mode: replace
      - name: Delete comment
        if: ${{ steps.fc.outputs.comment-id != '' && steps.save_diff.outputs.diff == '' && env.AZURE_TOKEN != '' }}
        uses: actions/github-script@v7
        with:
          script: |
            github.rest.issues.deleteComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              comment_id: ${{ steps.fc.outputs.comment-id }}
            })

  create_ui_report:
    name: Create UI Report
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
      - uses: ./.github/workflows/setup-with-retry
      - name: Build openpilot
        run: ${{ env.RUN }} "scons -j$(nproc)"
      - name: Create Test Report
        run: >
            ${{ env.RUN }} "PYTHONWARNINGS=ignore &&
                            source selfdrive/test/setup_xvfb.sh &&
                            python selfdrive/ui/tests/test_ui/run.py"
      - name: Upload Test Report
        uses: actions/upload-artifact@v4
        with:
          name: report
          path: selfdrive/ui/tests/test_ui/report


stale

name: stale
on:
  schedule:
    - cron: '30 1 * * *'
  workflow_dispatch:

env:
  DAYS_BEFORE_PR_CLOSE: 3
  DAYS_BEFORE_PR_STALE: 14

jobs:
  stale:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/stale@v9
        with:
          exempt-milestones: true

          # pull request config
          stale-pr-message: 'This PR has had no activity for ${{ env.DAYS_BEFORE_PR_STALE }} days. It will be automatically closed in ${{ env.DAYS_BEFORE_PR_CLOSE }} days if there is no activity.'
          close-pr-message: 'This PR has been automatically closed due to inactivity. Feel free to re-open once activity resumes.'
          stale-pr-label: stale
          delete-branch: ${{ github.event.pull_request.head.repo.full_name == 'commaai/openpilot' }} # only delete branches on the main repo
          exempt-pr-labels: "ignore stale,needs testing,car port,car" # if wip or it needs testing from the community, don't mark as stale
          days-before-pr-stale: ${{ env.DAYS_BEFORE_PR_STALE }}
          days-before-pr-close: ${{ env.DAYS_BEFORE_PR_CLOSE }}

          # issue config
          days-before-issue-stale: -1 # ignore issues for now


tools

name: tools

on:
  push:
    branches:
      - master
  pull_request:

concurrency:
  group: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.run_id || github.head_ref || github.ref }}-${{ github.workflow }}-${{ github.event_name }}
  cancel-in-progress: true

env:
  BASE_IMAGE: openpilot-base
  DOCKER_LOGIN: docker login ghcr.io -u ${{ github.actor }} -p ${{ secrets.GITHUB_TOKEN }}

  BUILD: selfdrive/test/docker_build.sh base

  RUN: docker run --shm-size 1G -v $GITHUB_WORKSPACE:/tmp/openpilot -w /tmp/openpilot -e FILEREADER_CACHE=1 -e PYTHONPATH=/tmp/openpilot -e NUM_JOBS -e JOB_ID -e GITHUB_ACTION -e GITHUB_REF -e GITHUB_HEAD_REF -e GITHUB_SHA -e GITHUB_REPOSITORY -e GITHUB_RUN_ID -v $GITHUB_WORKSPACE/.ci_cache/scons_cache:/tmp/scons_cache -v $GITHUB_WORKSPACE/.ci_cache/comma_download_cache:/tmp/comma_download_cache -v $GITHUB_WORKSPACE/.ci_cache/openpilot_cache:/tmp/openpilot_cache $BASE_IMAGE /bin/bash -c

jobs:
  simulator_driving:
    name: simulator driving
    runs-on: ubuntu-latest
    timeout-minutes: 20
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
    - name: Build openpilot
      run: |
        ${{ env.RUN }} "scons -j$(nproc)"
    - name: Run bridge test
      run: |
        ${{ env.RUN }} "source selfdrive/test/setup_xvfb.sh && \
                        source selfdrive/test/setup_vsound.sh && \
                        CI=1 pytest tools/sim/tests/test_metadrive_bridge.py"

  test_compatibility:
    name: test 20.04 + 3.11
    runs-on: ubuntu-20.04
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - name: Installing ubuntu dependencies
      run: INSTALL_EXTRA_PACKAGES=no tools/install_ubuntu_dependencies.sh
    - name: Installing python
      uses: actions/setup-python@v5
      with:
        python-version: '3.11.4'
    - name: Installing pip
      run: pip install pip==24.0
    - name: Installing poetry
      run: pip install poetry==1.7.0
    - name: Installing python dependencies
      run: poetry install --no-cache --no-root
    - name: git LFS
      run: git lfs pull
    - run: echo "CACHE_COMMIT_DATE=$(git log -1 --pretty='format:%cd' --date=format:'%Y-%m-%d-%H:%M')" >> $GITHUB_ENV
    - name: Getting scons cache
      uses: 'actions/cache@v4'
      with:
        path: /tmp/scons_cache
        key: scons-${{ runner.arch }}-ubuntu2004-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}
        restore-keys: |
          scons-${{ runner.arch }}-ubuntu2004-${{ env.CACHE_COMMIT_DATE }}
          scons-${{ runner.arch }}-ubuntu2004
    - name: Building openpilot
      run: poetry run scons -u -j$(nproc)
    - name: Saving scons cache
      uses: actions/cache/save@v4
      if: github.ref == 'refs/heads/master'
      with:
        path: /tmp/scons_cache
        key: scons-${{ runner.arch }}-ubuntu2004-${{ env.CACHE_COMMIT_DATE }}-${{ github.sha }}

  devcontainer:
    name: devcontainer
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
      with:
        submodules: true
    - uses: ./.github/workflows/setup-with-retry
    - name: Use local image for testing devcontainer with latest base image
      run: |
        echo "USE_LOCAL_IMAGE=true" >> "$GITHUB_ENV"
    - name: Setup Dev Container CLI
      run: npm install -g @devcontainers/cli
    - name: Build dev container image
      run: ./scripts/retry.sh devcontainer build --workspace-folder .
    - name: Run dev container
      run: |
        mkdir -p /tmp/devcontainer_scons_cache/
        cp -r $GITHUB_WORKSPACE/.ci_cache/scons_cache/. /tmp/devcontainer_scons_cache/
        devcontainer up --workspace-folder .
    - name: Test environment
      run: |
        devcontainer exec --workspace-folder . scons -j$(nproc) cereal/ common/
        devcontainer exec --workspace-folder . pip install pip-install-test
        devcontainer exec --workspace-folder . touch /home/batman/.comma/auth.json
        devcontainer exec --workspace-folder . sudo touch /root/test.txt


on:
  workflow_dispatch

jobs:
  add_macos_extra_env_to_cache:
    name: Add MacOS x86_64 extra env to cache
    runs-on: macos-13
    steps:
      - name: Install pyenv and python 3.7 and pack to zip file
        run: |
          brew update
          brew install pyenv

          pyenv install 3.7.17
          cd $HOME/.pyenv/versions/ && ln -s 3.7.17 ./3.7
          cd $HOME/.pyenv/versions/ && zip -r -y ${{ github.workspace }}/python_3.7_macos_x86_64.zip *

      - name: Save to cache
        uses: actions/cache/save@v4
        with:
          enableCrossOsArchive: true # to be used on macOS arm64 as well
          path: python_3.7_macos_x86_64.zip
          key: python_3.7_macos_x86_64-${{ github.run_id }} # Can use time based key as well

  add_macos_arm64_extra_env_to_cache:
    name: Add MacOS Arm64 extra env to cache
    runs-on: macos-14
    steps:
      - name: Install pyenv and python versions and pack to zip file
        run: |
          brew update
          brew install pyenv

          pyenv install 3.7.17
          cd $HOME/.pyenv/versions/ && ln -s 3.7.17 ./3.7
          pyenv install 3.8.18
          cd $HOME/.pyenv/versions/ && ln -s 3.8.18 ./3.8
          pyenv install 3.9.19
          cd $HOME/.pyenv/versions/ && ln -s 3.9.19 ./3.9
          pyenv install 3.10.14
          cd $HOME/.pyenv/versions/ && ln -s 3.10.14 ./3.10
          pyenv install 3.11.9
          cd $HOME/.pyenv/versions/ && ln -s 3.11.9 ./3.11
          pyenv install 3.12.3
          cd $HOME/.pyenv/versions/ && ln -s 3.12.3 ./3.12
          cd $HOME/.pyenv/versions/ && zip -r -y ${{ github.workspace }}/python_versions_macos_arm64.zip *

      - name: Save to cache
        uses: actions/cache/save@v4
        with:
          enableCrossOsArchive: true # to be used on macOS x86_64 as well
          path: python_versions_macos_arm64.zip
          key: python_versions_macos_arm64-${{ github.run_id }} # Can use time based key as well