ci-pytest

name: ci-pytest

on:
  push:
    branches:
      - master
  pull_request:
    branches:
      - master
  workflow_dispatch:

env:
  # Updates or changes to this, or the runner OS or arch will invalidate the cache
  python_version: "3.10" # Python version to use for testing - update when needed

jobs:
  test:
    runs-on: ubuntu-20.04
    # Default shell needs to be bash for conda
    # https://github.com/conda-incubator/setup-miniconda?tab=readme-ov-file#important
    defaults:
      run:
        shell: bash -el {0}
    steps:
      - name: Setup Miniconda
        uses: conda-incubator/setup-miniconda@030178870c779d9e5e1b4e563269f3aa69b04081 # v3.0.3, devs recommend using hash
        with:
          miniconda-version: "latest"
          python-version: ${{ env.python_version }}
          auto-update-conda: true
          activate-environment: "test-env"

      - name: Configure Conda to use only .tar.bz2
        run: |
          conda config --set use_only_tar_bz2 true

      - name: Checkout code
        uses: actions/checkout@v4 # Checkout PR code to 'antspy-pr'
        with:
          path: antspy-pr

      - name: Install dependencies and ANTsPy from PR
        run: |
          conda create -n antspy-env python=${{ env.python_version }} -y
          conda activate antspy-env
          conda install -c conda-forge coverage
          conda info
          pip install ./antspy-pr
          conda list

      - name: Run tests
        run: |
          conda activate antspy-env
          bash antspy-pr/tests/run_tests.sh -c

      - name: Coveralls
        uses: coverallsapp/github-action@v2
        with:
          files: antspy-pr/tests/coverage.xml


codecov

on:
  push:
    branches:
      - master

  workflow_dispatch:

name: codecov

jobs:
  test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5

      - name: Install package
        run: python -m pip install .

      - name: Install coverage
        run: python -m pip install coverage

      - name: Unit tests
        run: tests/run_tests.sh -c

      - name: Coveralls
        uses: coverallsapp/github-action@v2.2.3
        with:
          file: tests/coverage.xml


name: build_wheels

on:
  schedule:
    - cron: "0 5 * * *"
  release:
    types: [created]
  workflow_dispatch:

jobs:
  build_wheels:
    name: Build wheel for cp${{ matrix.cibw_python }}-${{ matrix.platform_id }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        include:
          # Have to specify python version twice so that the same python is used to build and test
          # Need to quote decimal versions as string to avoid the "Norway problem"

          # Windows 64-bit
          - os: windows-latest
            python: '3.9'
            cibw_python: 39
            platform_id: win_amd64
          - os: windows-latest
            python: '3.10'
            cibw_python: 310
            platform_id: win_amd64
          - os: windows-latest
            python: '3.11'
            cibw_python: 311
            platform_id: win_amd64
          - os: windows-latest
            python: '3.12'
            cibw_python: 312
            platform_id: win_amd64

          # Linux 64-bit
          - os: ubuntu-latest
            python: '3.8'
            cibw_python: 38
            platform_id: manylinux_x86_64
          - os: ubuntu-latest
            python: '3.9'
            cibw_python: 39
            platform_id: manylinux_x86_64
          - os: ubuntu-latest
            python: '3.10'
            cibw_python: 310
            platform_id: manylinux_x86_64
          - os: ubuntu-latest
            python: '3.11'
            cibw_python: 311
            platform_id: manylinux_x86_64
          - os: ubuntu-latest
            python: '3.12'
            cibw_python: 312
            platform_id: manylinux_x86_64

          # macOS on Intel 64-bit
          - os: macos-12
            python: '3.9'
            cibw_python: 39
            arch: x86_64
            platform_id: macosx_x86_64
          - os: macos-12
            python: '3.10'
            cibw_python: 310
            arch: x86_64
            platform_id: macosx_x86_64
          - os: macos-12
            python: '3.11'
            cibw_python: 311
            arch: x86_64
            platform_id: macosx_x86_64
          - os: macos-12
            python: '3.12'
            cibw_python: 312
            arch: x86_64
            platform_id: macosx_x86_64

          # macOS on Apple M1 64-bit, supported for Python 3.10 and later
          - os: macos-14
            python: '3.10'
            cibw_python: 310
            arch: arm64
            platform_id: macosx_arm64
          - os: macos-14
            python: '3.11'
            cibw_python: 311
            arch: arm64
            platform_id: macosx_arm64
          - os: macos-14
            python: '3.12'
            cibw_python: 312
            arch: arm64
            platform_id: macosx_arm64

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true

      - uses: actions/setup-python@v5
        name: Install Python host for cibuildwheel
        with:
          python-version: ${{ matrix.python }}

        # Visual Studio
      - name: Set up MSVC x64
        if: matrix.platform_id == 'win_amd64'
        uses: ilammy/msvc-dev-cmd@v1

      - name: Install cibuildwheel
        run: python -m pip install cibuildwheel==2.16.2 setuptools toml

      - name: Get package name and version (Linux / Mac)
        if: ${{ ! startsWith(matrix.os, 'windows-') }}
        run: |
          echo "PACKAGE_NAME=$( python -c "import toml; print(toml.load('pyproject.toml')['project']['name'])" )" >> $GITHUB_ENV
          echo "PACKAGE_VERSION=$( python -c "import toml; print(toml.load('pyproject.toml')['project']['version'])" )" >> $GITHUB_ENV

      # Some shells require "-Encoding utf8" to append to GITHUB_ENV
      # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions?tool=powershell#environment-files
      - name: Get package name and version (Windows)
        if: startsWith(matrix.os, 'windows-')
        run: |
          echo "PACKAGE_NAME=$( python -c "import toml; print(toml.load('pyproject.toml')['project']['name'])" )" | Out-File -FilePath $env:GITHUB_ENV ` -Append
          echo "PACKAGE_VERSION=$( python -c "import toml; print(toml.load('pyproject.toml')['project']['version'])" )" | Out-File -FilePath $env:GITHUB_ENV ` -Append

      - name: Determine macOS version
        if: startsWith(matrix.os, 'macos-')
        run: |
          macos_version=$(sw_vers -productVersion | awk -F '.' '{print $1".0"}')
          echo "MACOSX_DEPLOYMENT_TARGET=${macos_version}" >> $GITHUB_ENV

      - name: Build wheels
        env:
          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
          CIBW_MANYLINUX_I686_IMAGE: manylinux2014
          CIBW_BUILD: cp${{ matrix.cibw_python }}-${{ matrix.platform_id }}
          CIBW_TEST_SKIP: "cp*" # We will test during install and test step

          # Include latest Python beta
          CIBW_PRERELEASE_PYTHONS: True

          CIBW_BEFORE_ALL_LINUX: |
            echo "Installing system dependencies with yum"
            yum install -y gcc-c++ libpng-devel libpng
            echo "pip installing cmake and ninja"
            python -m pip install cmake ninja setuptools

          CIBW_BEFORE_ALL_WINDOWS: |
            python -m pip install cmake ninja setuptools

          CIBW_ARCHS_MACOS: ${{ matrix.arch }}

          CIBW_ENVIRONMENT_MACOS: |
            CMAKE_OSX_ARCHITECTURES=${{ matrix.arch }} CMAKE_PREFIX_PATH=/usr/local

          CIBW_BEFORE_ALL_MACOS: |
            python -m pip install cmake ninja setuptools
            brew update
            if ! brew list libpng &>/dev/null; then
                HOMEBREW_NO_AUTO_UPDATE=1 brew install libpng
            fi

        run: python -m cibuildwheel --output-dir wheelhouse/cp${{ matrix.cibw_python }}-${{matrix.platform_id }}

      - name: Install and test (Linux / Mac)
        if: ${{ ! startsWith(matrix.os, 'windows-') }}
        run: |
          python -m pip install wheelhouse/cp${{ matrix.cibw_python }}-${{matrix.platform_id }}/*.whl
          tests/run_tests.sh

      - name: Install and test (Windows)
        if: startsWith(matrix.os, 'windows-')
        shell: bash
        run: |
          python -m pip install --find-links=./wheelhouse/cp${{ matrix.cibw_python }}-${{ matrix.platform_id }} antspyx
          bash tests/run_tests.sh

      - uses: actions/upload-artifact@v4
        with:
          name: ${{ env.PACKAGE_NAME }}-${{ env.PACKAGE_VERSION }}-cp${{ matrix.cibw_python }}-${{ matrix.platform_id }}
          path: ./wheelhouse/cp${{ matrix.cibw_python }}-${{ matrix.platform_id }}/*.whl

      - name: Upload release asset
        # Previously was using actions/upload-release-asset@v1 , but this had some
        # errors with large files
        uses: ncipollo/release-action@v1.11.1
        if: ${{ github.event_name == 'release' }}
        with:
          allowUpdates: true
          omitBodyDuringUpdate: true
          omitNameDuringUpdate: true
          artifacts: ./wheelhouse/cp${{ matrix.cibw_python }}-${{ matrix.platform_id }}/*.whl
          token: ${{ secrets.GITHUB_TOKEN }}

      - name: Publish prep
        if: ${{ (github.event_name == 'release') && (runner.os == 'Linux') }}
        run: |
          mywhl=`find ~/ -name "antspyx*.whl"`
          extrawheeldir=`dirname $mywhl`
          wheeldirx=`dirname $extrawheeldir`
          wheeldir=`dirname $wheeldirx`
          wheeldir=${wheeldir}/dist
          mkdir -p $wheeldir
          echo $wheeldir
          mv $mywhl $wheeldir
          rm -r -f $extrawheeldir $wheeldirx

      - name: Publish package
        uses: pypa/gh-action-pypi-publish@27b31702a0e7fc50959f5ad993c78deac1bdfc29
        if: ${{ (github.event_name == 'release') && (runner.os == 'Linux') }}
        with:
          user: __token__
          password: ${{ secrets.PYPI_API_TOKEN }}
          verify_metadata: false
          skip_existing: true
          verbose: true


name: wheels_faster

on:
  workflow_dispatch:

jobs:
  build_wheels:
    name: Build wheel for cp${{ matrix.cibw_python }}-${{ matrix.platform_id }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        include:
          # Have to specify python version twice so that the same python is used to build and test
          # Need to quote decimal versions as string to avoid the "Norway problem"

          # Windows 64-bit
          - os: windows-latest
            python: '3.11'
            cibw_python: 311
            platform_id: win_amd64

          # Linux 64-bit
          - os: ubuntu-latest
            python: '3.11'
            cibw_python: 311
            platform_id: manylinux_x86_64

          # macOS on Intel 64-bit
          - os: macos-12
            python: '3.11'
            cibw_python: 311
            arch: x86_64
            platform_id: macosx_x86_64

          # macOS on Apple M1 64-bit, supported for Python 3.10 and later
          - os: macos-14
            python: '3.11'
            cibw_python: 311
            arch: arm64
            platform_id: macosx_arm64

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true

      - uses: actions/setup-python@v5
        name: Install Python host for cibuildwheel
        with:
          python-version: ${{ matrix.python }}

        # Visual Studio
      - name: Set up MSVC x64
        if: matrix.platform_id == 'win_amd64'
        uses: ilammy/msvc-dev-cmd@v1

      - name: Install cibuildwheel
        run: python -m pip install cibuildwheel==2.16.2 setuptools toml

      - name: Get package name and version (Linux / Mac)
        if: ${{ ! startsWith(matrix.os, 'windows-') }}
        run: |
          echo "PACKAGE_NAME=$( python -c "import toml; print(toml.load('pyproject.toml')['project']['name'])" )" >> $GITHUB_ENV
          echo "PACKAGE_VERSION=$( python -c "import toml; print(toml.load('pyproject.toml')['project']['version'])" )" >> $GITHUB_ENV

      # Some shells require "-Encoding utf8" to append to GITHUB_ENV
      # https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions?tool=powershell#environment-files
      - name: Get package name and version (Windows)
        if: startsWith(matrix.os, 'windows-')
        run: |
          echo "PACKAGE_NAME=$( python -c "import toml; print(toml.load('pyproject.toml')['project']['name'])" )" | Out-File -FilePath $env:GITHUB_ENV ` -Append
          echo "PACKAGE_VERSION=$( python -c "import toml; print(toml.load('pyproject.toml')['project']['version'])" )" | Out-File -FilePath $env:GITHUB_ENV ` -Append

      - name: Determine macOS version
        if: startsWith(matrix.os, 'macos-')
        run: |
          macos_version=$(sw_vers -productVersion | awk -F '.' '{print $1".0"}')
          echo "MACOSX_DEPLOYMENT_TARGET=${macos_version}" >> $GITHUB_ENV

      - name: Build wheels
        env:
          CIBW_MANYLINUX_X86_64_IMAGE: manylinux2014
          CIBW_MANYLINUX_I686_IMAGE: manylinux2014
          CIBW_BUILD: cp${{ matrix.cibw_python }}-${{ matrix.platform_id }}
          CIBW_TEST_SKIP: "cp*" # We will test during install and test step

          # Include latest Python beta
          CIBW_PRERELEASE_PYTHONS: True

          CIBW_BEFORE_ALL_LINUX: |
            echo "Installing system dependencies with yum"
            yum install -y gcc-c++ libpng-devel libpng
            echo "pip installing cmake and ninja"
            python -m pip install cmake ninja setuptools

          CIBW_BEFORE_ALL_WINDOWS: |
            python -m pip install cmake ninja setuptools

          CIBW_ARCHS_MACOS: ${{ matrix.arch }}

          CIBW_ENVIRONMENT_MACOS: |
            CMAKE_OSX_ARCHITECTURES=${{ matrix.arch }} CMAKE_PREFIX_PATH=/usr/local

          CIBW_BEFORE_ALL_MACOS: |
            python -m pip install cmake ninja setuptools
            brew update
            if ! brew list libpng &>/dev/null; then
                HOMEBREW_NO_AUTO_UPDATE=1 brew install libpng
            fi

        run: python -m cibuildwheel --output-dir wheelhouse/cp${{ matrix.cibw_python }}-${{matrix.platform_id }}

      - name: Install and test (Linux / Mac)
        if: ${{ ! startsWith(matrix.os, 'windows-') }}
        run: |
          python -m pip install wheelhouse/cp${{ matrix.cibw_python }}-${{matrix.platform_id }}/*.whl
          tests/run_tests.sh

      - name: Install and test (Windows)
        if: startsWith(matrix.os, 'windows-')
        shell: bash
        run: |
          python -m pip install --find-links=./wheelhouse/cp${{ matrix.cibw_python }}-${{ matrix.platform_id }} antspyx
          bash tests/run_tests.sh

      - uses: actions/upload-artifact@v4
        with:
          name: ${{ env.PACKAGE_NAME }}-${{ env.PACKAGE_VERSION }}-cp${{ matrix.cibw_python }}-${{ matrix.platform_id }}
          path: ./wheelhouse/cp${{ matrix.cibw_python }}-${{ matrix.platform_id }}/*.whl


on: [push, pull_request, workflow_dispatch]

jobs:
  conan_stable_windows:
    runs-on: windows-latest
    strategy:
      fail-fast: false
      matrix:
        config:
          - { name: "Conan Stable - Python 3.7", python: 3.7, toxenv: "py37-conan-latest", pyver: "py37" }
    steps:
    - uses: actions/checkout@v2
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.config.python }}
    - name: Setup MSBuild.exe
      uses: microsoft/setup-msbuild@v1.1
      with:
        vs-version: 16.11
    - name: Install dependencies
      env:
        PYVER: ${{ matrix.config.pyver }}
      run: |
        .github/workflows/install.bat
    - name: ${{ matrix.config.name }}
      env:
        CONAN_VISUAL_VERSIONS: 17
        PYVER: ${{ matrix.config.pyver }}
      run: |
        .github/workflows/run.bat

  conan_stable_linux:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        config:
          - { name: "Conan Stable - Python 3.7", python: 3.7, toxenv: "py37-conan-latest", pyver: "py37" }
    steps:
    - uses: actions/checkout@v2
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.config.python }}
    - name: Install dependencies
      env:
        PYVER: ${{ matrix.config.pyver }}
      run: |
        chmod +x .github/workflows/install.sh
        .github/workflows/install.sh
    - name: ${{ matrix.config.name }}
      env:
        TOXENV: ${{ matrix.config.toxenv }}
        PYVER: ${{ matrix.config.pyver }}
      run: |
        chmod +x .github/workflows/run.sh
        .github/workflows/run.sh
    - name: Codecov
      run: |
        bash <(curl -s https://codecov.io/bash)

  conan_stable_osx:
    runs-on: macos-latest
    strategy:
      fail-fast: false
      matrix:
        config:
          - { name: "Conan Stable - Python 3.7", python: 3.7, toxenv: "py37-conan-latest", pyver: "py37" }
    steps:
    - uses: actions/checkout@v2
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.config.python }}
    - name: Install dependencies
      env:
        PYVER: ${{ matrix.config.pyver }}
      run: |
        chmod +x .github/workflows/install.sh
        .github/workflows/install.sh
    - name: ${{ matrix.config.name }}
      env:
        TOXENV: ${{ matrix.config.toxenv }}
        PYVER: ${{ matrix.config.pyver }}
      run: |
        chmod +x .github/workflows/run.sh
        .github/workflows/run.sh

  conan_develop_linux:
    if: github.ref != 'refs/head/master' && !startsWith(github.ref, 'refs/heads/release/') && !startsWith(github.ref, 'refs/tags')
    runs-on: ubuntu-latest
    continue-on-error: True
    strategy:
      fail-fast: false
      matrix:
        config:
          - { name: "Conan Develop - Python 3.7", python: 3.7, toxenv: "py37-conan-dev", pyver: "py37" }
    steps:
    - uses: actions/checkout@v2
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.config.python }}
    - name: Install dependencies
      env:
        PYVER: ${{ matrix.config.pyver }}
      run: |
        export
        env
        chmod +x .github/workflows/install.sh
        .github/workflows/install.sh
    - name: ${{ matrix.config.name }}
      env:
        TOXENV: ${{ matrix.config.toxenv }}
        PYVER: ${{ matrix.config.pyver }}
      run: |
        chmod +x .github/workflows/run.sh
        .github/workflows/run.sh

  conan_develop_osx:
    if: github.ref != 'refs/head/master' && !startsWith(github.ref, 'refs/heads/release/') && !startsWith(github.ref, 'refs/tags')
    runs-on: macos-latest
    continue-on-error: True
    strategy:
      fail-fast: false
      matrix:
        config:
          - { name: "Conan Develop - Python 3.7", python: 3.7, toxenv: "py37-conan-dev", pyver: "py37" }
    steps:
    - uses: actions/checkout@v2
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.config.python }}
    - name: Install dependencies
      env:
        PYVER: ${{ matrix.config.pyver }}
      run: |
        export
        env
        chmod +x .github/workflows/install.sh
        .github/workflows/install.sh
    - name: ${{ matrix.config.name }}
      env:
        TOXENV: ${{ matrix.config.toxenv }}
        PYVER: ${{ matrix.config.pyver }}
      run: |
        chmod +x .github/workflows/run.sh
        .github/workflows/run.sh

  deploy_release:
    if: startsWith(github.ref, 'refs/tags')
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v2
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: 3.7
    - name: Install dependencies
      run: pip install pygithub
    - name: Generate changelog
      run: python .github/deploy/release_changelog.py
    - name: Generate Dist/
      run: python setup.py sdist
    - uses: pypa/gh-action-pypi-publish@release/v1
      with:
        user: __token__
        password: ${{ secrets.PYPI_API_TOKEN }}


Changelog

name: Changelog

on:
  pull_request:
    # should also be re-run when changing labels
    types: [opened, reopened, labeled, unlabeled, synchronize]

env:
  FRAGMENT_NAME: "docs/changes/${{ github.event.number }}.*.rst"

jobs:
  changelog:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Check for news fragment
        if: ${{ ! contains( github.event.pull_request.labels.*.name, 'no-changelog-needed')}}
        uses: andstor/file-existence-action@v3
        with:
          files: ${{ env.FRAGMENT_NAME }}
          fail: true


CI

name: CI

on:
  push:
    branches:
      - main
    tags:
      - "**"
  pull_request:

env:
  NUMBA_NUM_THREADS: 1
  OMP_NUM_THREADS: 1
  MKL_NUM_THREADS: 1
  MPLBACKEND: Agg
  PYTEST_ADDOPTS: --color=yes
  GITHUB_PR_NUMBER: ${{ github.event.number }}
  SONAR_HOST_URL: https://sonar-cta-dpps.zeuthen.desy.de

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - uses: actions/setup-python@v5
        with:
          python-version: "3.10"
      - name: Check README syntax
        run: |
          pip install restructuredtext-lint
          restructuredtext-lint README.rst
      - uses: pre-commit/action@v3.0.1
        with:
          extra_args: --files $(git diff origin/main --name-only)
  tests:
    runs-on: ${{ matrix.os }}
    needs: lint
    strategy:
      matrix:
        include:
          - os: ubuntu-latest
            python-version: "3.10"
            install-method: mamba

          - os: ubuntu-latest
            python-version: "3.10"
            install-method: pip

          - os: ubuntu-latest
            python-version: "3.11"
            install-method: mamba

          - os: ubuntu-latest
            python-version: "3.12"
            install-method: mamba
            extra-args: ["codecov"]

          - os: ubuntu-latest
            python-version: "3.12"
            install-method: pip

          # macos 13 image is x86-based
          - os: macos-13
            python-version: "3.10"
            install-method: pip

          # macos 14 image is arm64 based
          - os: macos-14
            python-version: "3.12"
            install-method: mamba

    defaults:
      run:
        # We need login shells (-l) for micromamba to work.
        shell: bash -leo pipefail {0}

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: cache test data
        uses: actions/cache@v4
        with:
          path: ~/.cache/ctapipe
          key: ctapipe-test-data

      - name: Prepare mamba installation
        if: matrix.install-method == 'mamba' &&  contains(github.event.pull_request.labels.*.name, 'documentation-only') == false
        env:
          PYTHON_VERSION: ${{ matrix.python-version }}
        run: |
          # setup correct python version
          sed -i -e "s/- python=.*/- python=$PYTHON_VERSION/g" environment.yml

      - name: mamba setup
        if: matrix.install-method == 'mamba' && contains(github.event.pull_request.labels.*.name, 'documentation-only') == false
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: environment.yml
          cache-downloads: true

      - name: Python setup
        if: matrix.install-method == 'pip' && contains(github.event.pull_request.labels.*.name, 'documentation-only') == false
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          check-latest: true

      - if: matrix.install-method == 'pip' && runner.os == 'macOS' && contains(github.event.pull_request.labels.*.name, 'documentation-only') == false
        name: Fix Python PATH on macOS
        # See https://github.com/actions/setup-python/issues/132 and
        # https://github.com/actions/setup-python/issues/132#issuecomment-779406058
        # Login shells on macOS prepend system paths, so we need to
        # prepend the python path from actions/setup-python.
        # Micromamba sets up ~/.bash_profile, where we need to set the path now.
        run: |
          tee -a ~/.bash_profile <<<'export PATH="$pythonLocation/bin:$PATH"'

      - name: Install dependencies
        if: contains(github.event.pull_request.labels.*.name, 'documentation-only') == false
        env:
          PYTHON_VERSION: ${{ matrix.python-version }}
        run: |
          python --version | grep "Python ${PYTHON_VERSION}"
          pip install -e .[tests]
          pip install ./test_plugin
          pip freeze
          pip list

      - name: ctapipe-info
        if: contains(github.event.pull_request.labels.*.name, 'documentation-only') == false
        run: |
          ctapipe-info --all

      - name: Tests
        if: contains(github.event.pull_request.labels.*.name, 'documentation-only') == false
        run: |
          pytest -n auto --dist loadscope \
            --cov --cov-report=xml \
            --doctest-modules --doctest-glob='*.rst' \
            --ignore=docs/conf.py \
            src/ctapipe docs

          ctapipe-info --version

      # check code quality and coverage with SonarScanner
      - uses: SonarSource/sonarqube-scan-action@v2
        if: contains(matrix.extra-args, 'codecov') && contains(github.event.pull_request.labels.*.name, 'documentation-only') == false
        env:
          SONAR_TOKEN: ${{ secrets.SONAR_TOKEN }}

  docs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"

      - name: Install doc dependencies
        run: |
          sudo apt update --yes && sudo apt install --yes git build-essential pandoc graphviz ffmpeg
          pip install -U pip towncrier setuptools
          pip install -e .[docs]
          pip install ./test_plugin
          git describe --tags
          python -c 'import ctapipe; print(ctapipe.__version__)'

      - name: Produce Changelog
        run: |
          towncrier build --yes

      - name: Build docs
        run: make doc


Deploy to PyPi

name: Deploy to PyPi

on:
  push:
    tags:
      - 'v*'

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      # make sure we have version info
      - run: git fetch --tags

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Install dependencies
        run: |
          python --version
          pip install -U build
          python -m build

      - name: Publish package
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          user: __token__
          password: ${{ secrets.pypi_password }}


Release Drafter

name: Release Drafter

on:
  push:
    # branches to consider in the event; optional, defaults to all
    branches:
      - main

jobs:
  update_release_draft:
    runs-on: ubuntu-latest
    steps:
      # Drafts your next Release notes as Pull Requests are merged into "master"
      - uses: release-drafter/release-drafter@v6
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


build

name: build
on:
  push:
  pull_request:
  # build weekly at 4:00 AM UTC
  schedule:
    - cron: '0 4 * * 1'

jobs:
  lint:
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
    name: "lint on ${{ matrix.os }} "
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
      - name: Identify week number
        shell: bash
        run: |
          date +'%V' > week-number.txt
          date +'week-number=%V' >> $GITHUB_ENV
      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
        id: setup-python
        with:
          python-version: "3.12"
          cache: "pip"
          cache-dependency-path: |
            .github/workflows/build.yaml
            setup.cfg
            setup.py
            tox.ini
            week-number.txt
      - name: Restore cache
        id: restore-cache
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
        with:
          path: |
            .tox/
            .venv/
          key: >
            lint
            week=${{ env.week-number }}
            os=${{ matrix.os }}
            python=${{ steps.setup-python.outputs.python-version }}
            hash=${{ hashFiles('.github/workflows/build.yaml', 'tox.ini', 'setup.cfg', 'setup.py') }}
      - name: Identify venv path
        shell: bash
        run: echo 'venv-path=${{ runner.os == 'Windows' && '.venv/Scripts' || '.venv/bin' }}' >> $GITHUB_ENV
      - name: install tox
        if: steps.restore-cache.outputs.cache-hit == false
        run: |
          python -m venv .venv
          ${{ env.venv-path }}/pip install --upgrade pip setuptools wheel
          ${{ env.venv-path }}/pip install tox
      - name: run mypy
        run: ${{ env.venv-path }}/tox -e mypy

  test:
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
        # we do not want a large number of windows and macos builds, so
        # enumerate them explicitly
        include:
          - os: windows-latest
            python-version: "3.12"
          - os: macos-latest
            python-version: "3.12"
    name: "test py${{ matrix.python-version }} on ${{ matrix.os }}"
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
      - name: Identify week number
        shell: bash
        run: |
          date +'%V' > week-number.txt
          date +'week-number=%V' >> $GITHUB_ENV
      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
        id: setup-python
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"
          cache-dependency-path: |
            .github/workflows/build.yaml
            setup.cfg
            setup.py
            tox.ini
            week-number.txt
      - name: Restore cache
        id: restore-cache
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
        with:
          path: |
            .tox
            .venv
          key: >
            test
            week=${{ env.week-number }}
            os=${{ matrix.os }}
            python=${{ steps.setup-python.outputs.python-version }}
            hash=${{ hashFiles('.github/workflows/build.yaml', 'setup.cfg', 'setup.py', 'tox.ini') }}
      - name: Identify venv path
        shell: bash
        run: echo 'venv-path=${{ runner.os == 'Windows' && '.venv/Scripts' || '.venv/bin' }}' >> $GITHUB_ENV
      - name: install tox
        if: steps.restore-cache.outputs.cache-hit == false
        run: |
          python -m venv .venv
          ${{ env.venv-path }}/pip install --upgrade pip setuptools wheel
          ${{ env.venv-path }}/pip install tox
      - name: run tests
        run: ${{ env.venv-path }}/tox -e py

  docs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
      - name: Identify week number
        shell: bash
        run: |
          date +'%V' > week-number.txt
          date +'week-number=%V' >> $GITHUB_ENV
      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
        id: setup-python
        with:
          python-version: "3.12"
          cache: "pip"
          cache-dependency-path: |
            .github/workflows/build.yaml
            setup.cfg
            setup.py
            tox.ini
            week-number.txt
      - name: Restore cache
        id: restore-cache
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
        with:
          path: |
            .tox
            .venv
          key: >
            docs
            week=${{ env.week-number }}
            python=${{ steps.setup-python.outputs.python-version }}
            hash=${{ hashFiles('.github/workflows/build.yaml', 'setup.cfg', 'setup.py', 'tox.ini') }}
      - name: install tox
        if: steps.restore-cache.outputs.cache-hit == false
        run: |
          python -m venv .venv
          .venv/bin/pip install --upgrade pip setuptools wheel
          .venv/bin/pip install tox
      - name: test reference docs generation
        run: .venv/bin/tox -e reference

  test-package-metadata:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
      - name: Identify week number
        shell: bash
        run: |
          date +'%V' > week-number.txt
          date +'week-number=%V' >> $GITHUB_ENV
      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
        id: setup-python
        with:
          python-version: "3.12"
          cache: "pip"
          cache-dependency-path: |
            .github/workflows/build.yaml
            setup.cfg
            setup.py
            tox.ini
            week-number.txt
      - name: Restore cache
        id: restore-cache
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
        with:
          path: |
            .tox
            .venv
          key: >
            test-package-metadata
            week=${{ env.week-number }}
            python=${{ steps.setup-python.outputs.python-version }}
            hash=${{ hashFiles('.github/workflows/build.yaml', 'setup.cfg', 'setup.py', 'tox.ini') }}
      - name: install tox
        if: steps.restore-cache.outputs.cache-hit == false
        run: |
          python -m venv .venv
          .venv/bin/pip install --upgrade pip setuptools wheel
          .venv/bin/pip install tox
      - name: check package metadata
        run: .venv/bin/tox -e twine-check

  test-mindeps:
    runs-on: ubuntu-latest
    name: "mindeps"
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
      - name: Identify week number
        shell: bash
        run: |
          date +'%V' > week-number.txt
          date +'week-number=%V' >> $GITHUB_ENV
      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
        id: setup-python
        with:
          python-version: "3.8"
          cache: "pip"
          cache-dependency-path: |
            .github/workflows/build.yaml
            setup.cfg
            setup.py
            tox.ini
            week-number.txt
      - name: Restore cache
        id: restore-cache
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
        with:
          path: |
            .tox
            .venv
          key: >
            test-mindeps
            week=${{ env.week-number }}
            python=${{ steps.setup-python.outputs.python-version }}
            hash=${{ hashFiles('.github/workflows/build.yaml', 'setup.cfg', 'setup.py', 'tox.ini') }}
      - name: install tox
        if: steps.restore-cache.outputs.cache-hit == false
        run: |
          python -m venv .venv
          .venv/bin/pip install --upgrade pip setuptools wheel
          .venv/bin/pip install tox
      - name: test
        run: .venv/bin/tox -e py-mindeps

  test-sdk-main:
    strategy:
      matrix:
        python-version: ["3.8", "3.12"]
    runs-on: ubuntu-latest
    name: "sdk-main"
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
      - name: Identify week number
        shell: bash
        run: |
          date +'%V' > week-number.txt
          date +'week-number=%V' >> $GITHUB_ENV
      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
        id: setup-python
        with:
          python-version: ${{ matrix.python-version }}
          cache: "pip"
          cache-dependency-path: |
            .github/workflows/build.yaml
            setup.cfg
            setup.py
            tox.ini
            week-number.txt
      - name: Restore cache
        id: restore-cache
        uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
        with:
          path: |
            .tox
            .venv
          key: >
            test-sdk-main
            week=${{ env.week-number }}
            python=${{ steps.setup-python.outputs.python-version }}
            hash=${{ hashFiles('.github/workflows/build.yaml', 'setup.cfg', 'setup.py', 'tox.ini') }}
      - name: install tox
        if: steps.restore-cache.outputs.cache-hit == false
        run: |
          python -m venv .venv
          .venv/bin/pip install --upgrade pip setuptools wheel
          .venv/bin/pip install tox
      - run: .venv/bin/tox -e py-sdkmain


Build Docs

# build and release new docs as a release asset
name: Build Docs
on:
  release:
    types: [created]
jobs:
  doc:
    name: Build Docs
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
      - name: Setup Python
        uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
        with:
          python-version: "3.12"
      - name: Install CLI
        run: |
          python -m pip install -e .
      - name: Generate Autodoc
        run: |
          ./reference/_generate.py --debug
          # bundle as tarball without the _generate.py script or .gitignore
          # use `-h` to dereference the changelog link
          tar --exclude "*.py" --exclude '.gitignore' -czf cli-reference.tar.gz -h reference/
      # upload as a release asset
      - name: Upload Autodoc
        env:
          GH_TOKEN: ${{ github.token }}
          GH_REPO: ${{ github.repository }}
        run: gh release upload "${{ github.ref_name }}" cli-reference.tar.gz


name: has_changelog
on:
  pull_request:
    types:
      - labeled
      - unlabeled
      - opened
      - reopened
      - synchronize

jobs:
  check_has_news_in_changelog_dir:
    if: |
      ! (
        contains(github.event.pull_request.labels.*.name, 'no-news-is-good-news') ||
        github.event.pull_request.user.login == 'pre-commit-ci[bot]' ||
        github.event.pull_request.user.login == 'dependabot[bot]'
      )
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
        with:  # do a deep fetch to allow merge-base and diff
          fetch-depth: 0
      - name: check PR adds a news file
        run: |
          news_files="$(git diff --name-only "$(git merge-base origin/main "$GITHUB_SHA")" "$GITHUB_SHA" -- changelog.d/*.md)"
          if [ -n "$news_files" ]; then
            echo "Saw new files. changelog.d:"
            echo "$news_files"
          else
            echo "No news files seen"
            exit 1
          fi


Publish PyPI Release

name: Publish PyPI Release

on:
  release:
    types: [published]

jobs:
  build-dists:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
        with:
          python-version: "3.12"

      - run: python -m pip install build

      - name: Build Dists
        run: python -m build .

      - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
        with:
          name: packages
          path: dist/*

  publish_pypi:
    needs: [build-dists]
    runs-on: ubuntu-latest
    environment: publish-pypi
    permissions:
      id-token: write

    steps:
      - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
        with:
          name: packages
          path: dist

      - name: Publish to PyPI
        uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14


Publish Test PyPI Release

name: Publish Test PyPI Release

on:
  push:
    tags: ["*"]

jobs:
  build-dists:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
      - uses: actions/setup-python@82c7e631bb3cdc910f68e0081d67478d79c6982d # v5.1.0
        with:
          python-version: "3.12"

      - run: python -m pip install build

      - name: Set dev version prior to upload
        run: python ./scripts/set_dev_version.py

      - name: Build Dists
        run: python -m build .

      - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
        with:
          name: packages
          path: dist/*

  publish:
    needs: [build-dists]
    runs-on: ubuntu-latest
    environment: publish-test-pypi
    permissions:
      id-token: write

    steps:
      - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
        with:
          name: packages
          path: dist

      - name: Publish to TestPyPI
        uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14
        with:
          repository-url: https://test.pypi.org/legacy/


anisble-lint-workflow

name: anisble-lint-workflow
on:
  workflow_dispatch:
  pull_request:
    branches:
      - master

env:
  # Global env vars used on all jobs
  REPO_URL: 'https://github.com/rcbops/rpc-maas'
  RE_JOB_BRANCH: ${{ github.event.pull_request.head.sha }}
  RE_JOB_ACTION: 'tox-test'

# Per-Job env var info:
#   RE_JOB_SCENARIO: 'Linting scenarios used by rpc-maas test scripts. Values(ansible-lint,ansible-syntax,bandit,bashate,docs,pep8,releasenotes)'

jobs:

  ansible-lint:
    runs-on: ubuntu-latest
    steps:

    - name: Cancel Previous Runs
      uses: styfle/cancel-workflow-action@0.5.0
      with:
        access_token: ${{ github.token }}

    - name: Check out the repo on the runner's workspace
      uses: actions/checkout@v2
      with:
        ref: ${{ github.event.pull_request.head.sha }}

    - name: Run the tox test for this scenario
      run: ./gating/github_action_scripts/linting_run.sh
      env:
        RE_JOB_SCENARIO: "ansible-lint"


anisble-syntax-workflow

name: anisble-syntax-workflow
on:
  workflow_dispatch:
  pull_request:
    branches:
      - master

env:
  # Global env vars used on all jobs
  REPO_URL: 'https://github.com/rcbops/rpc-maas'
  RE_JOB_BRANCH: ${{ github.event.pull_request.head.sha }}
  RE_JOB_ACTION: 'tox-test'

# Per-Job env var info:
#   RE_JOB_SCENARIO: 'Linting scenarios used by rpc-maas test scripts. Values(ansible-lint,ansible-syntax,bandit,bashate,docs,pep8,releasenotes)'

jobs:

  ansible-syntax:
    runs-on: ubuntu-latest
    steps:

    - name: Cancel Previous Runs
      uses: styfle/cancel-workflow-action@0.5.0
      with:
        access_token: ${{ github.token }}

    - name: Check out the repo on the runner's workspace
      uses: actions/checkout@v2
      with:
        ref: ${{ github.event.pull_request.head.sha }}

    - name: Run the tox test for this scenario
      run: ./gating/github_action_scripts/linting_run.sh
      env:
        RE_JOB_SCENARIO: "ansible-syntax"


bandit-workflow

name: bandit-workflow
on:
  workflow_dispatch:
  pull_request:
    branches:
      - master

env:
  # Global env vars used on all jobs
  REPO_URL: 'https://github.com/rcbops/rpc-maas'
  RE_JOB_BRANCH: ${{ github.event.pull_request.head.sha }}
  RE_JOB_ACTION: 'tox-test'

# Per-Job env var info:
#   RE_JOB_SCENARIO: 'Linting scenarios used by rpc-maas test scripts. Values(ansible-lint,ansible-syntax,bandit,bashate,docs,pep8,releasenotes)'

jobs:

  bandit:
    runs-on: ubuntu-latest
    steps:

    - name: Cancel Previous Runs
      uses: styfle/cancel-workflow-action@0.5.0
      with:
        access_token: ${{ github.token }}

    - name: Check out the repo on the runner's workspace
      uses: actions/checkout@v2
      with:
        ref: ${{ github.event.pull_request.head.sha }}

    - name: Run the tox test for this scenario
      run: ./gating/github_action_scripts/linting_run.sh
      env:
        RE_JOB_SCENARIO: "bandit"


bashate-workflow

name: bashate-workflow
on:
  workflow_dispatch:
  pull_request:
    branches:
      - master

env:
  # Global env vars used on all jobs
  REPO_URL: 'https://github.com/rcbops/rpc-maas'
  RE_JOB_BRANCH: ${{ github.event.pull_request.head.sha }}
  RE_JOB_ACTION: 'tox-test'

# Per-Job env var info:
#   RE_JOB_SCENARIO: 'Linting scenarios used by rpc-maas test scripts. Values(ansible-lint,ansible-syntax,bandit,bashate,docs,pep8,releasenotes)'

jobs:

  bashate:
    runs-on: ubuntu-latest
    steps:

    - name: Cancel Previous Runs
      uses: styfle/cancel-workflow-action@0.5.0
      with:
        access_token: ${{ github.token }}

    - name: Check out the repo on the runner's workspace
      uses: actions/checkout@v2
      with:
        ref: ${{ github.event.pull_request.head.sha }}

    - name: Run the tox test for this scenario
      run: ./gating/github_action_scripts/linting_run.sh
      env:
        RE_JOB_SCENARIO: "bashate"


docs-workflow

name: docs-workflow
on:
  workflow_dispatch:
  pull_request:
    branches:
      - master

env:
  # Global env vars used on all jobs
  REPO_URL: 'https://github.com/rcbops/rpc-maas'
  RE_JOB_BRANCH: ${{ github.event.pull_request.head.sha }}
  RE_JOB_ACTION: 'tox-test'

# Per-Job env var info:
#   RE_JOB_SCENARIO: 'Linting scenarios used by rpc-maas test scripts. Values(ansible-lint,ansible-syntax,bandit,bashate,docs,pep8,releasenotes)'

jobs:

  docs:
    runs-on: ubuntu-latest
    steps:

    - name: Cancel Previous Runs
      uses: styfle/cancel-workflow-action@0.5.0
      with:
        access_token: ${{ github.token }}

    - name: Check out the repo on the runner's workspace
      uses: actions/checkout@v2
      with:
        ref: ${{ github.event.pull_request.head.sha }}

    - name: Run the tox test for this scenario
      run: ./gating/github_action_scripts/linting_run.sh
      env:
        RE_JOB_SCENARIO: "docs"


name: pep8-workflow
on:
  workflow_dispatch:
  pull_request:
    branches:
      - master

env:
  # Global env vars used on all jobs
  REPO_URL: 'https://github.com/rcbops/rpc-maas'
  RE_JOB_BRANCH: ${{ github.event.pull_request.head.sha }}
  RE_JOB_ACTION: 'tox-test'

# Per-Job env var info:
#   RE_JOB_SCENARIO: 'Linting scenarios used by rpc-maas test scripts. Values(ansible-lint,ansible-syntax,bandit,bashate,docs,pep8,releasenotes)'

jobs:

  pep8:
    runs-on: ubuntu-latest
    steps:

    - name: Cancel Previous Runs
      uses: styfle/cancel-workflow-action@0.5.0
      with:
        access_token: ${{ github.token }}

    - name: Check out the repo on the runner's workspace
      uses: actions/checkout@v2
      with:
        ref: ${{ github.event.pull_request.head.sha }}

    - name: Run the tox test for this scenario
      run: ./gating/github_action_scripts/linting_run.sh
      env:
        RE_JOB_SCENARIO: "pep8"


releasenotes-workflow

name: releasenotes-workflow
on:
  workflow_dispatch:
  pull_request:
    branches:
      - master

env:
  # Global env vars used on all jobs
  REPO_URL: 'https://github.com/rcbops/rpc-maas'
  RE_JOB_BRANCH: ${{ github.event.pull_request.head.sha }}
  RE_JOB_ACTION: 'tox-test'

# Per-Job env var info:
#   RE_JOB_SCENARIO: 'Linting scenarios used by rpc-maas test scripts. Values(ansible-lint,ansible-syntax,bandit,bashate,docs,pep8,releasenotes)'

jobs:

  releasenotes:
    runs-on: ubuntu-latest
    steps:

    - name: Cancel Previous Runs
      uses: styfle/cancel-workflow-action@0.5.0
      with:
        access_token: ${{ github.token }}

    - name: Check out the repo on the runner's workspace
      uses: actions/checkout@v2
      with:
        ref: ${{ github.event.pull_request.head.sha }}

    - name: Run the tox test for this scenario
      run: ./gating/github_action_scripts/linting_run.sh
      env:
        RE_JOB_SCENARIO: "releasenotes"


Platform Build

name: Platform Build

on:
  workflow_dispatch:
    inputs:
      docker_push:
        description: 'Push the docker image to dockerhub and outout reference'
        required: true
        default: false
        type: boolean
      ignore_unfixed:
        description: 'Include unfixed vulnerabilities in scan [true,false]'
        required: true
        default: true
        type: boolean
      cve_severity:
        description: 'Severities of vulnerabilities to scanned for [LOW, MEDIUM, HIGH, CRITICAL, SKIP]'
        required: false
        default: 'CRITICAL,HIGH'
      oasislmf_branch:
        description: 'If set, pip install oasislmf branch [git ref]'
        required: false
        type: string
      ods_branch:
        description: 'If set, pip install ods-tools branch [git ref]'
        required: false
        type: string

  workflow_call:
    inputs:
      docker_push:
        description: 'Push the docker image to dockerhub and outout reference'
        required: true
        default: 'false'
        type: string
      ignore_unfixed:
        description: 'Include unfixed vulnerabilities in scan [true,false]'
        required: true
        default: 'true'
        type: string
      cve_severity:
        description: 'Severities of vulnerabilities to scanned for, fails build if any found'
        required: false
        default: 'CRITICAL,HIGH'
        type: string
      oasislmf_branch:
        description: 'If set, pip install oasislmf branch [git ref]'
        required: false
        type: string
      ods_branch:
        description: 'If set, pip install ods-tools branch [git ref]'
        required: false
        type: string

    outputs:
      server_image:
        description:
        value:  'coreoasis/github-actions:api_server-${{ github.sha }}'
      worker_image:
        description:
        value:  'coreoasis/github-actions:model_worker-${{ github.sha }}'
      worker_deb_image:
        description:
        value:  'coreoasis/github-actions:model_worker_deb-${{ github.sha }}'
      worker_controller_image:
        description:
        value:  'coreoasis/github-actions:worker_controller-${{ github.sha }}'

env:
  # inputs
  DOCKER_PUSH: 'false'
  IGNORE_UNFIXED: 'true'
  SEVERITY: 'CRITICAL,HIGH'
  IMAGE_REPO: 'coreoasis/github-actions'

jobs:
  build:
    name: Docker build
    runs-on: ubuntu-latest
    strategy:
      matrix:
        image: [worker, worker_deb, server, worker_controller]
        include:

        - image: server
          tag: 'api_server-${{ github.sha }}'
          file: 'Dockerfile.api_server'
          report: 'server-scan.sarif'
          dive: 'server-layers.txt'
          exit-code: '1'
          context: '.'

        - image: worker
          tag: 'model_worker-${{ github.sha }}'
          file: 'Dockerfile.model_worker'
          report: 'worker-scan.sarif'
          dive: 'worker-layers.txt'
          exit-code: '1'
          context: '.'

        - image: worker_deb
          tag:  'model_worker_deb-${{ github.sha }}'
          file: 'Dockerfile.model_worker_debian'
          report: 'worker-deb-scan.sarif'
          dive: 'worker-deb-layers.txt'
          exit-code: '0' # scan but don't fail
          context: '.'

        - image: worker_controller
          tag:  'worker_controller-${{ github.sha }}'
          file: 'kubernetes/worker-controller/Dockerfile'
          report: 'worker-controller-scan.sarif'
          dive: 'worker-controller-layers.txt'
          exit-code: '1'
          context: 'kubernetes/worker-controller/'

    steps:
    - name: Set inputs ${{ github.event_name }}
      run: |
         [[ -z "${{ inputs.cve_severity }}" ]] || echo "SEVERITY=${{ inputs.cve_severity }}" >> $GITHUB_ENV
         [[ -z "${{ inputs.docker_push }}" ]] || echo "DOCKER_PUSH=${{ inputs.docker_push }}" >> $GITHUB_ENV
         [[ -z "${{ inputs.ignore_unfixed }}" ]] || echo "IGNORE_UNFIXED=${{ inputs.ignore_unfixed }}" >> $GITHUB_ENV

    - name: Github context
      run:   echo "$GITHUB_CONTEXT"
      shell: bash
      env:
       GITHUB_CONTEXT: ${{ toJson(github) }}

    - uses: actions/checkout@v3
      with:
        ref: ${{ github.ref }}

    - name: Login to Docker Hub
      if: env.DOCKER_PUSH == 'true'
      uses: docker/login-action@v2
      with:
        username: ${{ secrets.DOCKERHUB_USERNAME }}
        password: ${{ secrets.DOCKERHUB_TOKEN }}

    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v2

    - name: Docker Build ${{ matrix.image }}
      uses: docker/build-push-action@v3
      with:
        file: ${{ matrix.file }}
        context: ${{ matrix.context }}
        push: ${{ env.DOCKER_PUSH }}
        tags: ${{ env.IMAGE_REPO }}:${{ matrix.tag }}
        build-args: |
          oasislmf_branch=${{ inputs.oasislmf_branch }}
          ods_tools_branch=${{ inputs.ods_branch }}

    - name: Test image efficiency
      if: env.SEVERITY != 'SKIP'
      run: |
        ./scripts/imagesize.sh ${{ env.IMAGE_REPO }}:${{ matrix.tag }} ${{ matrix.dive }}
      env:
        DIVE_HIGHEST_USER_WASTED: '0.20'
        DIVE_HIGHEST_WASTED_BYTES: '50mb'
        DIVE_LOWSET_EFFICIENCY: '0.95'

    - name: Store image size report
      if: success() || failure()
      uses: actions/upload-artifact@v3
      with:
        name: ${{ matrix.dive }}
        path: ${{ matrix.dive }}
        retention-days: 3

    - name: Vulnerability scanner
      if: env.SEVERITY != 'SKIP'
      uses: aquasecurity/trivy-action@master
      with:
        image-ref: ${{ env.IMAGE_REPO }}:${{ matrix.tag }}
        scan-type: 'image'
        exit-code: ${{ matrix.exit-code }}
        ignore-unfixed: ${{ env.IGNORE_UNFIXED }}
        severity: ${{ env.SEVERITY }}
        scanners: 'vuln'

    - name: Create Report
      if: ( success() || failure() ) && env.SEVERITY != 'SKIP' && matrix.exit-code == '1'
      uses: aquasecurity/trivy-action@master
      with:
        image-ref: ${{ env.IMAGE_REPO }}:${{ matrix.tag }}
        format: 'sarif'
        output: ${{ matrix.report }}
        scan-type: 'image'
        exit-code: '0'
        ignore-unfixed: ${{ env.IGNORE_UNFIXED }}
        severity: ${{ env.SEVERITY }}
        scanners: 'vuln'

    - name: Upload scan results to Security tab
      if: ( success() || failure() ) && env.SEVERITY != 'SKIP' && matrix.exit-code == '1'
      uses: github/codeql-action/upload-sarif@v2
      with:
        sarif_file:  ${{ matrix.report }}

    - name: Store CVE report
      if: success() || failure() && env.SEVERITY != 'SKIP'
      uses: actions/upload-artifact@v3
      with:
        name: ${{ matrix.report }}
        path: ${{ matrix.report }}
        retention-days: 3

    - name: Docker summary
      if: env.DOCKER_PUSH == 'true'
      run: |
        echo "#### Docker Image: " >> $GITHUB_STEP_SUMMARY
        echo "* [${{ env.IMAGE_REPO }}:${{ matrix.tag }}](https://hub.docker.com/r/${{ env.IMAGE_REPO }}/tags?name=${{ matrix.tag }})" >> $GITHUB_STEP_SUMMARY


Platform Schema (OpenAPI)

name: Platform Schema (OpenAPI)

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
      - stable**
  workflow_dispatch:
    inputs:
      ods_branch:
        description: 'If set, pip install ods-tools branch [git ref]'
        required: false
  workflow_call:
    inputs:
      ods_branch:
        description: 'If set, pip install ods-tools branch [git ref]'
        required: false
        type: string

jobs:
  ods_tools:
    if: ${{ github.event_name != 'workflow_dispatch' }}
    uses: OasisLMF/ODS_Tools/.github/workflows/build.yml@main
    secrets: inherit
    with:
      ods_branch: ${{ github.event_name != 'workflow_dispatch' && 'main' || inputs.ods_branch }}

  schema:
    if: ${{ ! failure() || ! cancelled() }}
    needs: ods_tools
    env:
      SCHEMA_ALL: 'reports/openapi-schema.json'
      SCHEMA_V1: 'reports/v1-openapi-schema.json'
      SCHEMA_V2: 'reports/v2-openapi-schema.json'
    runs-on: ubuntu-22.04
    steps:
    - uses: actions/checkout@v3

    - name: Set up Python 3.10
      uses: actions/setup-python@v4
      with:
        python-version: '3.10'

    - name: Install requirments
      run: pip install -r requirements-server.txt

    - name: Download package
      if: needs.ods_tools.outputs.whl_filename != ''
      uses: actions/download-artifact@v3
      with:
        name: bin_package
        path: ${{ github.workspace }}/

    - name: Install package
      if: needs.ods_tools.outputs.whl_filename != ''
      run: |
        pip uninstall ods-tools -y
        pip install ${{ needs.ods_tools.outputs.whl_filename }}

    - name: Generate OpenAPI
      run: |
        test -d $(dirname ${{ env.SCHEMA_ALL }}) || mkdir -p $(dirname ${{ env.SCHEMA_ALL }})
        python ./manage.py migrate
        python ./manage.py generate_swagger ${{ env.SCHEMA_ALL }}

    - name: Generate OpenAPI (only v1)
      run: |
        test -d $(dirname ${{ env.SCHEMA_V1 }}) || mkdir -p $(dirname ${{ env.SCHEMA_V1 }})
        python ./manage.py migrate
        python ./manage.py generate_swagger ${{ env.SCHEMA_V1 }}
      env:
        OASIS_GEN_SWAGGER_V1: 1

    - name: Generate OpenAPI (only v2)
      run: |
        test -d $(dirname ${{ env.SCHEMA_V2 }}) || mkdir -p $(dirname ${{ env.SCHEMA_V2 }})
        python ./manage.py migrate
        python ./manage.py generate_swagger ${{ env.SCHEMA_V2 }}
      env:
        OASIS_GEN_SWAGGER_V2: 1

    - name: Store OpenAPI schema
      uses: actions/upload-artifact@v3
      with:
        name: openapi-schema
        path: ${{ env.SCHEMA_ALL }}
        retention-days: 3

    - name: Store OpenAPI schema (only v1)
      uses: actions/upload-artifact@v3
      with:
        name: v1-openapi-schema
        path: ${{ env.SCHEMA_V1 }}
        retention-days: 3

    - name: Store OpenAPI schema (only v2)
      uses: actions/upload-artifact@v3
      with:
        name: v2-openapi-schema
        path: ${{ env.SCHEMA_V2 }}
        retention-days: 3

    - name: Test Schema
      run: ./scripts/build-maven.sh $(cat VERSION)


Code Quality

# This workflow performs code quality checks like:
#   - PEP8: the workflow fails if code is not PEP8 compliant
#   - flake8: the problems identified by flake 8 are listed but the workflow
#             presently doesn't fail if flake reports errors.

name: Code Quality

on: [push, pull_request]

env:
  max_line_length: 150

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"

      - name: install pip tools
        run: |
          python -m pip install --upgrade pip
          pip install --upgrade pip-tools

      - name: install code quality tools
        run: pip install --upgrade autopep8 flake8

      - name: Run (partial) flake8
        if: ${{ ! cancelled() }}
        run: flake8 --select F401,F522,F524,F541 --show-source src/ --config tox.ini

      - name: check PEP8 compliance
        if: ${{ ! cancelled() }}
        id: autopep8
        run: |
          autopep8 --diff --exit-code --recursive --max-line-length ${{ env.max_line_length }} --ignore E402 .


Add PR to project

name: Add PR to project
on:
  pull_request:
    types:
      - opened
jobs:
  track_pr:
    runs-on: ubuntu-latest
    steps:
      - name: Get project data
        env:
          GITHUB_TOKEN: ${{ secrets.PROJECT_ACCESS_TOKEN }}
          ORGANIZATION: Oasislmf
          PROJECT_NUMBER: 44
        run: |
          gh api graphql -f query='
            query($org: String!, $number: Int!) {
              organization(login: $org){
                projectV2(number: $number) {
                  id
                  fields(first:20) {
                    nodes {
                      ... on ProjectV2Field {
                        id
                        name
                      }
                      ... on ProjectV2SingleSelectField {
                        id
                        name
                        options {
                          id
                          name
                        }
                      }
                    }
                  }
                }
              }
            }' -f org=$ORGANIZATION -F number=$PROJECT_NUMBER > project_data.json

          echo 'PROJECT_ID='$(jq '.data.organization.projectV2.id' project_data.json) >> $GITHUB_ENV

      - name: Add PR to project
        env:
          GITHUB_TOKEN: ${{ secrets.PROJECT_ACCESS_TOKEN }}
          PR_ID: ${{ github.event.pull_request.node_id }}
        run: |
          item_id="$( gh api graphql -f query='
            mutation($project:ID!, $pr:ID!) {
              addProjectV2ItemById(input: {projectId: $project, contentId: $pr}) {
                item {
                  id
                }
              }
            }' -f project=$PROJECT_ID -f pr=$PR_ID --jq '.data.addProjectV2ItemById.projectV2Item.id')"


Add issue to project

name: Add issue to project
on:
  issues:
    types:
      - opened
jobs:
  track_issue:
    runs-on: ubuntu-latest
    steps:
      - name: Get project data
        env:
          GITHUB_TOKEN: ${{ secrets.PROJECT_ACCESS_TOKEN }}
          ORGANIZATION: Oasislmf
          PROJECT_NUMBER: 44
        run: |
          gh api graphql -f query='
            query($org: String!, $number: Int!) {
              organization(login: $org){
                projectV2(number: $number) {
                  id
                  fields(first:20) {
                    nodes {
                      ... on ProjectV2Field {
                        id
                        name
                      }
                      ... on ProjectV2SingleSelectField {
                        id
                        name
                        options {
                          id
                          name
                        }
                      }
                    }
                  }
                }
              }
            }' -f org=$ORGANIZATION -F number=$PROJECT_NUMBER > project_data.json

          echo 'PROJECT_ID='$(jq '.data.organization.projectV2.id' project_data.json) >> $GITHUB_ENV

      - name: Add issue to project
        env:
          GITHUB_TOKEN: ${{ secrets.PROJECT_ACCESS_TOKEN }}
          ISSUE_ID: ${{ github.event.issue.node_id }}
        run: |
          item_id="$( gh api graphql -f query='
            mutation($project:ID!, $issue:ID!) {
              addProjectV2ItemById(input: {projectId: $project, contentId: $issue}) {
                item {
                  id
                }
              }
            }' -f project=$PROJECT_ID -f issue=$ISSUE_ID --jq '.data.addProjectV2ItemById.projectV2Item.id')"


Platform Release

name: Platform Release

on:
  workflow_dispatch:
    inputs:
      push_latest:
        description: 'Push images with the "latest" tag'
        required: false
        type: boolean

      pre_release:
        description: 'Mark GitHub release as pre-release: [true, false]'
        required: true
        type: boolean

      release_tag:
        description: 'Release tag, for release: [{n}.{n}.{n}] e.g. 3.0.0,  for Pre-Release  [{n}.{n}.{n}rc{n}]'
        required: true

      prev_release_tag:
        description: '(OVERRIDE) The previous release version for changelog [semvar]'
        required: false

      piwind_branch:
        description: 'Branch to build piwind from'
        default: 'main'
        required: true

      oasislmf_release:
        description: '(OVERRIDE) The oasislmf version in this release [semvar]'
        required: false

      ods_tools_release:
        description: '(OVERRIDE) The ods-tools version in this release [semvar]'
        required: false

      ktools_release:
        description: '(OVERRIDE) The ktools version in this release [semvar] "v{n}.{n}.{n}"'
        required: false

      cve_severity:
        description: 'Severities of vulnerabilities to scanned for, fails build if any found'
        required: false
        default: 'CRITICAL,HIGH'
        type: string

jobs:
  update:
    uses: ./.github/workflows/version.yml
    secrets: inherit
    with:
      platform_version: ${{ inputs.release_tag }}
      oasislmf_version: ${{ inputs.oasislmf_release }}
      ods_tools_version: ${{ inputs.ods_tools_release }}

  build_schema:
    uses: ./.github/workflows/build-schema.yml
    secrets: inherit
    needs: update

  build_images:
    uses: ./.github/workflows/build-images.yml
    secrets: inherit
    needs: update
    with:
      docker_push: true
      ignore_unfixed: true
      cve_severity: ${{ inputs.cve_severity }}

  build_piwind:
    uses: OasisLMF/OasisPiWind/.github/workflows/build.yml@main
    secrets: inherit
    needs: build_images
    with:
      docker_push: true
      from_image: ${{ needs.build_images.outputs.worker_image }}
      piwind_branch: ${{ inputs.piwind_branch }}

  release:
    runs-on: ubuntu-latest
    needs: [build_images, build_schema, build_piwind]
    outputs:
      heading: ${{ steps.slack_vars.outputs.heading }}
      title: ${{ steps.slack_vars.outputs.title }}
      build_branch: ${{ steps.slack_vars.outputs.branch }}
      run_url: ${{ steps.slack_vars.outputs.run_url }}
      run_id: ${{ steps.slack_vars.outputs.run_id }}
      run_status: ${{ steps.slack_vars.outputs.run_status }}
      run_date: ${{ steps.slack_vars.outputs.run_date }}

    env:
      pre_release: ${{ inputs.pre_release == '' && 'false' || inputs.pre_release }}
      push_latest: ${{ inputs.push_latest == '' && 'false' || inputs.push_latest }}
      latest_tag: 'latest'

      release_tag: ${{ inputs.release_tag }}
      prev_release_tag: ${{ inputs.prev_release_tag }}

      oasislmf_release: ${{ inputs.oasislmf_release }}
      ods_tools_release: ${{ inputs.ods_tools_release }}
      ktools_release: ${{ inputs.ktools_release }}

      dir_platform: ${{ github.workspace }}/platform
      dir_oasislmf: ${{ github.workspace }}/oasislmf
      dir_ods_tools: ${{ github.workspace }}/ods_tools
      dir_ktools: ${{ github.workspace }}/ktools

      branch_platform: ${{ github.ref_name }}
      branch_oasislmf: 'main'
      branch_ods_tools: 'main'
      branch_ktools: 'master'

    steps:
     ## RELEASE CHECKS
    - name: check branchname is valid for release
      if: ${{ !startsWith(github.ref_name , 'release/') && !startsWith(github.ref_name, 'stable/') }}
      run: |
        echo "Releases must be trigged on branchs 'release/x.x.x' or 'stable/x.x.x' "
        exit 1

    - name: Check tag is valid for release
      if: env.pre_release == 'false'
      run: |
        VALID=$(echo ${{ env.release_tag }} | grep -oPc "^(\d+)\.(\d+)\.(\d+)$")
        if [[ ! "$VALID" == 1 ]]; then
            echo "Release Tag ${{ env.release_tag }} is not valid"
            exit 1
        fi

    - name: Check tag is valid for pre-release
      if: env.pre_release == 'true'
      run: |
        VALID=$(echo ${{ env.release_tag }} | grep -oPc "^(\d+)\.(\d+)\.(\d+)rc(\d+)$")
        if [[ ! "$VALID" == 1 ]]; then
            echo "Release Tag ${{ env.release_tag }} is not valid"
            exit 1
        fi

    - name: set Oasislmf to stable
      if: startsWith(github.ref_name, 'stable/')
      run: echo "branch_oasislmf=${{ env.branch_platform }}" >> $GITHUB_ENV

    ## CLONE REPOS (plat / oasislmf / ktools / UI)
    - name: Setup github user
      run: |
        git config --global user.email ${{ env.GIT_EMAIL }}
        git config --global user.name ${{ env.GIT_USERNAME }}
        git config --global pull.ff only
      env:
        GIT_EMAIL: ${{ secrets.BUILD_GIT_EMAIL }}
        GIT_USERNAME: ${{ secrets.BUILD_GIT_USERNAME }}

    - name: Checkout Platform
      uses: actions/checkout@v3
      with:
        path: ${{ env.dir_platform }}
        repository: Oasislmf/OasisPlatform
        ref: ${{ github.ref_name }}
        fetch-depth: 0

    - name: Checkout Oasislmf
      uses: actions/checkout@v3
      with:
        path: ${{ env.dir_oasislmf }}
        repository: Oasislmf/Oasislmf
        ref: ${{ env.branch_oasislmf }}
        fetch-depth: 0

    - name: Checkout ODS_Tools
      uses: actions/checkout@v3
      with:
        path: ${{ env.dir_ods_tools }}
        repository: Oasislmf/ODS_Tools
        ref: ${{ env.branch_ods_tools }}
        fetch-depth: 0

    - name: Checkout Ktools
      uses: actions/checkout@v3
      with:
        path: ${{ env.dir_ktools }}
        repository: Oasislmf/ktools
        ref: ${{ env.branch_ktools }}
        fetch-depth: 0

    # If overrides not set, auto-load the current / prev release for each repo
    - name: Find 'prev_release_tag'
      if: inputs.prev_release_tag == ''
      working-directory: ${{ env.dir_platform }}
      run: |
        tag=$( ${{ env.dir_platform }}/scripts/find_release.sh -p "${{ env.pre_release }}" -t 1)
        echo "prev_release_tag=$tag" >> $GITHUB_ENV

    - name: Extract prev component versions
      continue-on-error: true
      run: |
        docker pull coreoasis/model_worker:${{ env.prev_release_tag }}
        docker run --entrypoint "oasislmf" coreoasis/model_worker:${{ env.prev_release_tag }} 'version' > OASISLMF_VERSION_PREV
        docker run --entrypoint "python3" coreoasis/model_worker:${{ env.prev_release_tag }} '-cimport ods_tools; print(ods_tools.__version__)' > ODS_VERSION_PREV
        docker run --entrypoint "eve" coreoasis/model_worker:${{ env.prev_release_tag }} '-v' 2> KTOOLS_VERSION_PREV

    - name: Set prev component versions
      run: |
        PREV_OASISLMF_VER=$(cat OASISLMF_VERSION_PREV)
        PREV_ODS_VER=$(cat ODS_VERSION_PREV)
        PREV_KTOOL_VER=$(head -1 KTOOLS_VERSION_PREV | grep -oP "(\d+)\.(\d+)\.(\d+)rc(\d+)|(\d+)\.(\d+)\.(\d+)")
        echo "oasislmf_release_prev=$PREV_OASISLMF_VER" >> $GITHUB_ENV
        echo "ods_tools_release_prev=$PREV_ODS_VER" >> $GITHUB_ENV
        echo "ktools_release_prev=v$PREV_KTOOL_VER" >> $GITHUB_ENV

    - name: oasislmf tag (OVERRIDE)
      working-directory: ${{ env.dir_oasislmf }}
      if: env.oasislmf_release != ''
      run: |
        git checkout ${{ env.branch_oasislmf }}
        git reset  --hard $(git rev-list -n 1 ${{ env.oasislmf_release }} )

    - name: Find 'oasislmf_release'
      if: inputs.oasislmf_release == ''
      working-directory: ${{ env.dir_oasislmf }}
      run: |
        tag=$( ${{ env.dir_platform }}/scripts/find_release.sh -p "${{ env.pre_release }}" -t 1)
        echo "oasislmf_release=$tag" >> $GITHUB_ENV

    - name: ods_tools tag (OVERRIDE)
      working-directory: ${{ env.dir_ods_tools }}
      if: env.ods_tools_release != ''
      run: |
        git checkout ${{ env.branch_ods_tools }}
        git reset  --hard $(git rev-list -n 1 ${{ env.ods_tools_release }} )

    - name: Find 'ods_tools_release'
      if: inputs.ods_tools_release == ''
      working-directory: ${{ env.dir_ods_tools }}
      run: |
        tag=$( ${{ env.dir_platform }}/scripts/find_release.sh -p "${{ env.pre_release }}" -t 1)
        echo "ods_tools_release=$tag" >> $GITHUB_ENV

    - name: Ktools tag (OVERRIDE)
      working-directory: ${{ env.dir_ktools }}
      #if: ! inputs.ktools_release == ''
      if: env.ktools_release != ''   # TESTING
      run: |
        git checkout ${{ env.branch_ktools }}
        git reset  --hard $(git rev-list -n 1 v${{ env.ktools_release }} )

    - name: Find 'ktools_release'
      if: inputs.ktools_release == ''
      working-directory: ${{ env.dir_ktools }}
      run: |
        tag=$( ${{ env.dir_platform }}/scripts/find_release.sh -p "${{ env.pre_release }}" -t 1 -v 'v')
        echo "ktools_release=$tag" >> $GITHUB_ENV

    ## TAG & assests
    - name: Login to Docker Hub
      uses: docker/login-action@v2
      with:
        username: ${{ secrets.DOCKERHUB_USERNAME }}
        password: ${{ secrets.DOCKERHUB_TOKEN }}

    ## pull a rename build images
    - name: Pull and re-tag images
      run: |
        # Server
        docker pull ${{ needs.build_images.outputs.server_image }}
        docker tag  ${{ needs.build_images.outputs.server_image }} coreoasis/api_server:${{ env.release_tag }}
        docker tag  ${{ needs.build_images.outputs.server_image }} coreoasis/api_server:${{ env.latest_tag }}
        # Worker
        docker pull ${{ needs.build_images.outputs.worker_image }}
        docker tag  ${{ needs.build_images.outputs.worker_image }} coreoasis/model_worker:${{ env.release_tag }}
        docker tag  ${{ needs.build_images.outputs.worker_image }} coreoasis/model_worker:${{ env.latest_tag }}
        # Worker-dev
        docker pull ${{ needs.build_images.outputs.worker_deb_image }}
        docker tag  ${{ needs.build_images.outputs.worker_deb_image }} coreoasis/model_worker:${{ env.release_tag }}-debian
        # Worker-controller
        docker pull ${{ needs.build_images.outputs.worker_controller_image }}
        docker tag  ${{ needs.build_images.outputs.worker_controller_image }} coreoasis/worker_controller:${{ env.release_tag }}
        docker tag  ${{ needs.build_images.outputs.worker_controller_image }} coreoasis/worker_controller:${{ env.latest_tag }}
        # Worker-PiWind
        docker pull ${{ needs.build_piwind.outputs.piwind_image }}
        docker tag  ${{ needs.build_piwind.outputs.piwind_image }} coreoasis/piwind_worker:${{ env.release_tag }}
        docker tag  ${{ needs.build_piwind.outputs.piwind_image }} coreoasis/piwind_worker:${{ env.latest_tag }}

    - name: Check tag matches (Repo)
      working-directory: ${{ env.dir_platform }}
      run: |
        BUILD_VER=$(cat VERSION)
        [[ "${{ env.release_tag }}" = "$BUILD_VER" ]] && ERROR_CODE=0 || ERROR_CODE=1
        if [[ "$ERROR_CODE" == 1 ]]; then
            echo "BUILD_VER: $BUILD_VER stored in Repo 'VERSION' dosn't match RELEASE_TAG: $RELEASE_VER" && exit $ERROR_CODE
        fi

    - name: Check tags match (worker image)
      run: |
        container_id=$(docker create "${{ needs.build_images.outputs.worker_image }}")
        docker cp "$container_id:/home/worker/VERSION" "./IMAGE_WORKER_VERSION"
        docker rm "$container_id"

        BUILD_VER=$(cat ./IMAGE_WORKER_VERSION)
        RELEASE_VER=${{ env.release_tag }}
        [[ "$RELEASE_VER" = "$BUILD_VER" ]] && ERROR_CODE=0 || ERROR_CODE=1
        if [[ "$ERROR_CODE" == 1 ]]; then
            echo "BUILD_VER: $BUILD_VER stored in image '${{ needs.build_images.outputs.worker_image }}' dosn't match RELEASE_TAG: $RELEASE_VER" && exit $ERROR_CODE
        fi

    - name: Check tags match (piwind image)
      run: |
        container_id=$(docker create "${{ needs.build_piwind.outputs.piwind_image }}")
        docker cp "$container_id:/home/worker/VERSION" "./IMAGE_PIWIND_VERSION"
        docker rm "$container_id"

        BUILD_VER=$(cat ./IMAGE_PIWIND_VERSION)
        RELEASE_VER=${{ env.release_tag }}
        [[ "$RELEASE_VER" = "$BUILD_VER" ]] && ERROR_CODE=0 || ERROR_CODE=1
        if [[ "$ERROR_CODE" == 1 ]]; then
            echo "BUILD_VER: $BUILD_VER stored in image '${{ needs.build_piwind.outputs.piwind_image }}' dosn't match RELEASE_TAG: $RELEASE_VER" && exit $ERROR_CODE
        fi

    - name: Check new component versions
      continue-on-error: true
      run: |
        docker run --entrypoint "oasislmf" ${{ needs.build_images.outputs.worker_image }} 'version' > OASISLMF_VERSION
        docker run --entrypoint "python3" ${{ needs.build_images.outputs.worker_image }} '-cimport ods_tools; print(ods_tools.__version__)' > ODS_VERSION
        docker run --entrypoint "eve" ${{ needs.build_images.outputs.worker_image }} '-v' 2> KTOOLS_VERSION

    - name: 'store OASISLMF_VERSION'
      uses: actions/upload-artifact@v3
      with:
        name: OASISLMF_VERSION
        path: OASISLMF_VERSION
        retention-days: 5

    - name: 'store ODS_VERSION'
      uses: actions/upload-artifact@v3
      with:
        name: ODS_VERSION
        path: ODS_VERSION
        retention-days: 5

    - name: 'store KTOOLS_VERSION'
      uses: actions/upload-artifact@v3
      with:
        name: KTOOLS_VERSION
        path: KTOOLS_VERSION
        retention-days: 5

    - name: Check for Oasislmf version match
      run: test "$(cat OASISLMF_VERSION)" = ${{ env.oasislmf_release }} || exit 1

    - name: Check for ods-tools version match
      run: test "$(cat ODS_VERSION)" = ${{ env.ods_tools_release }} || exit 1

    - name: Check for ktools version match
      run: |
        KTOOL_VER=$(head -1 KTOOLS_VERSION | grep -oP "(\d+)\.(\d+)\.(\d+)rc(\d+)|(\d+)\.(\d+)\.(\d+)")
        test "v$KTOOL_VER" = ${{ env.ktools_release }} || exit 1

    - name: Download API schema
      uses: actions/download-artifact@v3
      with:
        name: openapi-schema
        path: ${{ github.workspace }}/

    - name: Download API schema (v1)
      uses: actions/download-artifact@v3
      with:
        name: v1-openapi-schema
        path: ${{ github.workspace }}/

    - name: Download API schema (v2)
      uses: actions/download-artifact@v3
      with:
        name: v2-openapi-schema
        path: ${{ github.workspace }}/

    - name: Name API schema
      id: api_schema
      run: |
        schema_filename="openapi-schema-${{ env.release_tag }}.json"
        v1_schema_filename="v1-openapi-schema-${{ env.release_tag }}.json"
        v2_schema_filename="v2-openapi-schema-${{ env.release_tag }}.json"

        mv openapi-schema.json  $schema_filename
        mv v1-openapi-schema.json  $v1_schema_filename
        mv v2-openapi-schema.json  $v2_schema_filename
        echo "filename_all=$schema_filename" >> $GITHUB_OUTPUT
        echo "filename_v1=$v1_schema_filename" >> $GITHUB_OUTPUT
        echo "filename_v2=$v2_schema_filename" >> $GITHUB_OUTPUT

    # --- Create Changelog --- #
    - name: Tag Release
      working-directory: ${{ env.dir_platform }}
      env:
        GITHUB_TOKEN: ${{ secrets.BUILD_GIT_TOKEN }}
      run: |
        git checkout ${{ github.ref_name }}
        git tag ${{ env.release_tag }}

    - name: Setup python
      uses: actions/setup-python@v4
      with:
        python-version: '3.9'

    - name: Setup Changelog builder
      working-directory: ${{ env.dir_platform  }}
      run: pip install -r scripts/requirments-changelog.txt

    - name: Create changelog
      working-directory: ${{ env.dir_platform }}
      env:
        GITHUB_TOKEN: ${{ secrets.BUILD_GIT_TOKEN }}
      run: |
        ${{ env.dir_platform }}/scripts/update-changelog.py build-changelog \
          --repo OasisPlatform \
          --from-tag ${{ env.prev_release_tag }} \
          --to-tag ${{ env.release_tag }} \
          --github-token ${{ secrets.BUILD_GIT_TOKEN }} \
          --local-repo-path ./ \
          --output-path ./CHANGELOG.rst \
          --apply-milestone
        git add ./CHANGELOG.rst
        git commit -m 'Update changelog'

    - name: Create Release notes
      run: |
        ${{ env.dir_platform }}/scripts/update-changelog.py build-release-platform \
          --platform-repo-path ${{ env.dir_platform }}  \
          --platform-from-tag ${{ env.prev_release_tag }} \
          --platform-to-tag ${{ env.release_tag }} \
          --lmf-repo-path ${{ env.dir_oasislmf }} \
          --lmf-from-tag ${{ env.oasislmf_release_prev }} \
          --lmf-to-tag ${{ env.oasislmf_release }} \
          --ods-repo-path ${{ env.dir_ods_tools }} \
          --ods-from-tag ${{ env.ods_tools_release_prev }} \
          --ods-to-tag ${{ env.ods_tools_release }} \
          --ktools-repo-path ${{ env.dir_ktools }} \
          --ktools-from-tag ${{ env.ktools_release_prev }} \
          --ktools-to-tag ${{ env.ktools_release }} \
          --github-token ${{ secrets.BUILD_GIT_TOKEN }} \
          --output-path ./RELEASE.md
        cat ./RELEASE.md >> $GITHUB_STEP_SUMMARY

    ## --- Push Images --- #
    - name: Push images
      run: |
        docker push coreoasis/api_server:${{ env.release_tag }}
        docker push coreoasis/model_worker:${{ env.release_tag }}
        docker push coreoasis/worker_controller:${{ env.release_tag }}
        docker push coreoasis/model_worker:${{ env.release_tag }}-debian
        docker push coreoasis/piwind_worker:${{ env.release_tag }}

    - name: Push images (Production)
      if: ${{ env.pre_release == 'false' && env.push_latest == 'true' }}
      run: |
        docker push coreoasis/api_server:${{ env.latest_tag }}
        docker push coreoasis/model_worker:${{ env.latest_tag }}
        docker push coreoasis/worker_controller:${{ env.latest_tag }}
        docker push coreoasis/piwind_worker:${{ env.latest_tag }}

    # --- Create Release --- #
    - name: Push changes
      working-directory: ${{ env.dir_platform }}
      run: |
        git push origin ${{ env.release_tag  }}
        git push

    - name: Create Release
      id: create_release
      uses: actions/create-release@v1
      env:
        GITHUB_TOKEN: ${{ secrets.BUILD_GIT_TOKEN }}
      with:
        tag_name: ${{ env.release_tag }}
        release_name: Release ${{ env.release_tag }}
        body_path: ./RELEASE.md
        draft: false
        prerelease: ${{ env.pre_release }}

    # --- Attach build assest --- #
    - name: Upload Schema (base)
      id: upload-source-release-asset
      uses: actions/upload-release-asset@v1
      env:
        GITHUB_TOKEN: ${{ secrets.BUILD_GIT_TOKEN }}
      with:
        upload_url: ${{ steps.create_release.outputs.upload_url }}
        asset_path: ${{ github.workspace }}/${{ steps.api_schema.outputs.filename_all }}
        asset_name: ${{ steps.api_schema.outputs.filename_all }}
        asset_content_type: application/json

    - name: Upload Schema (v1)
      id: upload-source-release-asset-v1
      uses: actions/upload-release-asset@v1
      env:
        GITHUB_TOKEN: ${{ secrets.BUILD_GIT_TOKEN }}
      with:
        upload_url: ${{ steps.create_release.outputs.upload_url }}
        asset_path: ${{ github.workspace }}/${{ steps.api_schema.outputs.filename_v1 }}
        asset_name: ${{ steps.api_schema.outputs.filename_v1 }}
        asset_content_type: application/json

    - name: Upload Schema (v2)
      id: upload-source-release-asset-v2
      uses: actions/upload-release-asset@v1
      env:
        GITHUB_TOKEN: ${{ secrets.BUILD_GIT_TOKEN }}
      with:
        upload_url: ${{ steps.create_release.outputs.upload_url }}
        asset_path: ${{ github.workspace }}/${{ steps.api_schema.outputs.filename_v2 }}
        asset_name: ${{ steps.api_schema.outputs.filename_v2 }}
        asset_content_type: application/json
    # --- Slack notify --- #
    - name: slack message vars
      id: slack_vars
      run: |
        HEAD=$(echo "*${{ github.event.repository.name}} Release* (${{ env.release_tag }})")
        DATE=$(date)
        TITLE=$(echo "• <https://github.com/${{ github.repository }}/releases/tag/${{ env.release_tag }}|${{ github.event.repository.name }} ${{ env.release_tag }} - Release Notes>")
        JOB_URL=${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        echo "heading=$HEAD" >> $GITHUB_OUTPUT
        echo "run_date=$DATE" >> $GITHUB_OUTPUT
        echo "title=$TITLE" >> $GITHUB_OUTPUT
        echo "run_url=$JOB_URL" >> $GITHUB_OUTPUT
        echo "run_id=${{ github.run_id }}"  >> $GITHUB_OUTPUT
        echo "branch=${{ github.ref_name }}" >> $GITHUB_OUTPUT
        echo "run_status=${{ job.status }}" >> $GITHUB_OUTPUT

  tag_piwind:
    uses: OasisLMF/OasisPiWind/.github/workflows/tag-release.yml@main
    secrets: inherit
    needs: release
    with:
      release_tag: ${{ inputs.release_tag }}
      piwind_branch: ${{ inputs.piwind_branch }}

  # --- Notify Slack --- #
  slack:
    uses: OasisLMF/OasisLMF/.github/workflows/notify.yml@main
    secrets: inherit
    needs: release
    with:
      heading: ${{ needs.release.outputs.heading }}
      title: ${{ needs.release.outputs.title }}
      build_branch: ${{ needs.release.outputs.build_branch }}
      run_url: ${{ needs.release.outputs.run_url }}
      run_id: ${{ needs.release.outputs.run_id }}
      run_status: ${{ needs.release.outputs.run_status }}
      run_date: ${{ needs.release.outputs.run_date }}


External Vulnerability Scanning

name: External Vulnerability Scanning

on:
  workflow_dispatch:
    inputs:
      cve_severity:
        description: 'Severities of vulnerabilities to scanned for, [UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL]'
        required: false
        default: 'CRITICAL,HIGH'
      ignore_unfixed:
        description: 'Include unfixed vulnerabilities in scan [true,false]'
        required: false
        default: 'true'

env:
  SEVERITY: 'HIGH,CRITICAL'
  IGNORE_UNFIXED: 'true'

jobs:
  scan_external_images:
    name: Scan Image
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        image: [keycloak, redis, rabbitmq, postgres]
        include:
        - image: keycloak
          report: 'keycloak-scan.sarif'
          exit-code: '1'

        - image: redis
          report: 'redis-scan.sarif'
          exit-code: '1'

        - image: rabbitmq
          report: 'rabbitmq-scan.sarif'
          exit-code: '1'

        - image: postgres
          report: 'postgres-scan.sarif'
          exit-code: '1'

    steps:
    - name: Set inputs
      if: github.event_name == 'workflow_call' || github.event_name == 'workflow_dispatch'
      run: |
        echo "SEVERITY=${{ inputs.cve_severity }}" >> $GITHUB_ENV
        echo "IGNORE_UNFIXED=${{ inputs.ignore_unfixed }}" >> $GITHUB_ENV

    - name: Checkout code
      uses: actions/checkout@v3

    - name: Get Image Name
      id: img
      uses: mikefarah/yq@master
      with:
        cmd: yq '.images.${{ matrix.image }}.image' kubernetes/charts/oasis-platform/values.yaml

    - name: Get Image Version
      id: ver
      uses: mikefarah/yq@master
      with:
        cmd: yq '.images.${{ matrix.image }}.version' kubernetes/charts/oasis-platform/values.yaml

    - name: Vulnerability scanner
      if: env.SEVERITY != ''
      uses: aquasecurity/trivy-action@master
      with:
        image-ref: ${{ steps.img.outputs.result }}:${{ steps.ver.outputs.result }}
        scan-type: 'image'
        exit-code: ${{ matrix.exit-code }}
        ignore-unfixed: ${{ env.IGNORE_UNFIXED }}
        severity: ${{ env.SEVERITY }}
        scanners: 'vuln'

    - name: Create Report
      if: success() || failure()
      uses: aquasecurity/trivy-action@master
      with:
        image-ref: ${{ steps.img.outputs.result }}:${{ steps.ver.outputs.result }}
        format: 'sarif'
        output: ${{ matrix.report }}
        scan-type: 'image'
        exit-code: '0'
        ignore-unfixed: ${{ env.IGNORE_UNFIXED }}
        severity: ${{ env.SEVERITY }}
        scanners: 'vuln'

    - name: Upload Trivy scan results to GitHub Security tab
      if: success() || failure()
      uses: github/codeql-action/upload-sarif@v2
      with:
        sarif_file: ${{ matrix.report }}

    - name: Store report
      if: success() || failure()
      uses: actions/upload-artifact@v3
      with:
        name: ${{ matrix.report }}
        path: ${{ matrix.report }}
        retention-days: 3


Platform Vulnerability Scanning

name: Platform Vulnerability Scanning

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
      - stable**

  workflow_dispatch:
    inputs:
      cve_severity:
        description: 'Severities of vulnerabilities to scanned for, [UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL]'
        required: false
        default: 'CRITICAL,HIGH'
      ignore_unfixed:
        description: 'Include unfixed vulnerabilities in scan [true,false]'
        required: false
        default: 'true'
  workflow_call:
    inputs:
      cve_severity:
        description: 'Severities of vulnerabilities to scanned for, [UNKNOWN,LOW,MEDIUM,HIGH,CRITICAL]'
        required: false
        default: 'CRITICAL,HIGH'
        type: string
      ignore_unfixed:
        description: 'Include unfixed vulnerabilities in scan [true,false]'
        required: false
        default: 'true'
        type: string
      exit_code:
        description: 'Exit code if vulnerabilities found [0,1]'
        required: false
        default: '1'
        type: string

jobs:
  scan_repo:
    name: Scan Repo
    env:
      SEVERITY: 'HIGH,CRITICAL'
      REPORT: 'repo-results.sarif'
      IGNORE_UNFIXED: 'true'
      EXIT_CODE: '1'
    runs-on: ubuntu-latest
    steps:
    - name: Adding code-scanning URL
      run: echo "### [code-scanning results](https://github.com/${{ github.repository }}/security/code-scanning?query=branch%3A${{ github.ref_name }})" >> $GITHUB_STEP_SUMMARY

    - name: Set inputs
      if: github.event_name == 'workflow_call' || github.event_name == 'workflow_dispatch'
      run: |
        echo "SEVERITY=${{ inputs.cve_severity }}" >> $GITHUB_ENV
        echo "IGNORE_UNFIXED=${{ inputs.ignore_unfixed }}" >> $GITHUB_ENV

    - name: Checkout code
      uses: actions/checkout@v3

    - name: Trivy vulnerability scanner
      if: env.SEVERITY != ''
      uses: aquasecurity/trivy-action@master
      with:
        scan-type: 'fs'
        exit-code: '1'
        ignore-unfixed: ${{ env.IGNORE_UNFIXED }}
        severity: ${{ env.SEVERITY }}
        scanners: 'vuln'
        skip-dirs: './docker'

    - name: Trivy Configuration scanner (no fail)
      if: env.SEVERITY != ''
      uses: aquasecurity/trivy-action@master
      with:
        scan-type: 'fs'
        exit-code: '0'
        ignore-unfixed: ${{ env.IGNORE_UNFIXED }}
        severity: ${{ env.SEVERITY }}
        scanners: 'config,secret'
        skip-dirs: './docker'

    - name: Create Report
      if: success() || failure()
      uses: aquasecurity/trivy-action@master
      with:
        format: 'sarif'
        output:  ${{ env.REPORT }}
        scan-type: 'fs'
        exit-code: '0'
        ignore-unfixed: ${{ env.IGNORE_UNFIXED }}
        scanners: 'vuln'
        skip-dirs: './docker'

    - name: Upload Trivy scan results to GitHub Security tab
      if: success() || failure()
      uses: github/codeql-action/upload-sarif@v2
      with:
        sarif_file:  ${{ env.REPORT }}

    - name: Store CVE report
      if: success() || failure()
      uses: actions/upload-artifact@v3
      with:
        name: ${{ env.REPORT }}
        path: ${{ env.REPORT }}
        retention-days: 3