Bump Version

---
name: Bump Version
on:
  push:
    branches: [ master, dev]
jobs:
  version-bump:
    runs-on: ubuntu-latest
    if: ${{ !contains(github.actor, '[bot]') }}
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v3

      - name: Setup python
        uses: actions/setup-python@v4.1.0
        with:
            python-version: 3.10.9
            architecture: x64

      - name: Install bump2version
        run: pip install bump2version

      - name: Determine Version Increment
        id: bump
        run: |
          BRANCH_NAME=${GITHUB_REF#refs/heads/}
          COMMIT_MESSAGE=$(git log -1 --pretty=%B)
          git config --global user.email "bump2version@dk64randomizer.com"
          git config --global user.name "Bump2Version"
          git config --unset-all http.https://github.com/.extraheader
          echo "Branch: $BRANCH_NAME, Commit Message: $COMMIT_MESSAGE"
          if [[ "$BRANCH_NAME" == "dev" ]]; then
              if [[ "$COMMIT_MESSAGE" == *"Major Release"* ]]; then
                  echo "Bumping major version"
                  echo "VERSION_PART=major" >> $GITHUB_OUTPUT
              elif [[ "$COMMIT_MESSAGE" == *"Breaks older cosmetics packs"* ]]; then
                  echo "Bumping minor version"
                  echo "VERSION_PART=minor" >> $GITHUB_OUTPUT
              else
                  echo "Bumping patch version"
                  echo "VERSION_PART=patch" >> $GITHUB_OUTPUT
              fi
          fi

      - name: Bump Version
        run: bump2version ${{ steps.bump.outputs.VERSION_PART }}

      - id: create_token
        uses: tibdex/github-app-token@v1
        with:
          app_id: 715479
          installation_id: 45211287
          private_key: ${{ secrets.BRANCHPROTECTION }}

      - name: Push Changes
        uses: ad-m/github-push-action@master
        with:
          github_token: ${{ steps.create_token.outputs.token }}
          branch: ${{ github.ref }}


name: 'Delete Inactive Users in Github Organization'

on:
  schedule:
  - cron: '0 0 */32,1-7 * 1'
  workflow_dispatch:
  push:
    branches:
      - "add-inactive-user-removal-automation"

jobs:
  org-config-generation-check:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/setup-python@v5
        with:
          python-version: 3.9
      - uses: actions/checkout@v4
        with:
          path: community
      - name: Clean inactive github org users
        id: uds
        run: |
          python -m pip install --upgrade pip
          pip install -r community/org/requirements.txt
          python community/org/org_user_management.py
        env:
          GH_TOKEN: ${{ secrets.GH_TOKEN }}
          INACTIVE_USER_MANAGEMENT_TAG_USERS: ${{ secrets.INACTIVE_USER_MANAGEMENT_TAG_USERS }}
      - name: Create Pull Request
        if: ${{ steps.uds.outputs.inactive_users_pr_description }}
        uses: peter-evans/create-pull-request@v5
        with:
          path: community
          add-paths: org/contributors.yml
          commit-message: Delete inactive users
          branch: delete-inactive-users
          title: 'Inactive users to be deleted'
          body: ${{ steps.uds.outputs.inactive_users_pr_description }}


Check Github Organization Settings PRs

name: 'Check Github Organization Settings PRs'
on:
  pull_request:
    paths:
      - 'org/*'
      - 'toc/TOC.md'
      - 'toc/working-groups/*.md'
      - '.github/workflows/org-management-check-prs.yml'
jobs:
  org-config-generation-check:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/setup-python@v5
        with:
          python-version: 3.9
      - uses: actions/checkout@v4
        with:
          path: community
      - name: Generate github org configuration
        run: |
          python -m pip install --upgrade pip
          pip install -r community/org/requirements.txt
          python community/org/org_management.py -o cloudfoundry.out.yml -b branchprotection.out.yml


Dump Github Organization Settings

name: 'Dump Github Organization Settings'
on:
  workflow_dispatch

jobs:
  generate-peribolos-dump:
    runs-on: ubuntu-latest
    concurrency:
      group: peribolos
    services:
      ghproxy:
        image: rkoster/ghproxy
        options: >-
          --mount type=bind,source=/etc/passwd,target=/etc/passwd,readonly
          --mount type=bind,source=/etc/group,target=/etc/group,readonly
        ports:
          - 8888:8888
        volumes:
          - ${{ github.workspace }}/ghproxy-cache:/cache
    steps:
      - name: ghproxy-cache
        uses: actions/cache@v4
        with:
          path: ${{ github.workspace }}/ghproxy-cache
          key: ghproxy-cache-${{ github.run_number }}
          restore-keys: |
            ghproxy-cache-
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0 # full clone so a PR can be created if needed
          path: community
      - name: write github private key
        run: |
          echo "${GH_PRIVATE_KEY}" > private_key
          echo "${GH_TOKEN}" > token
        env:
          GH_TOKEN: ${{ secrets.GH_TOKEN }}
          GH_PRIVATE_KEY: ${{ secrets.GH_PRIVATE_KEY }}
      - name: generate-peribolos-dump
        uses: docker://gcr.io/k8s-prow/peribolos
        with:
          entrypoint: /bin/sh
          # Switch back to app auth once following PR gets merged: https://github.com/kubernetes/test-infra/pull/24882
          # args: --dump-full --dump cloudfoundry --github-app-id=${{ secrets.GH_APP_ID }} --github-app-private-key-path=private_key > org/cloudfoundry.yml
          args: -c "/ko-app/peribolos --dump-full --dump cloudfoundry --github-endpoint http://ghproxy:8888 --github-token-path=token > community/org/cloudfoundry.yml"
      - name: Create Pull Request
        uses: peter-evans/create-pull-request@v5
        with:
          path: community
          add-paths: org/cloudfoundry.yml
          commit-message: Run peribolos -dump-full
          branch: peribolos-dump
          draft: true
          title: 'Sync org/cloudfoundry.yml with reality'
          body: |
            This PR contains a fresh peribolos dump.
            PR is intended for debugging. Don't merge as-is because parts of the cloudfoundry org configuration are generated from WG charters.


Sync Github Organization Settings

name: 'Sync Github Organization Settings'
on:
  push:
    branches:
      - 'main'
    paths:
      - 'org/*'
      - 'toc/TOC.md'
      - 'toc/working-groups/*.md'
      - '.github/workflows/org-management.yml'
  schedule:
    - cron: '0 */7 * * *'

jobs:
  peribolos:
    runs-on: ubuntu-latest
    concurrency:
      group: peribolos
    services:
      ghproxy:
        image: rkoster/ghproxy
        options: >-
          --mount type=bind,source=/etc/passwd,target=/etc/passwd,readonly
          --mount type=bind,source=/etc/group,target=/etc/group,readonly
        ports:
          - 8888:8888
        volumes:
          - ${{ github.workspace }}/ghproxy-cache:/cache
    steps:
      - name: ghproxy-cache
        uses: actions/cache@v4
        with:
          path: ${{ github.workspace }}/ghproxy-cache
          key: ghproxy-cache-${{ github.run_number }}
          restore-keys: |
            ghproxy-cache-
      - uses: actions/setup-python@v5
        with:
          python-version: 3.9
      - uses: actions/checkout@v4
        with:
          path: community
      - name: Generate github org configuration
        run: |
          python -m pip install --upgrade pip
          pip install -r community/org/requirements.txt
          python community/org/org_management.py -o cloudfoundry.out.yml -b branchprotection.out.yml
      - name: write github private key
        run: |
          echo "${GH_PRIVATE_KEY}" > private_key
          echo "${GH_TOKEN}" > token
        env:
          GH_TOKEN: ${{ secrets.GH_TOKEN }}
          GH_PRIVATE_KEY: ${{ secrets.GH_PRIVATE_KEY }}
      - name: peribolos
        id: peribolos
        uses: docker://gcr.io/k8s-prow/peribolos
        with:
          entrypoint: /ko-app/peribolos
          args: >-
            --confirm=true
            --github-endpoint http://ghproxy:8888
            --required-admins=thelinuxfoundation
            --min-admins=5
            --github-app-id=${{ secrets.GH_APP_ID }}
            --github-app-private-key-path=private_key
            --require-self=false
            --config-path=cloudfoundry.out.yml
            --fix-org
            --fix-org-members
            --fix-repos
            --fix-teams
            --fix-team-members
            --fix-team-repos
            --allow-repo-archival
  branchprotector:
    needs: peribolos
    runs-on: ubuntu-latest
    concurrency:
      group: peribolos
    services:
      ghproxy:
        image: rkoster/ghproxy
        options: >-
          --mount type=bind,source=/etc/passwd,target=/etc/passwd,readonly
          --mount type=bind,source=/etc/group,target=/etc/group,readonly
        ports:
          - 8888:8888
        volumes:
          - ${{ github.workspace }}/ghproxy-cache:/cache
    steps:
      - name: ghproxy-cache
        uses: actions/cache@v4
        with:
          path: ${{ github.workspace }}/ghproxy-cache
          key: ghproxy-cache-${{ github.run_number }}
          restore-keys: |
            ghproxy-cache-
      - uses: actions/setup-python@v5
        with:
          python-version: 3.9
      - uses: actions/checkout@v4
        with:
          path: community
      - name: Generate github org configuration
        run: |
          python -m pip install --upgrade pip
          pip install -r community/org/requirements.txt
          python community/org/org_management.py -o cloudfoundry.out.yml -b branchprotection.out.yml
      - name: write github private key
        run: |
          echo "${GH_PRIVATE_KEY}" > private_key
          echo "${GH_TOKEN}" > token
        env:
          GH_TOKEN: ${{ secrets.GH_TOKEN }}
          GH_PRIVATE_KEY: ${{ secrets.GH_PRIVATE_KEY }}
      - name: branchprotector
        id: branchprotector
        uses: docker://gcr.io/k8s-prow/branchprotector
        with:
          args: >-
            --confirm=true
            --github-endpoint http://ghproxy:8888
            --github-app-id=${{ secrets.GH_APP_ID }}
            --github-app-private-key-path=private_key
            --config-path=branchprotection.out.yml


Sync working group project boards

name: 'Sync working group project boards'

on:
  schedule:
  - cron: '0 */1 * * *'

jobs:
  configs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - id: matrix
        run: |
          echo "::set-output name=matrix::$(./org/generate_working_group_projects_sync_config.sh)"
    outputs:
      matrix: ${{ steps.matrix.outputs.matrix }}

  sync_projects:
    needs: configs
    runs-on: ubuntu-latest
    container:
      image: rkoster/github-multi-repo-project-card-sync

    strategy:
      fail-fast: false
      matrix:
        config: ${{ fromJson(needs.configs.outputs.matrix) }}

    steps:
    - name: write-config
      run:  echo '${{ toJSON(matrix.config) }}' > config.json
    - name: github-projects-sync
      run: /root/sync --config config.json
      env:
        GITHUB_APP_ID: ${{ secrets.GH_APP_ID }}
        GITHUB_PRIVATE_KEY: ${{ secrets.GH_PRIVATE_KEY }}


Remove individual access to repos

name: 'Remove individual access to repos'
on:
  push:
    branches:
      - main
  schedule:
  - cron: '0 */1 * * *'

jobs:
  remove-individual-access-to-repos:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          path: community
      - name: Remove individual access to repos
        env:
          GH_TOKEN: ${{ secrets.GH_TOKEN }}
        run: |
          set -e
          set -o pipefail

          ./community/toc/working-groups/parsable-working-groups.sh | jq -r .[].areas[].repositories[] | grep -E '^cloudfoundry/' | while read -r repo; do
              gh api "repos/${repo}/collaborators?affiliation=direct" | jq -r .[].login | while read -r user; do
                  echo "remove ${user} from ${repo}"
                  gh api -X delete "/repos/${repo}/collaborators/${user}"
              done
          done


ExoBLAS

name: ExoBLAS

on:
  push:
    branches:
      - main
  pull_request:

jobs:
  build-test:
    uses: exo-lang/ExoBLAS/.github/workflows/build-test.yml@main
    with:
      exo-ref: ${{ github.ref }}
      exo-blas-ref: main


Gemmini CI

name: Gemmini CI
on:
  push:
    branches:
      - main
  pull_request:
jobs:
  gemmini:
    runs-on: ubuntu-22.04

    container:
      image: ghcr.io/exo-lang/gemmini:latest
      credentials:
        username: ${{ github.actor }}
        password: ${{ secrets.github_token }}

    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - name: Install dependencies
        run: |
          apt-get update
          apt-get install -y ninja-build
          python -m pip install -r requirements.txt
          python -m pip install -r dev-requirements.txt
          python -m pip install cmake build

      - name: Install exo
        run: |
          python -m build
          python -m pip install dist/*.whl

      - name: Configure, build, and run GEMMINI tests
        shell: bash
        run: |
          cd /root/chipyard
          source $(conda info --base)/etc/profile.d/conda.sh
          source env.sh
          cd -
          cmake -G Ninja -S apps/gemmini -B build -DCMAKE_BUILD_TYPE=Release
          cmake --build build --verbose
          cd build
          ctest -V


Lint

name: Lint
on:
  pull_request_target:
jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.ref }}

      # Format C files
      - uses: DoozyX/clang-format-lint-action@v0.13
        with:
          source: '.'
          exclude: './dependencies ./tests/gemmini/gemmini-rocc-tests'
          clangFormatVersion: 12
          inplace: ${{ github.event.pull_request.head.repo.full_name == github.repository }}

      # Format Python files
      - uses: psf/black@stable
        with:
          options: ""
          version: "22.10.0"

      # Push changes to internal PRs
      - uses: EndBug/add-and-commit@v4
        if: github.event.pull_request.head.repo.full_name == github.repository
        with:
          message: '🤖 apply linter changes (will not trigger CI)'
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


CI

name: CI
on:
  push:
    branches: [ 'main' ]
  pull_request:
jobs:
  build:
    strategy:
      matrix:
        runner: [ ubuntu-latest, macos-14 ]
        include:
          - runner: ubuntu-latest
            CC: clang-13
            CXX: clang++-13
            CLANG: clang-13
          - runner: macos-14
            CC: clang # This will be system AppleClang
            CXX: clang++ # This will be system AppleClang
            CLANG: /usr/local/opt/llvm/bin/clang
    env:
      CC: ${{matrix.CC}}
      CXX: ${{matrix.CXX}}
      CLANG: ${{matrix.CLANG}}
    runs-on: ${{matrix.runner}}
    steps:
      - name: Checkout sources (including submodules)
        uses: actions/checkout@v4
        with:
          submodules: recursive

      - name: Setup Python environment
        uses: actions/setup-python@v4
        with:
          python-version: '3.10'
          cache: 'pip'

      - name: Install Python dependencies
        run: |
          python -m pip install -U pip wheel
          python -m pip install tox==3.24.5

      - name: Get latest CMake and ninja
        uses: lukka/get-cmake@latest

      - name: Setup SDE binaries
        if: runner.os != 'macOS'
        uses: petarpetrovt/setup-sde@v2.3

      - name: Install Clang 13 (Linux)
        if: runner.os == 'Linux'
        run: |
          wget https://apt.llvm.org/llvm.sh
          chmod +x llvm.sh
          sudo ./llvm.sh 13

      - name: Install Clang 13 (macOS)
        if: runner.os == 'macOS'
        run: brew install llvm@13

      - name: Run tox
        run: tox

      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v4
        with:
          fail_ci_if_error: true
          token: ${{ secrets.CODECOV_TOKEN }}


Packaging

name: Packaging
on:
  release:
    types: [ created ]
  push:
    branches: [ 'main' ]
  pull_request:
jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v4

      - name: Install Python dependencies
        run: |
          python -m pip install -U pip
          python -m pip install setuptools wheel twine build

      - name: Build Python package
        run: python -m build --sdist --wheel --outdir dist/ .

      # Will only run on release.
      - name: Publish distribution to PyPI
        if: startsWith(github.ref, 'refs/tags')
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.PYPI_API_TOKEN }}


Draft PDF

name: Draft PDF

on: [push]

jobs:
  paper:
    runs-on: ubuntu-latest
    name: Paper Draft
    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: Build draft PDF
        uses: openjournals/openjournals-draft-action@master
        with:
          journal: joss
          paper-path: paper/paper.md
      - name: Upload
        uses: actions/upload-artifact@v1
        with:
          name: paper
          path: paper/paper.pdf


Tests

name: Tests

on: [push, pull_request]

jobs:
  unittest:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [3.8, 3.9, '3.10', '3.11']
    steps:
    - name: Checkout PyAutoConf
      uses: actions/checkout@v2
      with:
        repository: rhayes777/PyAutoConf
        path: PyAutoConf
    - name: Checkout PyAutoFit
      uses: actions/checkout@v2
      with:
        repository: rhayes777/PyAutoFit
        path: PyAutoFit
    - name: Checkout PyAutoArray
      uses: actions/checkout@v2
      with:
        repository: Jammy2211/PyAutoArray
        path: PyAutoArray
    - name: Checkout PyAutoGalaxy
      uses: actions/checkout@v2
      with:
        path: PyAutoGalaxy
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}
    - uses: actions/cache@v2
      id: cache-pip
      with:
        path: ~/.cache/pip
        key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements.txt') }}
        restore-keys: |
          ${{ runner.os }}-pip-
    - name: Extract branch name
      shell: bash
      run: |
        cd PyAutoGalaxy
        echo "##[set-output name=branch;]$(echo ${GITHUB_REF#refs/heads/})"
      id: extract_branch
    - name: Change to same branch if exists in deps
      shell: bash
      run: |
        export PACKAGES=("PyAutoConf" "PyAutoFit" "PyAutoArray")
        export BRANCH="${{ steps.extract_branch.outputs.branch }}"
        for PACKAGE in ${PACKAGES[@]}; do
          pushd $PACKAGE
          export existed_in_remote=$(git ls-remote --heads origin ${BRANCH})
          if [[ -z ${existed_in_remote} ]]; then
            echo "Branch $BRANCH did not exist in $PACKAGE"
          else
            echo "Branch $BRANCH did exist in $PACKAGE"
            git fetch
            git checkout $BRANCH
          fi
          popd
        done
    - name: Install dependencies
   #   if: steps.cache-pip.outputs.cache-hit != 'true'
      run: |
        pip3 install --upgrade pip
        pip3 install setuptools
        pip3 install wheel
        pip3 install pytest coverage pytest-cov
        pip3 install -r PyAutoConf/requirements.txt
        pip3 install -r PyAutoFit/requirements.txt
        pip3 install -r PyAutoArray/requirements.txt
        pip3 install -r PyAutoArray/optional_requirements.txt
        pip3 install -r PyAutoGalaxy/requirements.txt
        pip3 install -r PyAutoGalaxy/optional_requirements.txt
    - name: Run tests
      run: |
        export ROOT_DIR=`pwd`
        export PYTHONPATH=$PYTHONPATH:$ROOT_DIR/PyAutoConf
        export PYTHONPATH=$PYTHONPATH:$ROOT_DIR/PyAutoFit
        export PYTHONPATH=$PYTHONPATH:$ROOT_DIR/PyAutoArray
        export PYTHONPATH=$PYTHONPATH:$ROOT_DIR/PyAutoGalaxy
        pushd PyAutoGalaxy
        python3 -m pytest --cov autogalaxy --cov-report xml:coverage.xml
    - name: Slack send
      if: ${{ failure() }}
      id: slack
      uses: slackapi/slack-github-action@v1.21.0
      env:
        SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
      with:
        channel-id: C03S98FEDK2
        payload: |
                {
                  "text": "${{ github.repository }}/${{ github.ref_name }} (Python ${{ matrix.python-version }}) build result: ${{ job.status }}\n${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}"
                }


Labeler

name: Labeler

on:
    push:
        branches:
            - main
            - master

jobs:
    labeler:
        runs-on: ubuntu-latest
        steps:
            - name: Check out the repository
              uses: actions/checkout@v4

            - name: Run Labeler
              uses: crazy-max/ghaction-github-labeler@v4.1.0
              with:
                  skip-delete: true


Run Notebooks

name: Run Notebooks

on:
    - push
    - pull_request_target

jobs:
    run:
        runs-on: ubuntu-latest
        strategy:
            matrix:
                notebook: [
                        "docs/tutorials/notebooks/ehrapy_introduction.ipynb",
                        "docs/tutorials/notebooks/mimic_2_introduction.ipynb",
                        "docs/tutorials/notebooks/mimic_2_survival_analysis.ipynb",
                        "docs/tutorials/notebooks/mimic_2_fate.ipynb",
                        "docs/tutorials/notebooks/mimic_2_causal_inference.ipynb",
                        # "docs/tutorials/notebooks/mimic_3_demo.ipynb",
                        # "docs/tutorials/notebooks/medcat.ipynb",
                    ]
        steps:
            - uses: actions/checkout@v4
              with:
                  submodules: "true"
                  token: "${{ secrets.CT_SYNC_TOKEN }}"
            - name: Set up Python
              uses: actions/setup-python@v5
              with:
                  python-version: "3.11"

            - name: Install UV
              run: pip install uv

            - name: Install ehrapy
              run: uv pip install --system .

            - name: Install additional dependencies
              run: uv pip install --system cellrank

            - name: Install nbconvert ipykernel
              run: uv pip install --system nbconvert ipykernel

            - name: Run ${{ matrix.notebook }} Notebook
              run: jupyter nbconvert --to notebook --execute ${{ matrix.notebook }}


Test

name: Test

on:
    push:
        branches: [main]
    pull_request:
        branches: [main]

concurrency:
    group: ${{ github.workflow }}-${{ github.ref }}
    cancel-in-progress: true

jobs:
    test:
        runs-on: ${{ matrix.os }}
        defaults:
            run:
                shell: bash -e {0} # -e to fail on error

        strategy:
            fail-fast: false
            matrix:
                include:
                    - os: ubuntu-latest
                      python: "3.10"
                    - os: ubuntu-latest
                      python: "3.11"
                    - os: ubuntu-latest
                      python: "3.11"
                      pip-flags: "--pre"

        env:
            OS: ${{ matrix.os }}
            PYTHON: ${{ matrix.python }}

        steps:
            - uses: actions/checkout@v4
            - name: Set up Python ${{ matrix.python }}
              uses: actions/setup-python@v5
              with:
                  python-version: ${{ matrix.python }}

            - name: Install test dependencies
              run: python -m pip install --upgrade uv wheel
            - name: Install dependencies
              run: uv pip install --system ${{ matrix.pip-flags }} .[dev,test]

            - name: Test
              env:
                  MPLBACKEND: agg
                  PLATFORM: ${{ matrix.os }}
                  DISPLAY: :42
              run: pytest -v --cov --color=yes

            - name: Upload coverage
              uses: codecov/codecov-action@v3


Release Drafter

name: Release Drafter
on:
    push:
        branches:
            - main
    pull_request:
        branches:
            - main
        types:
            - opened
            - reopened
            - synchronize
jobs:
    update_release_draft:
        runs-on: ubuntu-latest
        steps:
            - uses: release-drafter/release-drafter@v5
              env:
                  GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"


Check Build

name: Check Build

on:
    push:
        branches: [main]
    pull_request:
        branches: [main]

concurrency:
    group: ${{ github.workflow }}-${{ github.ref }}
    cancel-in-progress: true

jobs:
    package:
        runs-on: ubuntu-latest
        steps:
            - uses: actions/checkout@v3

            - name: Set up Python 3.11
              uses: actions/setup-python@v5
              with:
                  python-version: "3.11"
                  cache: "pip"
                  cache-dependency-path: "**/pyproject.toml"

            - name: Install build dependencies
              run: python -m pip install --upgrade pip wheel twine build

            - name: Build package
              run: python -m build

            - name: Check package
              run: twine check --strict dist/*.whl


Release

name: Release

on:
    push:
        tags:
            - "*.*.*"

jobs:
    release:
        name: Release
        runs-on: ubuntu-latest
        steps:
            - name: Checkout code
              uses: actions/checkout@v3

            - name: Set up Python 3.11
              uses: actions/setup-python@v5
              with:
                  python-version: "3.11"

            - name: Install hatch
              run: pip install hatch

            - name: Build project for distribution
              run: hatch build

            - name: Publish a Python distribution to PyPI
              uses: pypa/gh-action-pypi-publish@release/v1
              with:
                  password: ${{ secrets.PYPI_TOKEN }}


Publish Release

name: "Publish Release"

on:
    release:
        types: [published]

defaults:
    run:
        working-directory: ./pyscript.core

jobs:
    publish-release:
        runs-on: ubuntu-latest
        permissions:
            id-token: write
            contents: read
        steps:
            - name: Checkout
              uses: actions/checkout@v4

            - name: Install node
              uses: actions/setup-node@v4
              with:
                  node-version: 18.x

            - name: Cache node modules
              uses: actions/cache@v4
              env:
                  cache-name: cache-node-modules
              with:
                  # npm cache files are stored in `~/.npm` on Linux/macOS
                  path: ~/.npm
                  key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
                  restore-keys: |
                      ${{ runner.os }}-build-${{ env.cache-name }}-
                      ${{ runner.os }}-build-
                      ${{ runner.os }}-

            - name: npm install
              run: npm install && npx playwright install

            - name: build
              run: npm run build

            - name: Generate index.html in snapshot
              working-directory: .
              run: sed 's#_PATH_#https://pyscript.net/releases/${{ github.ref_name }}/#' ./public/index.html > ./pyscript.core/dist/index.html

            - name: Generate release.tar from snapshot and put it in dist/
              working-directory: .
              run: tar -cvf ../release.tar * && mv ../release.tar .

            - name: Configure AWS credentials
              uses: aws-actions/configure-aws-credentials@v4
              with:
                  aws-region: ${{ secrets.AWS_REGION }}
                  role-to-assume: ${{ secrets.AWS_OIDC_RUNNER_ROLE }}

            - name: Sync to S3
              run:
                  | # Update /latest and create an explicitly versioned directory under releases/YYYY.MM.MICRO/
                  aws s3 sync --quiet ./dist/ s3://pyscript.net/latest/
                  aws s3 sync --quiet ./dist/ s3://pyscript.net/releases/${{ github.ref_name }}/


Prepare Release

name: "Prepare Release"

on:
    push:
        tags:
            - "[0-9][0-9][0-9][0-9].[0-9][0-9].[0-9]+" # YYYY.MM.MICRO

defaults:
    run:
        working-directory: ./pyscript.core

jobs:
    prepare-release:
        runs-on: ubuntu-latest
        steps:
            - name: Checkout
              uses: actions/checkout@v4

            - name: Install node
              uses: actions/setup-node@v4
              with:
                  node-version: 18.x

            - name: Cache node modules
              uses: actions/cache@v4
              env:
                  cache-name: cache-node-modules
              with:
                  # npm cache files are stored in `~/.npm` on Linux/macOS
                  path: ~/.npm
                  key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
                  restore-keys: |
                      ${{ runner.os }}-build-${{ env.cache-name }}-
                      ${{ runner.os }}-build-
                      ${{ runner.os }}-

            - name: NPM Install
              run: npm install && npx playwright install

            - name: Build
              run: npm run build

            - name: Generate index.html
              working-directory: .
              run: sed 's#_PATH_#./#' ./public/index.html > ./pyscript.core/dist/index.html

            - name: Zip dist folder
              run: zip -r -q ./build.zip ./dist

            - name: Prepare Release
              uses: softprops/action-gh-release@v2
              with:
                  draft: true
                  prerelease: true
                  generate_release_notes: true
                  files: ./build.zip


Publish Snapshot

name: "Publish Snapshot"

on:
    workflow_dispatch:
        inputs:
            snapshot_version:
                description: "The calver version of this snapshot: 2022.09.1 or 2022.09.1.RC1"
                type: string
                required: true

defaults:
    run:
        working-directory: ./pyscript.core

jobs:
    publish-snapshot:
        runs-on: ubuntu-latest
        permissions:
            contents: read
            id-token: write
        steps:
            - name: Checkout
              uses: actions/checkout@v4

            - name: Install node
              uses: actions/setup-node@v4
              with:
                  node-version: 18.x

            - name: Cache node modules
              uses: actions/cache@v4
              env:
                  cache-name: cache-node-modules
              with:
                  # npm cache files are stored in `~/.npm` on Linux/macOS
                  path: ~/.npm
                  key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
                  restore-keys: |
                      ${{ runner.os }}-build-${{ env.cache-name }}-
                      ${{ runner.os }}-build-
                      ${{ runner.os }}-

            - name: Install Dependencies
              run: npm install && npx playwright install

            - name: Build Pyscript.core
              run: npm run build

            - name: Configure AWS credentials
              uses: aws-actions/configure-aws-credentials@v4
              with:
                  aws-region: ${{ secrets.AWS_REGION }}
                  role-to-assume: ${{ secrets.AWS_OIDC_RUNNER_ROLE }}

            - name: Generate index.html in snapshot
              working-directory: .
              run: sed 's#_PATH_#https://pyscript.net/snapshots/${{ inputs.snapshot_version }}/#' ./public/index.html > ./pyscript.core/dist/index.html

            - name: Copy to Snapshot
              run: >
                  aws s3 sync ./dist/ s3://pyscript.net/snapshots/${{ inputs.snapshot_version }}/


Test Report

name: Test Report
on:
    workflow_run:
        workflows: ['\[CI\] Test']
        types:
            - completed
jobs:
    report:
        runs-on: ubuntu-latest-8core
        steps:
            - uses: dorny/test-reporter@v1.9.0
              with:
                  artifact: test_results
                  name: Test reports
                  path: "*.xml"
                  reporter: java-junit


Publish Unstable

name: "Publish Unstable"

on:
    push: # Only run on merges into main that modify files under pyscript.core/ and examples/
        branches:
            - main
        paths:
            - pyscript.core/**
            - examples/**

    workflow_dispatch:

jobs:
    publish-unstable:
        runs-on: ubuntu-latest
        permissions:
            id-token: write
            contents: read
        defaults:
            run:
                working-directory: ./pyscript.core
        steps:
            - name: Checkout
              uses: actions/checkout@v4

            - name: Install node
              uses: actions/setup-node@v4
              with:
                  node-version: 18.x

            - name: Cache node modules
              uses: actions/cache@v4
              env:
                  cache-name: cache-node-modules
              with:
                  # npm cache files are stored in `~/.npm` on Linux/macOS
                  path: ~/.npm
                  key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
                  restore-keys: |
                      ${{ runner.os }}-build-${{ env.cache-name }}-
                      ${{ runner.os }}-build-
                      ${{ runner.os }}-

            - name: NPM Install
              run: npm install && npx playwright install

            - name: Build
              run: npm run build

            - name: Generate index.html in snapshot
              working-directory: .
              run: sed 's#_PATH_#https://pyscript.net/unstable/#' ./public/index.html > ./pyscript.core/dist/index.html

            - name: Configure AWS credentials
              uses: aws-actions/configure-aws-credentials@v4
              with:
                  aws-region: ${{ secrets.AWS_REGION }}
                  role-to-assume: ${{ secrets.AWS_OIDC_RUNNER_ROLE }}

            - name: Sync to S3
              run: aws s3 sync --quiet ./dist/ s3://pyscript.net/unstable/


[CI] Test

name: "[CI] Test"

on:
    push: # Only run on merges into main that modify certain files
        branches:
            - main
        paths:
            - pyscript.core/**
            - .github/workflows/test.yml

    pull_request: # Only run on merges into main that modify certain files
        branches:
            - main
        paths:
            - pyscript.core/**
            - .github/workflows/test.yml
    workflow_dispatch:

jobs:
    BuildAndTest:
        runs-on: ubuntu-latest-8core
        env:
            MINICONDA_PYTHON_VERSION: py38
            MINICONDA_VERSION: 4.11.0
        steps:
            - name: Checkout
              uses: actions/checkout@v4
              with:
                  fetch-depth: 3

            # display a git log: when you run CI on PRs, github automatically
            # merges the PR into main and run the CI on that commit. The idea
            # here is to show enough of git log to understand what is the
            # actual commit (in the PR) that we are using. See also
            # 'fetch-depth: 3' above.
            - name: git log
              run: git log --graph -3

            - name: Install node
              uses: actions/setup-node@v4
              with:
                  node-version: 20.x

            - name: Cache node modules
              uses: actions/cache@v4
              env:
                  cache-name: cache-node-modules
              with:
                  # npm cache files are stored in `~/.npm` on Linux/macOS
                  path: ~/.npm
                  key: ${{ runner.os }}-build-${{ env.cache-name }}-${{ hashFiles('**/package-lock.json') }}
                  restore-keys: |
                      ${{ runner.os }}-build-${{ env.cache-name }}-
                      ${{ runner.os }}-build-
                      ${{ runner.os }}-

            - name: setup Miniconda
              uses: conda-incubator/setup-miniconda@v3

            - name: Create and activate virtual environment
              run: |
                  python3 -m venv test_venv
                  source test_venv/bin/activate
                  echo PATH=$PATH >> $GITHUB_ENV
                  echo VIRTUAL_ENV=$VIRTUAL_ENV >> $GITHUB_ENV

            - name: Setup dependencies in virtual environment
              run: |
                  make setup

            - name: Build
              run: make build

            - name: Integration Tests
              #run: make test-integration-parallel
              run: |
                  make test-integration

            - uses: actions/upload-artifact@v4
              with:
                  name: pyscript
                  path: |
                      pyscript.core/dist/
                  if-no-files-found: error
                  retention-days: 7

            - uses: actions/upload-artifact@v4
              if: success() || failure()
              with:
                  name: test_results
                  path: test_results/
                  if-no-files-found: error


Lint code

name: Lint code

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

defaults:
  run:
    shell: bash

jobs:
  stable:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: ['ubuntu-latest']
        python-version: ["3.10"]

    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Display Python version
        run: python -c "import sys; print(sys.version)"
      - name: Install flake8 and related packages
        run: python -m pip install \
          flake8 flake8-absolute-import flake8-black flake8-docstrings \
          flake8-isort flake8-pyproject flake8-unused-arguments \
          flake8-use-fstring pep8-naming
      - name: Check xcp_d
        run: python -m flake8 xcp_d


Build Docker

name: Build Docker

on:
  pull_request:
    types:
      - closed
    branches:
      - main
  push:

jobs:
  test:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python 3.11
        uses: actions/setup-python@v3
        with:
          python-version: '3.11'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          if [ -f backend/requirements.txt ]; then pip install -r backend/requirements.txt; fi
          pip install pytest
      - name: Test
        working-directory: ./backend/src
        run: |
          mkdir -p config
          pytest

  webui-test:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Install Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20

      - uses: pnpm/action-setup@v4
        name: Install pnpm
        with:
          version: 9
          run_install: false

      - name: Get pnpm store directory
        shell: bash
        run: |
          echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV

      - uses: actions/cache@v4
        name: Setup pnpm cache
        with:
          path: ${{ env.STORE_PATH }}
          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
          restore-keys: |
            ${{ runner.os }}-pnpm-store-

      - name: Install dependencies
        run: cd webui && pnpm install

      - name: build test
        run: |
          cd webui && pnpm test:build

  version-info:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: If release
        id: release
        run: |
          if [[ '${{ github.event_name }}' == 'pull_request' && '${{ github.event.pull_request.head.ref }}' == *'dev'* ]]; then
            if [ ${{ github.event.pull_request.merged }} == true ]; then
              echo "release=1" >> $GITHUB_OUTPUT
            else
              echo "release=0" >> $GITHUB_OUTPUT
            fi
          elif [[ '${{ github.event_name }}' == 'push' && (${{ github.ref }} == *'alpha'* || ${{ github.ref }} == *'beta'*) ]]; then
            echo "release=1" >> $GITHUB_OUTPUT
          else
            echo "release=0" >> $GITHUB_OUTPUT
          fi
      - name: If dev
        id: dev
        run: |
          if [[ '${{ github.event_name }}' == 'push' && (${{ github.ref }} == *'alpha'* || ${{ github.ref }} == *'beta'*) ]]; then
            echo "dev=1" >> $GITHUB_OUTPUT
          else
            echo "dev=0" >> $GITHUB_OUTPUT
          fi
      - name: Check version
        id: version
        run: |
          if [ '${{ github.event_name }}' == 'pull_request' ]; then
            if [ ${{ github.event.pull_request.merged }} == true ]; then
              echo "version=${{ github.event.pull_request.title }}" >> $GITHUB_OUTPUT
            fi
          elif [[ ${{ github.event_name }} == 'push' && (${{ github.ref }} == *'alpha'* || ${{ github.ref }} == *'beta'*) ]]; then
            echo "version=${{ github.ref_name }}" >> $GITHUB_OUTPUT
          else
            echo "version=Test" >> $GITHUB_OUTPUT
          fi
      - name: Check result
        run: |
          echo "release: ${{ steps.release.outputs.release }}"
          echo "dev: ${{ steps.dev.outputs.dev }}"
          echo "version: ${{ steps.version.outputs.version }}"
    outputs:
      release: ${{ steps.release.outputs.release }}
      dev: ${{ steps.dev.outputs.dev }}
      version: ${{ steps.version.outputs.version }}

  build-webui:
    runs-on: ubuntu-latest
    needs: [test, webui-test, version-info]
    if: ${{ needs.version-info.outputs.release == 1 || needs.version-info.outputs.dev == 1 }}
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Install Node.js
        uses: actions/setup-node@v4
        with:
          node-version: 20

      - uses: pnpm/action-setup@v4
        name: Install pnpm
        with:
          version: 9
          run_install: false

      - name: Get pnpm store directory
        shell: bash
        run: |
          echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV

      - uses: actions/cache@v4
        name: Setup pnpm cache
        with:
          path: ${{ env.STORE_PATH }}
          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
          restore-keys: |
            ${{ runner.os }}-pnpm-store-

      - name: Install dependencies
        run: cd webui && pnpm install

      - name: Build
        run: |
          cd webui && pnpm build

      - name: Upload artifact
        uses: actions/upload-artifact@v3
        with:
          name: dist
          path: webui/dist

  build-docker:
    runs-on: ubuntu-latest
    needs: [build-webui, version-info]
    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Create Version info via tag
        working-directory: ./backend/src
        run: |
          echo ${{ needs.version-info.outputs.version }}
          echo "VERSION='${{ needs.version-info.outputs.version }}'" >> module/__version__.py

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2

      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v2

      - name: Docker metadata main
        if: ${{ needs.version-info.outputs.release == 1 && needs.version-info.outputs.dev != 1 }}
        id: meta
        uses: docker/metadata-action@v4
        with:
          images: |
            estrellaxd/auto_bangumi
            ghcr.io/${{ github.repository }}
          tags: |
            type=raw,value=${{ needs.version-info.outputs.version }}
            type=raw,value=latest

      - name: Docker metadata dev
        if: ${{ needs.version-info.outputs.dev == 1 }}
        id: meta-dev
        uses: docker/metadata-action@v4
        with:
          images: |
            estrellaxd/auto_bangumi
            ghcr.io/${{ github.repository }}
          tags: |
            type=raw,value=${{ needs.version-info.outputs.version }}
            type=raw,value=dev-latest

      - name: Login to DockerHub
        if: ${{ needs.version-info.outputs.release == 1 }}
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}

      - name: Login to ghcr.io
        if: ${{ needs.version-info.outputs.release == 1 }}
        uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.ACCESS_TOKEN }}

      - name: Download artifact
        uses: actions/download-artifact@v3
        with:
          name: dist
          path: backend/src/dist

      - name: Build and push
        if: ${{ needs.version-info.outputs.release == 1 && needs.version-info.outputs.dev != 1 }}
        uses: docker/build-push-action@v4
        with:
          context: .
          builder: ${{ steps.buildx.output.name }}
          platforms: linux/amd64,linux/arm64,linux/arm/v7
          push: True
          tags: ${{ steps.meta.outputs.tags }}
          labels: ${{ steps.meta.outputs.labels }}
          cache-from: type=gha, scope=${{ github.workflow }}
          cache-to: type=gha, scope=${{ github.workflow }}

      - name: Build and push dev
        if: ${{ needs.version-info.outputs.dev == 1 }}
        uses: docker/build-push-action@v4
        with:
          context: .
          builder: ${{ steps.buildx.output.name }}
          platforms: linux/amd64,linux/arm64,linux/arm/v7
          push: ${{ github.event_name == 'push' }}
          tags: ${{ steps.meta-dev.outputs.tags }}
          labels: ${{ steps.meta-dev.outputs.labels }}
          cache-from: type=gha, scope=${{ github.workflow }}
          cache-to: type=gha, scope=${{ github.workflow }}

      - name: Build test
        if: ${{ needs.version-info.outputs.release == 0 }}
        uses: docker/build-push-action@v4
        with:
          context: .
          builder: ${{ steps.buildx.output.name }}
          platforms: linux/amd64,linux/arm64,linux/arm/v7
          push: false
          tags: estrellaxd/auto_bangumi:test
          cache-from: type=gha, scope=${{ github.workflow }}
          cache-to: type=gha, scope=${{ github.workflow }}

  release:
    runs-on: ubuntu-latest
    needs: [build-docker, version-info]
    if: ${{ needs.version-info.outputs.release == 1 }}
    outputs:
      url: ${{ steps.release.outputs.url }}
      version: ${{ needs.version-info.outputs.version }}
    steps:
      - name: Checkout code
        uses: actions/checkout@v4

      - name: Download artifact webui
        uses: actions/download-artifact@v3
        with:
          name: dist
          path: webui/dist

      - name: Zip webui
        run: |
          cd webui && ls -al && tree && zip -r dist.zip dist

      - name: Download artifact app
        uses: actions/download-artifact@v3
        with:
          name: dist
          path: backend/src/dist

      - name: Create Version info via tag
        working-directory: ./backend/src
        run: |
          echo ${{ needs.version-info.outputs.version }}
          echo "VERSION='${{ needs.version-info.outputs.version }}'" >> module/__version__.py

      - name: Copy requirements.txt
        working-directory: ./backend
        run: cp requirements.txt src/requirements.txt

      - name: Zip app
        run: |
          cd backend && zip -r app-v${{ needs.version-info.outputs.version }}.zip src

      - name: Generate Release info
        id: release-info
        run: |
          if ${{ needs.version-info.outputs.dev == 1 }}; then
            echo "version=🌙${{ needs.version-info.outputs.version }}" >> $GITHUB_OUTPUT
            echo "pre_release=true" >> $GITHUB_OUTPUT
          else
            echo "version=🌟${{ needs.version-info.outputs.version }}" >> $GITHUB_OUTPUT
            echo "pre_release=false" >> $GITHUB_OUTPUT
          fi

      - name: Release
        id: release
        uses: softprops/action-gh-release@v1
        with:
          tag_name: ${{ needs.version-info.outputs.version }}
          name: ${{ steps.release-info.outputs.version }}
          body: ${{ github.event.pull_request.body }}
          draft: false
          prerelease: ${{ steps.release-info.outputs.pre_release == 'true' }}
          files: |
            webui/dist.zip
            backend/app-v${{ needs.version-info.outputs.version }}.zip
        env:
          GITHUB_TOKEN: ${{ secrets.ACCESS_TOKEN }}

  telegram:
    runs-on: ubuntu-latest
    needs: [release]
    steps:
      - name: send telegram message on push
        uses: appleboy/telegram-action@master
        with:
          to: ${{ secrets.TELEGRAM_TO }}
          token: ${{ secrets.TELEGRAM_TOKEN }}
          message: |
            New release: ${{ needs.release.outputs.version }}
            Link: ${{ needs.release.outputs.url }}


name: Node.js - Build & Auto-Commit to CLI data/report/**

on:
  pull_request:
    types:
      - closed
      # Trigger when PR is closed
    paths:
      - "static_report/**"
      - "piperider_cli/**"
      # Trigger when diffs detected in cli and report sources
    branches:
      - main
      # Only PR target is `main`

jobs:
  if_merged_build:
    if: github.event.pull_request.merged == true
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: ./static_report

    steps:
    - uses: actions/checkout@v3

    - name: Use Node.js ${{ matrix.node-version }}
      uses: actions/setup-node@v3
      with:
        node-version: 16

    - uses: pnpm/action-setup@v2.2.2
      with:
        version: 7

    - name: Install dependencies
      working-directory: ./static_report
      run: pnpm install --frozen-lockfile

    - name: Build Apps
      run: pnpm run build

    - name: Pull Remote Changes (before AC)
      run: git pull origin main

    - uses: stefanzweifel/git-auto-commit-action@v4
      with:
        commit_message: AUTO-COMMIT - build report statics to CLI data/report
        branch: ${{ github.base_ref }}
        # Auto-commits to target merge branch


name: Node.js - E2E, Unit Tests, Lint (for PR)

on:
  pull_request:
    types: [opened, synchronize, reopened]
      # Trigger when PR is created and/or head of branch is updated.
    paths:
      - "static_report/**"
      - "piperider_cli/**"
      # Trigger when diffs detected in statics source code
    branches:
      - main
      # Only PR target is `main`

jobs:
  build:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: ./static_report

    steps:
    - uses: actions/checkout@v3

    - name: Use Node.js ${{ matrix.node-version }}
      uses: actions/setup-node@v3
      with:
        node-version: 16

    - uses: pnpm/action-setup@v2.2.2
      with:
        version: 7

    - name: Install dependencies
      working-directory: ./static_report
      run: pnpm install --frozen-lockfile

    - name: Set up Python 3.8
      uses: actions/setup-python@v2
      with:
        python-version: 3.8

    - name: Prepare Piperider Data & E2E Builds (2+1 runs; 1+1 comparison)
      working-directory: .
      run: |
        pip install -r requirements.txt

        which piperider
        piperider version

        bash static_report/src/sdlc/generate-e2e-data.sh first

        cd static_report
        pnpm run embed:html e2e
        pnpm run bundle:e2e
        cd ..

        bash static_report/src/sdlc/generate-e2e-data.sh second

        cd static_report
        pnpm run embed:html e2e
        pnpm run bundle:e2e:case
        cd ..

        zip -r reports.zip ./static_report/build

    - name: Cypress run (E2E)
    # Parallelization only for cypress dashboard https://www.cypress.io/pricing/
      uses: cypress-io/github-action@v2
      with:
        install: false
        record: false
        working-directory: ./static_report
        config-file: cypress.config.ts
        start: |
          pnpm run serve:e2e:norm:single
          pnpm run serve:e2e:norm:comparison
          pnpm run serve:e2e:case:single-edge
          pnpm run serve:e2e:case:comparison-asymmetric
        wait-on: 'http://localhost:3000, http://localhost:3001, http://localhost:4000, http://localhost:4001'

    - uses: actions/upload-artifact@v3
      with:
        working-directory: ./static_report
        name: report-artifacts
        path: reports.zip

    - name: Build Apps
      run: pnpm run build


Run dbt tests

name: Run dbt tests

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

jobs:
  build:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        version: [ '3.8', '3.9', '3.10', '3.11' ]
        dbt: [ ">=1.3,<1.4", ">=1.4,<1.5", ">=1.5,<1.6", ">=1.6,<1.7", ">=1.7,<1.8" ]

    steps:
      - uses: actions/checkout@v2
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.version }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install pytest "dbt-core${{ matrix.dbt }}"
          pip install '.[duckdb]'
      - name: Run tests
        run: |
          set +e
          output=$(python -c "from dbt import version as v" 2>&1)
          exit_code=$?
          set -e

          if [ $exit_code -eq 0 ]; then
              python -mpytest tests/test_dbt_manifest_compatible.py tests/test_dbt_integeration.py tests/test_dbt_util.py
          else
              # https://github.com/dbt-labs/dbt-core/issues/6203#issuecomment-1319112890
              echo "Skip! dbt-core v1.3 only works with python less than 3.11"
          fi
      - name: Run jaffle shop
        if: matrix.dbt != '>=1.3,<1.4'
        run: |
          pip install "dbt-duckdb${{ matrix.dbt }}"

          ./e2e-jaffle-shop/run.sh

          path="/home/runner/work/piperider/piperider/e2e-jaffle-shop/jaffle_shop/.piperider/comparisons/latest/summary.md"
          if [ ! -e "$path" ]; then
            echo "The path '$path' does not exist."
            exit 1
          fi


Run dbt tests on win

name: Run dbt tests on win

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
  workflow_dispatch:

jobs:
  build:
    runs-on: windows-latest
    strategy:
      fail-fast: false
      matrix:
        version: [ '3.11' ]
        dbt: [ ">=1.5,<1.6", ">=1.6,<1.7" ]

    steps:
      - uses: actions/checkout@v2
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.version }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install pytest "dbt-core${{ matrix.dbt }}" "dbt-duckdb${{ matrix.dbt }}"
          pip install '.[duckdb]'

      - name: Run jaffle shop
        run: |
          ./e2e-jaffle-shop/run.bat
          python ./e2e-jaffle-shop/windows-result-checker.py


Lint by Flake8

name: Lint by Flake8

on: [ push, pull_request ]

jobs:
  flake8:
    runs-on: ubuntu-latest
    steps:
      - name: Setup Python
        uses: actions/setup-python@v1
        with:
          python-version: 3.9
          architecture: x64

      - name: Checkout Code
        uses: actions/checkout@master

      - name: Install flake8
        run: pip install flake8

      - name: Run flake8
        uses: suo/flake8-github-action@releases/v1
        with:
          checkName: 'flake8'   # NOTE: this needs to be the same as the job name
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


Nightly Build

name: Nightly Build
on:
  schedule:
    - cron: '0 18 * * 0,1,2,3,4' # run at 2 AM (UTC + 8) every working day
  workflow_dispatch:
    branches:
      - main
    inputs:
      alpha_version:
        description: 'Alpha version serial number'
        required: true
        type: number
        default: '0'

jobs:
  build-nightly-release:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
    steps:
      - name: Checkout code
        uses: actions/checkout@v3
        with:
          fetch-depth: 0 # fetch all the code, we need the full log history for shortcut integration

      - name: Set up Python 3.8
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi

      - name: Release to PyPi
        id: release_pypi
        run: |
          # remove tracking config for nightly build
          echo "" > ./piperider_cli/data/CONFIG

          # generate pypirc
          echo "$PYPIRC" > $HOME/.pypirc

          # change package name to piperider-nightly
          sed -i.bak "s/name='piperider'/name='piperider-nightly'/" setup.py

          # update version number
          echo "Nightly build version: $(date '+%Y%m%d')"
          if [ "$ALPHA_VERSION" != "" ]; then
            echo "Manually alpha version serial number: $ALPHA_VERSION"
            sed -i.bak "s/\.dev\$/\.$(date '+%Y%m%d')a$ALPHA_VERSION/" piperider_cli/VERSION
          else
            sed -i.bak "s/\.dev\$/\.$(date '+%Y%m%d')/" piperider_cli/VERSION
          fi
          echo "Nightly build version: $(cat piperider_cli/VERSION)"
          echo "::set-output name=nightly_version::$(cat piperider_cli/VERSION)"

          # add commit sha to version command
          echo "$GITHUB_SHA" > piperider_cli/data/COMMIT

          # put config includes tracking api key
          echo "$CONFIG" > ./piperider_cli/data/CONFIG

          # release to PyPI
          make release
        env:
          PYPIRC: ${{ secrets.PYPI_NIGHTLY }}
          CONFIG: ${{ secrets.DEV_CONFIG }}
          ALPHA_VERSION: ${{ inputs.alpha_version || '' }}

      - name: Mark Sentry Release
        uses: getsentry/action-release@v1
        env:
          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
          SENTRY_ORG: infuseai
          SENTRY_PROJECT: piperider
        with:
          environment: nightly
          ignore_empty: true
          ignore_missing: true
          version: ${{ steps.release_pypi.outputs.nightly_version }}

      - name: Notify Release to Slack and Update Shortcut Stories
        run: |
          if [ "$ALPHA_VERSION" != '' ]; then
            echo "[Skip] Notify release to Slack and Update Shortcut Stories"
            echo "PipeRider Alpha version: $(cat piperider_cli/VERSION)"
          else
            echo "Notify release to Slack and Update Shortcut Stories"
            echo "PipeRider version: $(cat piperider_cli/VERSION)"
            bash .github/scripts/deploy_notify.sh $(cat piperider_cli/VERSION)
          fi
        env:
          ALPHA_VERSION: ${{ inputs.alpha_version || '' }}
          SLACK_API_TOKEN: ${{ secrets.SLACK_API_TOKEN }}
          SHORTCUT_API_TOKEN: ${{ secrets.SHORTCUT_API_TOKEN }}
          SLACK_CHANNEL: '#dev'
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Mark Amplitude Release
        run: |
          if [ "$ALPHA_VERSION" != '' ]; then
            echo "[Skip] Mark Amplitude Release"
          else
            bash .github/scripts/bump_amplitude_release.sh $VERSION
          fi
        env:
          ALPHA_VERSION: ${{ inputs.alpha_version || '' }}
          VERSION: ${{ steps.release_pypi.outputs.nightly_version }}
          AMPLITUDE_API_KEY: ${{ secrets.AMPLITUDE_DEV_API_KEY }}
          AMPLITUDE_SECRET_KEY: ${{ secrets.AMPLITUDE_DEV_SECRET_KEY }}


PipeRider PR Title Checker

name: "PipeRider PR Title Checker"
on:
  pull_request_target:
    types:
      - opened
      - edited
      - synchronize
      - labeled
      - unlabeled

jobs:
  pr-title-check:
    runs-on: ubuntu-latest
    steps:
      - uses: thehanimo/pr-title-checker@v1.3.4
        with:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          pass_on_octokit_error: false
          configuration_path: ".github/PULL_REQUEST_TITLE_CHECKER.json"


Release to PyPI

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Release to PyPI

on:
  push:
    tags: 'v*'

jobs:
  build:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false

    steps:
      - uses: actions/checkout@v2
      - name: Set up Python 3.9
        uses: actions/setup-python@v2
        with:
          python-version: 3.9
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
      - name: Run tests
        run: |
          make test
      - name: Release
        id: release_pypi
        run: |
          # get version from GITHUB_REF
          # input: "refs/tags/v0.1.0"
          # output: "0.1.0"

          # update version number
          echo "${GITHUB_REF:11}" > ./piperider_cli/VERSION
          echo "::set-output name=version::$(cat piperider_cli/VERSION)"
          # put config includes tracking api key
          echo "$CONFIG" > ./piperider_cli/data/CONFIG

          # set sentry environment
          if [[ "$(cat piperider_cli/VERSION)" == *"a"* ]]; then
            echo "::set-output name=sentry_env::alpha"
          elif [[ "$(cat piperider_cli/VERSION)" == *"b"* ]]; then
            echo "::set-output name=sentry_env::beta"
          elif [[ "$(cat piperider_cli/VERSION)" == *"rc"* ]]; then
            echo "::set-output name=sentry_env::release-candidate"
          else
            echo "::set-output name=sentry_env::production"
          fi

          # generate pypirc
          echo "$PYPIRC" > $HOME/.pypirc

          # release to PyPI
          make release
        env:
          PYPIRC: ${{ secrets.PYPI }}
          CONFIG: ${{ secrets.CONFIG }}
      - name: Mark Sentry Release
        uses: getsentry/action-release@v1
        env:
          SENTRY_AUTH_TOKEN: ${{ secrets.SENTRY_AUTH_TOKEN }}
          SENTRY_ORG: infuseai
          SENTRY_PROJECT: piperider
        with:
          environment: ${{ steps.release_pypi.outputs.sentry_env || 'production' }}
          ignore_empty: true
          ignore_missing: true
          version: ${{ steps.release_pypi.outputs.version }}
      - name: Mark Amplitude Release
        run: |
          bash .github/scripts/bump_amplitude_release.sh $VERSION $ENVIRONMENT
        env:
          VERSION: ${{ steps.release_pypi.outputs.version }}
          ENVIRONMENT: ${{ steps.release_pypi.outputs.sentry_env || 'production' }}
          AMPLITUDE_API_KEY: ${{ secrets.AMPLITUDE_PROD_API_KEY }}
          AMPLITUDE_SECRET_KEY: ${{ secrets.AMPLITUDE_PROD_SECRET_KEY }}
      - name: Notify release to Slack
        id: slack
        uses: slackapi/slack-github-action@v1.19.0
        with:
          payload: |
            {
              "text": ":chai_dog::chai_dog::chai_dog:*PipeRider Release*:chai_dog::chai_dog::chai_dog:\n*Version*: ${{ steps.release_pypi.outputs.version }}\n*GitHub Release*: <https://github.com/InfuseAI/piperider/releases/tag/v${{ steps.release_pypi.outputs.version }}|v${{ steps.release_pypi.outputs.version }}>"
            }
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_API_TOKEN }}
    outputs:
      piperider_version: ${{ steps.release_pypi.outputs.version }}
      sentry_env: ${{ steps.release_pypi.outputs.sentry_env || 'production' }}

  deploy-docker:
    runs-on: ubuntu-latest
    needs: [ build ]
    strategy:
      fail-fast: false
    steps:
      - name: Checkout code
        uses: actions/checkout@v2

      - name: Login to Docker Hub
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}

      - name: Sleep for 30 seconds
        run: |
          sleep 30

      - name: Build & deploy Docker images
        run: |
          echo "Build & deploy Docker images: piperider:$VERSION"
          make docker-deploy VERSION=$VERSION
          if [ "$SENTRY_ENV" == "production" ]; then
            echo "Set piperider:$VERSION as piperider:latest"
            make docker-deploy-latest VERSION=$VERSION
          fi
        env:
          VERSION: ${{ needs.build.outputs.piperider_version }}
          SENTRY_ENV: ${{ needs.build.outputs.sentry_env }}

  generate-sample-report:
    runs-on: ubuntu-latest
    needs: [ build ]
    strategy:
      fail-fast: false
    steps:
      - name: Checkout code
        uses: actions/checkout@v2

      - name: Sleep for 30 seconds
        run: |
          sleep 30

      - name: Generate Sample Report by Repository Dispatch
        uses: peter-evans/repository-dispatch@v2
        with:
          token: ${{ secrets.PIPERIDER_INTERNAL_PROJECTS_GITHUB_TOKEN }}
          repository: InfuseAI/git-repo-analytics
          event-type: generate-sample-report
          client-payload: '{"piperider_version": "${{ needs.build.outputs.piperider_version }}"}'


Run tests

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Run tests

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.11
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
          pip install -e '.[duckdb]'
      - name: Run tests
        run: |
          make test
      - name: Upload coverage reports to Codecov
        uses: codecov/codecov-action@v3
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}


Run tox tests

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions

name: Run tox tests

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: "3.10"
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
          pip install -e '.[duckdb]'
          pip install tox
      - name: Run tox-sqlalchemy
        run: tox -c tox-sqlalchemy.ini
      - name: Run tox-ruamel
        run: tox -c tox-ruamel.ini
      - name: Upload coverage reports to Codecov
        uses: codecov/codecov-action@v3
        env:
          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}


CI

name: CI

on:
  # We run CI on pushes to the main branch
  push:
    branches:
      - main
  # and on all pull requests to the main branch
  pull_request:
    branches:
      - main
  # as well as upon manual triggers through the 'Actions' tab of the Github UI
  workflow_dispatch:

jobs:
  build-and-test:
    name: Testing on ${{matrix.os}} Py${{matrix.python-version}}- OpenMP ${{matrix.openmp}}
    runs-on: ${{matrix.os}}
    strategy:
      matrix:
        os: [ubuntu-20.04, macos-11, windows-2022]
        python-version: ["3.8", "3.12"]
        openmp: ['ON', 'OFF']

    steps:
    - uses: actions/checkout@v4
      with:
        submodules: 'recursive'

    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install OpenMP
      if: runner.os == 'macOS' && matrix.openmp == 'ON'
      run: brew install libomp

    - name: make build directory
      run: cmake -E make_directory ${{runner.workspace}}/build

    - name: configure cmake
      shell: bash
      working-directory: ${{runner.workspace}}/build
      run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=Debug -DBUILD_DOCS=OFF -DBUILD_BENCHMARKS=ON -DPY4DGEO_WITH_OPENMP=${{matrix.openmp}}

    - name: build
      shell: bash
      working-directory: ${{runner.workspace}}/build
      run: cmake --build .

    - name: Install Python package and development requirements
      run: |
        python -m pip install -r requirements-dev.txt
        python -m pip install -v .

    - name: Install py4dgeo test data
      shell: bash
      run: |
        mkdir -p $GITHUB_WORKSPACE/tests/data
        copy_py4dgeo_test_data $GITHUB_WORKSPACE/tests/data

    - name: run tests
      shell: bash
      working-directory: ${{runner.workspace}}/build
      run: ctest --output-on-failure -C Debug

    - name: Run Python tests
      run: |
        python -m pytest --nbval

  coverage-test:
    name: Coverage Testing
    runs-on: ubuntu-20.04

    steps:
    - uses: actions/checkout@v4
      with:
        submodules: 'recursive'

    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: 3.8

    - name: Install LCov
      run: |
        sudo apt-get install -y lcov

    - name: Install Python package and development requirements
      run: |
        python -m pip install -r requirements-dev.txt
        python -m pip install -v --editable .

    - name: Install py4dgeo test data
      shell: bash
      run: |
        mkdir -p $GITHUB_WORKSPACE/tests/data
        copy_py4dgeo_test_data $GITHUB_WORKSPACE/tests/data

    - name: make build directory
      run: cmake -E make_directory ${{runner.workspace}}/build

    - name: configure cmake
      shell: bash
      working-directory: ${{runner.workspace}}/build
      run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="--coverage" -DBUILD_DOCS=OFF

    - name: build
      shell: bash
      working-directory: ${{runner.workspace}}/build
      run: cmake --build .

    - name: run tests
      shell: bash
      working-directory: ${{runner.workspace}}/build
      run: ctest -C Debug

    - name: collect coverage report
      shell: bash
      working-directory: ${{runner.workspace}}
      run: |
        lcov --directory ./build --capture --output-file coverage.info
        lcov_cobertura coverage.info -o coverage2.xml

    - name: Run coverage tests
      run: |
        python -m pytest --cov=src --cov-report=xml

    - name: Upload coverage to Codecov.io
      working-directory: ${{runner.workspace}}
      run: |
        curl --connect-timeout 10 --retry 5 -Os https://uploader.codecov.io/latest/linux/codecov
        chmod +x codecov
        ./codecov -f py4dgeo/coverage.xml -F python
        ./codecov -f coverage2.xml -F cxx

  address-sanitizer:
    name: Address sanitizer run
    runs-on: ubuntu-20.04
    env:
      ASAN_OPTIONS: "detect_stack_use_after_return=1:check_initialization_order=1:strict_init_order=1"

    steps:
    - uses: actions/checkout@v4
      with:
        submodules: 'recursive'

    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: 3.9

    - name: Install Python development requirements
      run: |
        python -m pip install -r requirements-dev.txt
        python -m pip install -v .

    - name: Install py4dgeo test data
      shell: bash
      run: |
        mkdir -p $GITHUB_WORKSPACE/tests/data
        copy_py4dgeo_test_data $GITHUB_WORKSPACE/tests/data

    - name: make build directory
      run: cmake -E make_directory ${{runner.workspace}}/build

    - name: configure cmake
      shell: bash
      working-directory: ${{runner.workspace}}/build
      run: cmake $GITHUB_WORKSPACE -DCMAKE_BUILD_TYPE=Debug -DCMAKE_CXX_FLAGS="-fsanitize=address -fsanitize-address-use-after-scope -fno-omit-frame-pointer" -DCMAKE_EXE_LINKER_FLAGS="-fsanitize=address -fsanitize-address-use-after-scope -fno-omit-frame-pointer" -DBUILD_DOCS=OFF

    - name: build
      shell: bash
      working-directory: ${{runner.workspace}}/build
      run: cmake --build .

    - name: run c++ tests
      shell: bash
      working-directory: ${{runner.workspace}}/build
      run: ctest --rerun-failed --output-on-failure -C Debug

    - name: run python tests
      shell: bash
      working-directory: ${{runner.workspace}}/py4dgeo
      # for LD_PRELOAD see https://github.com/google/sanitizers/issues/934#issuecomment-649516500
      run: |
        mv ../build/_py4dgeo.*.so src/py4dgeo/.
        echo "leak:/usr/bin/bash" > supp.txt
        echo "leak:_PyObject_New" >> supp.txt
        echo "leak:_PyObject_GC" >> supp.txt
        echo "leak:insert_to_emptydict" >> supp.txt
        echo "leak:new_keys_object" >> supp.txt
        echo "leak:PyDict_Copy" >> supp.txt
        echo "leak:PyUnicode_New" >> supp.txt
        echo "leak:pyo3::types::function::PyCFunction::internal_new_from_pointers" >> supp.txt
        echo "leak:pyo3::types::function::PyCFunction::internal_new::" >> supp.txt
        # hack to prevent external libs from dlclosing libraries,
        # which otherwise results in <module not found> LSAN leaks that cannot be suppressed
        # https://github.com/google/sanitizers/issues/89#issuecomment-406316683
        echo "#include <stdio.h>" > dlclose.c
        echo "int dlclose(void *handle) { return 0; }" >> dlclose.c
        clang -shared dlclose.c -o libdlclose.so
        LSAN_OPTIONS=suppressions="$(pwd)/supp.txt" PYTHONPATH=src LD_PRELOAD="$(gcc -print-file-name=libasan.so) /usr/lib/x86_64-linux-gnu/libstdc++.so.6 $(pwd)/libdlclose.so" PYTHONMALLOC=malloc pytest -s


Build Wheels + PyPI deploy

name: Build Wheels + PyPI deploy

on:
  workflow_dispatch:
    inputs:
      deploy_to_testpypi:
        description: "Whether the build should be deployed to test.pypi.org"
        required: true
        default: "false"
      deploy_to_pypi:
        description: "Whether the build should be deployed to pypi.org"
        required: true
        default: "true"

jobs:
  build-wheels:
    name: Build wheels on ${{ matrix.os }}
    runs-on: ${{ matrix.os }}

    strategy:
      matrix:
        include:
          - os: ubuntu-20.04
            arch: "x86_64"
          - os: windows-2022
            arch: "AMD64"
          - os: macos-11
            arch: "x86_64"

    steps:
    - uses: actions/checkout@v4
      with:
        submodules: 'recursive'

    - name: Build wheels (arch=${{ matrix.arch }})
      uses: pypa/cibuildwheel@v2.17
      env:
        CIBW_ARCHS: "${{ matrix.arch }}"

    - uses: actions/upload-artifact@v4
      with:
        path: ./wheelhouse/*.whl

  build-sdist:
    name: Build source distribution
    runs-on: ubuntu-20.04

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: 'recursive'

      - name: Build SDist
        run: pipx run build --sdist

      - uses: actions/upload-artifact@v4
        with:
          name: artifact
          path: dist/*.tar.gz

  test-sdist:
    name: Test source distribution on ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    needs: [build-sdist]
    strategy:
      matrix:
        os:
          - ubuntu-20.04
          - windows-2022
          - macos-11

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: 'recursive'

      - uses: actions/setup-python@v5
        name: Install Python
        with:
          python-version: '3.9'

      - uses: actions/download-artifact@v4
        with:
          name: artifact
          path: dist

      - name: Install OpenMP
        if: runner.os == 'macOS'
        run: brew install libomp

      - name: Install from SDist
        shell: bash
        run:
          python -m pip install dist/*.tar.gz

      - name: Install test requirements
        run:
          python -m pip install -r requirements-dev.txt

      - name: Run test suite
        run:
          python -m pytest

  upload_testpypi:
    needs: [build-sdist, test-sdist, build-wheels]
    runs-on: ubuntu-20.04
    permissions:
      id-token: write

    steps:
      - uses: actions/download-artifact@v4
        with:
          name: artifact
          path: dist

      - uses: pypa/gh-action-pypi-publish@release/v1
        if: github.event.inputs.deploy_to_testpypi == 'true'
        with:
          repository_url: https://test.pypi.org/legacy/

  upload_pypi:
    needs: [build-sdist, build-wheels, upload_testpypi]
    runs-on: ubuntu-20.04
    permissions:
      id-token: write

    steps:
      - uses: actions/download-artifact@v4
        with:
          name: artifact
          path: dist

      - uses: pypa/gh-action-pypi-publish@release/v1
        if: github.event.inputs.deploy_to_pypi == 'true'


Database Iterators Standalone

name: Database Iterators Standalone

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
  workflow_dispatch:

jobs:
  build:
    name: Build and Test
    runs-on: ubuntu-22.04
    env:
      LLVM_SYMBOLIZER_PATH: /usr/lib/llvm-14/bin/llvm-symbolizer
    steps:
    - name: Set up Python
      uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
      with:
        python-version: 3.11

    - name: Checkout project
      uses: actions/checkout@v3
      with:
        path: sandbox

    - name: Install Ninja
      uses: llvm/actions/install-ninja@6a57890d0e3f9f35dfc72e7e48bc5e1e527cdd6c # Jan 17

    - name: Install doxygen
      run: |
        sudo apt-get install -y doxygen graphviz

    - name: Define build dir
      run: |
        echo "DATABASE_ITERATORS_STANDALONE_BUILD_DIR=${GITHUB_WORKSPACE}/sandbox/examples/database-iterators-standalone/build" >> $GITHUB_ENV

    - name: Setup CMake
      run: |
        mkdir ${DATABASE_ITERATORS_STANDALONE_BUILD_DIR}
        cd ${DATABASE_ITERATORS_STANDALONE_BUILD_DIR}
        cmake -G Ninja ..

    - name: Build
      run: |
        cd ${DATABASE_ITERATORS_STANDALONE_BUILD_DIR}
        ninja
        ninja DatabaseIteratorsDocs

    - name: Test
      run: |
        cd ${DATABASE_ITERATORS_STANDALONE_BUILD_DIR}
        ninja test


Structured Build and Test

name: Structured Build and Test

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
  workflow_dispatch:

jobs:
  build:
    name: Structured Build and Test (Release Asserts)
    runs-on: ubuntu-22.04
    env:
      LLVM_SYMBOLIZER_PATH: /usr/lib/llvm-14/bin/llvm-symbolizer
    steps:
    - name: Compute Structured base path
      run: |
        echo "STRUCTURED_MAIN_SRC_DIR=${GITHUB_WORKSPACE}/sandbox" | tee -a $GITHUB_ENV
        echo "STRUCTURED_MAIN_BINARY_DIR=${GITHUB_WORKSPACE}/sandbox/build" | tee -a $GITHUB_ENV
        echo "LLVM_SYSPATH=${GITHUB_WORKSPACE}/sandbox/build" | tee -a $GITHUB_ENV
        echo "MLIR_RUNNER_UTILS_LIB=${GITHUB_WORKSPACE}/sandbox/build/lib/libmlir_runner_utils.so" | tee -a $GITHUB_ENV
        echo "MLIR_C_RUNNER_UTILS_LIB=${GITHUB_WORKSPACE}/sandbox/build/lib/libmlir_c_runner_utils.so" | tee -a $GITHUB_ENV

    - name: Set up Python
      uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
      with:
        python-version: 3.11

    - name: Cache git folder
      uses: actions/cache@v3
      with:
        path: sandbox/.git
        key: git-folder

    - name: Install protobuf
      run: |
        sudo apt-get install -y protobuf-compiler libprotobuf-dev

    - name: Install libcurl
      run: |
        sudo apt-get install -y libcurl4-gnutls-dev

    - name: Checkout project
      uses: actions/checkout@v3
      with:
        path: sandbox
        submodules: recursive

    - name: Install Ninja
      uses: llvm/actions/install-ninja@6a57890d0e3f9f35dfc72e7e48bc5e1e527cdd6c # Jan 17

    - name: Ccache for C++ compilation
      uses: hendrikmuhs/ccache-action@621a41397ed83711c72862638d9ff6e63fca3041 # v1.2.3
      with:
        key: ${{ runner.os }}-structured
        # LLVM needs serious cache size
        max-size: 6G

    - name: Install Python depends
      run: |
        cd ${STRUCTURED_MAIN_SRC_DIR}
        python -m pip install -v -r requirements.txt

    - name: Configure CMake
      run: |
        cmake \
          -DPython3_EXECUTABLE=$(which python) \
          -DBUILD_SHARED_LIBS=ON \
          -DCMAKE_EXPORT_COMPILE_COMMANDS=TRUE \
          -DCMAKE_BUILD_TYPE=RelWithDebInfo \
          -DCMAKE_C_COMPILER:FILEPATH=clang \
          -DCMAKE_CXX_COMPILER:FILEPATH=clang++ \
          -DLLVM_ENABLE_PROJECTS="mlir;clang;clang-tools-extra" \
          -DLLVM_TARGETS_TO_BUILD="X86" \
          -DLLVM_ENABLE_ASSERTIONS=ON \
          -DLLVM_INCLUDE_TESTS=OFF \
          -DLLVM_INCLUDE_UTILS=ON \
          -DLLVM_INSTALL_UTILS=ON \
          -DLLVM_LIT_ARGS=-v \
          -DLLVM_EXTERNAL_PROJECTS=structured \
          -DLLVM_EXTERNAL_STRUCTURED_SOURCE_DIR=${STRUCTURED_MAIN_SRC_DIR} \
          -DLLVM_ENABLE_LLD=ON \
          -DLLVM_CCACHE_BUILD=ON \
          -DMLIR_INCLUDE_INTEGRATION_TESTS=ON \
          -DMLIR_ENABLE_BINDINGS_PYTHON=ON \
          -DMLIR_ENABLE_PYTHON_BENCHMARKS=ON \
          -S${STRUCTURED_MAIN_SRC_DIR}/third_party/llvm-project/llvm \
          -B${STRUCTURED_MAIN_BINARY_DIR} -G Ninja
        echo "PYTHONPATH=${PYTHONPATH}:${STRUCTURED_MAIN_BINARY_DIR}/tools/structured/python_packages" | tee -a $GITHUB_ENV

    - name: Build main project
      run: |
        cmake --build ${STRUCTURED_MAIN_BINARY_DIR} --target structured-all
        ccache -s

    - name: Run lit tests
      run: |
        cmake --build ${STRUCTURED_MAIN_BINARY_DIR} --target check-structured

    - name: Run benchmarks
      shell: bash # This enables the `-eo pipefail` flag to propagate script failures.
      run: |
        ${GITHUB_WORKSPACE}/sandbox/benchmarks/inner_product/run.sh test | tee /tmp/result.jsonl
        ${GITHUB_WORKSPACE}/sandbox/benchmarks/inner_product/plot.sh -i /tmp/result.jsonl

    - name: Archive benchmark plots
      uses: actions/upload-artifact@v3
      with:
        name: structured-benchmark-plots
        path: |
          sandbox/benchmarks/**/*.pdf


name: check_formatting
on: [push, pull_request]
jobs:
  formatting_job:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: Set up conda
        uses: conda-incubator/setup-miniconda@v2
        with:
          miniforge-version: latest
          miniforge-variant: mambaforge
          channel-priority: strict
          channels: conda-forge
          show-channel-urls: true
          use-only-tar-bz2: true

      - name: Install dependencies and check formatting
        shell: bash -l {0}
        run:
          mamba install --quiet --yes --file requirements.txt black &&
          black --version &&
          black tobac --check --diff


Check Zenodo JSON Formatting

name: Check Zenodo JSON Formatting
on: [push, pull_request]
jobs:
  check-json-formatting:
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -el {0}
    steps:
      - name: check out repository code
        uses: actions/checkout@v3
      - name: set up conda environment
        uses: conda-incubator/setup-miniconda@v2
        with:
          auto-update-conda: true
          auto-activate-base: false
          activate-environment: checkjson-env
      - name: Install check-jsonschema
        run: |
          pip install check-jsonschema
      - name: Check zenodo JSON formatting
        run: |
          check-jsonschema --schemafile https://zenodraft.github.io/metadata-schema-zenodo/latest/schema.json .zenodo.json


Jupyter Notebooks CI

name: Jupyter Notebooks CI
on: [push, pull_request]
jobs:
  Check-Notebooks:
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -el {0}
    steps:
      - name: check out repository code
        uses: actions/checkout@v3
      - name: set up conda environment
        uses: conda-incubator/setup-miniconda@v2
        with:
          miniforge-version: latest
          miniforge-variant: mambaforge
          channel-priority: strict
          channels: conda-forge
          show-channel-urls: true
          use-only-tar-bz2: true
          auto-update-conda: true
          auto-activate-base: false
          activate-environment: notebook-env
      - name: Install tobac dependencies
        run: |
          mamba install -c conda-forge --yes ffmpeg gcc jupyter pytables
          mamba install -c conda-forge --yes --file example_requirements.txt
      - name: Install tobac
        run: |
          pip install .
      - name: Find all notebook files
        run: |
          find . -type f -name '*.ipynb' > nbfiles.txt
          cat nbfiles.txt
      - name: Execute all notebook files
        run: |
          while IFS= read -r nbpath; do
            jupyter nbconvert --inplace --ClearMetadataPreprocessor.enabled=True --clear-output $nbpath
            jupyter nbconvert --to notebook --inplace --execute $nbpath
          done < nbfiles.txt


Codecov CI

name: Codecov CI

on: [push, pull_request]

jobs:
  run:
    runs-on: ubuntu-latest
    env:
      OS: ubuntu-latest
      PYTHON: "3.9"
    steps:
      - uses: actions/checkout@v2
      # Similar to MetPy install-conda action
      - name: Set up conda
        uses: conda-incubator/setup-miniconda@v2
        with:
          miniforge-version: latest
          miniforge-variant: mambaforge
          channel-priority: strict
          channels: conda-forge
          show-channel-urls: true
          use-only-tar-bz2: true

      - name: Install dependencies and generate report
        shell: bash -l {0}
        run:
          mamba install --quiet --yes --file requirements.txt coverage pytest-cov &&
          python -m coverage run -m pytest --cov=./ --cov-report=xml
      - name: Upload Coverage to Codecov
        uses: codecov/codecov-action@v3
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          fail_ci_if_error: true
          flags: unittests


Docker Image CI

name: Docker Image CI

on: [push, pull_request]

jobs:

  build:

    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v2
    - name: Build the Docker image
      run: docker build . --file tobac/tests/Dockerfile --tag my-image-name:$(date +%s)


Matrix Testing CI

name: Matrix Testing CI
# this is similar to the pyart CI action
on: [push, pull_request]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

# This job installs dependencies and runs tests across a matrix of python and OS versions.
#Add if: github.repository == 'tobac-project/tobac' to limit runs to tobac repo

jobs:
  build:
    name: ${{ matrix.os }}-${{ matrix.python-version }}
    runs-on: ${{ matrix.os }}-latest
    if: github.repository == 'tobac-project/tobac'
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11"]
        os: [macos, ubuntu, windows]

    steps:
      - uses: actions/checkout@v2

      # Install micromamba and dependencies
      - name: Setup Conda Environment
        uses: mamba-org/setup-micromamba@v1
        with:
          environment-file: environment-ci.yml
          activate-environment: pyart-dev
          cache-downloads: true
          channels: conda-forge
          channel-priority: strict
          python-version: ${{ matrix.python-version }}

      - name: Fetch all history for all tags and branches
        run: |
           git fetch --prune --unshallow

      - name: Install tobac
        shell: bash -l {0}
        run: |
          python -m pip install -e . --no-deps --force-reinstall

      - name: Run Tests
        id: run_tests
        shell: bash -l {0}
        run: |
          python -m pytest -v


Linting

name: Linting
on:
  pull_request_target:
    branches:
      - '*'
permissions:
  pull-requests: write

jobs:
  build:
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    steps:
      - name: Check out Git repository
        uses: actions/checkout@v3

      - name: Set up conda
        uses: conda-incubator/setup-miniconda@v2
        with:
          miniforge-version: latest
          miniforge-variant: mambaforge
          channel-priority: strict
          channels: conda-forge
          show-channel-urls: true
          use-only-tar-bz2: true

      - name: Install tobac and pylint
        run: |
          mamba install --yes pylint
          pip install .

      - name: Store the PR branch
        run: |
          echo "SHA=$(git rev-parse "$GITHUB_SHA")" >> $GITHUB_OUTPUT
        id: git

      - name: Checkout RC branch
        uses: actions/checkout@v3
        with:
          ref: ${{ github.base_ref }}

      - name: Get pylint score of RC branch
        run: |
          pylint tobac --disable=C --exit-zero
        id: main_score

      - name: Checkout PR branch
        uses: actions/checkout@v3
        with:
          ref: "${{ steps.git.outputs.SHA }}"

      - name: Get pylint score of PR branch
        run: |
          # use shell script to save only tail of output
          OUTPUT_PART=$(pylint tobac --disable=C --exit-zero | tail -n 2)
          # but post entire output in the action details
          pylint tobac --disable=C --exit-zero
          # define random delimiter for multiline string
          EOF=$(dd if=/dev/urandom bs=15 count=1 status=none | base64)
          echo "MESSAGE<<$EOF" >> "$GITHUB_OUTPUT"
          echo "$OUTPUT_PART" >> "$GITHUB_OUTPUT"
          echo "$EOF" >> "$GITHUB_OUTPUT"
        id: pr_score

      - name: Find Comment
        uses: peter-evans/find-comment@v2
        id: comment
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-author: 'github-actions[bot]'
          body-includes: Linting results by Pylint

      - name: Post result to PR
        uses: peter-evans/create-or-update-comment@v3
        with:
          issue-number: ${{ github.event.pull_request.number }}
          comment-id: ${{ steps.comment.outputs.comment-id }}
          edit-mode: replace
          body: |
            Linting results by Pylint:
            --------------------------
            ${{ steps.pr_score.outputs.MESSAGE }}
            <sub>The linting score is an indicator that reflects how well your code version follows Pylint’s coding standards and quality metrics with respect to the ${{ github.base_ref }} branch.
            A decrease usually indicates your new code does not fully meet style guidelines or has potential errors.<sup>


name: Check if installing from conda-forge works
on:
  workflow_dispatch:

jobs:
  Install-miniconda:
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      matrix:
        python-version: ["3.10", "3.11"]
    steps:
      - uses: actions/checkout@v4
      - uses: conda-incubator/setup-miniconda@v3
        with:
          auto-update-conda: true
          activate-environment: cellpy_dev
          environment-file: github_actions_environment.yml
          python-version: ${{ matrix.python-version }}

      - name: Conda info
        run: conda info

      - name: Conda list
        run: conda list

      - name: create conda environment
        run: |
          conda create -n cellpy_conda
          conda activate cellpy_conda
          conda install -c conda-forge cellpy

      - name: check cellpy
        run: |
          conda activate cellpy_conda
          cellpy setup -s


joss pdf

name: joss pdf
on:
  workflow_dispatch:

jobs:
  paper:
    runs-on: ubuntu-latest
    name: Paper Draft
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - name: Build draft PDF
        uses: openjournals/openjournals-draft-action@master
        with:
          journal: joss
          # This should be the path to the paper within your repo.
          paper-path: paper/paper.md
      - name: Upload
        uses: actions/upload-artifact@v3
        with:
          name: paper
          # This is the output path where Pandoc will write the compiled
          # PDF. Note, this should be the same directory as the input
          # paper.md
          path: paper/paper.pdf


Installing using pip on Mac M chip

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python

name: Installing using pip on Mac M chip

on:
  push:
    branches:
      - "**"
  pull_request:
    branches: [ "master" ]
  workflow_dispatch:

jobs:
  build:

    runs-on: macos-14
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.10", "3.11"]

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install mdbtools
      run: |
        brew install mdbtools
    - name: Install unixodbc
      run: |
            brew install unixodbc
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install pytest pytest-timeout pytest-benchmark lmfit
        python -m pip install cython
        brew install hdf5
        brew install c-blosc
        export HDF5_DIR=/opt/homebrew/opt/hdf5
        export BLOSC_DIR=/opt/homebrew/opt/c-blosc
        python -m pip install tables
        # if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
    - name: Install cellpy
      run: |
        python -m pip install -e .
    - name: Test with pytest
      run: |
        python -m pytest
    - name: Run cellpy setup
      run: |
        cellpy setup --silent
    - name: Run the cellpy check
      run: |
        cellpy info --check
    - name: Show the configuration
      run: |
        cellpy info -p


Installing using pip on posix

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python

name: Installing using pip on posix

on:
  push:
    branches:
      - "**"
  pull_request:
    branches: [ "master" ]
  workflow_dispatch:

jobs:
  build:

    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.10", "3.11"]

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install mdbtools
      run: |
        sudo apt-get update
        sudo apt-get install -y mdbtools
#        sudo apt-get install -y libhdf5-serial-dev
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install pytest pytest-timeout pytest-benchmark lmfit
        # if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
    - name: Install cellpy
      run: |
        python -m pip install -e .
    - name: Test with pytest
      run: |
        pytest
    - name: Run cellpy setup
      run: |
        cellpy setup --silent
    - name: Run the cellpy check
      run: |
        cellpy info --check
    - name: Show the configuration
      run: |
        cellpy info -p


Installing using pip on windows

# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python

name: Installing using pip on windows

on:
  push:
    branches:
      - "**"
  pull_request:
    branches: [ "master" ]
  workflow_dispatch:

jobs:
  build:

    runs-on: windows-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.10", "3.11"]

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install pytest pytest-timeout pytest-benchmark lmfit
        # if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
    - name: Install cellpy
      run: |
        python -m pip install -e .
    - name: Test with pytest
      run: |
        pytest tests/test_maccor.py
    - name: Run cellpy setup
      run: |
        cellpy setup --silent
    - name: Run the cellpy check
      run: |
        cellpy info --check
    - name: Show the configuration
      run: |
        cellpy info -p


Run pytest on macos (conda)

name: Run pytest on macos (conda)
on:
  push:
    branches:
      - "**"
  pull_request:
    branches: [ master ]
  workflow_dispatch:

jobs:
  Install-miniconda-and-run-pytest:
    runs-on: macos-latest
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      matrix:
        python-version: ["3.10", "3.11"]
    steps:
      - uses: actions/checkout@v3
      - uses: conda-incubator/setup-miniconda@v2
        with:
          auto-update-conda: true
          miniconda-version: latest
          activate-environment: cellpy_dev
          environment-file: github_actions_environment.yml
          python-version: ${{ matrix.python-version }}

      - name: Conda info
        run: conda info

      - name: Conda list
        run: conda list

      - name: Install mdbtools
        run: |
          brew install mdbtools
#          sudo apt-get install -y mdbtools

      - name: pytest
        run: pytest


Run pytest on macos M (conda)

name: Run pytest on macos M (conda)
on:
  push:
    branches:
      - "**"
  pull_request:
    branches: [ master ]
  workflow_dispatch:

jobs:
  Install-miniconda-and-run-pytest:
    runs-on: macos-14
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      matrix:
        python-version: ["3.10", "3.11"]
    steps:
      - uses: actions/checkout@v4
      - uses: conda-incubator/setup-miniconda@v3
        with:
          auto-update-conda: true
          miniconda-version: latest
          activate-environment: cellpy_dev
          environment-file: github_actions_environment.yml
          python-version: ${{ matrix.python-version }}

      - name: Conda info
        run: conda info

      - name: Conda list
        run: conda list

      - name: Install mdbtools
        run: |
          brew install mdbtools
#          sudo apt-get install -y mdbtools

      - name: pytest
        run: pytest