Pytest Testing with PyMC3

name: Pytest Testing with PyMC3
on:
  pull_request:
    types: [opened,reopened,synchronize]
  push:
    branches:
      - 'main'
      - 'releases/**'
jobs:
  test:
    name: pytest
    runs-on: 'ubuntu-latest'
    env:
      OS: 'ubuntu-latest'
      PYTHON: '3.10'
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{ github.event.pull_request.head.ref }}
      - uses: actions/setup-python@v5
        with:
          python-version: '3.10.14'
      - name: Setup Environment
        run: |
          python -m pip install --upgrade pip
          pip install .[hst,pymc3,test]
          conda install mkl-service
          mkdir $HOME/crds_cache
          echo "CRDS_PATH=$HOME/crds_cache" >> $GITHUB_ENV
          echo "CRDS_SERVER_URL=https://jwst-crds.stsci.edu" >> $GITHUB_ENV
      - name: Log the current Eureka version
        run: pip show eureka
      - name: Run tests, generate coverage report
        run: pytest tests --cov=./src/eureka --cov-report=xml
      - name: Upload coverage reports to Codecov with GitHub Action
        uses: codecov/codecov-action@v4
        with:
          env_vars: OS,PYTHON
          files: ./coverage.xml
          fail_ci_if_error: true
          verbose: true
        env:
          CODECOV_TOKEN: ${{secrets.CODECOV_TOKEN}}


CodeQL

name: "CodeQL"

on:
  push:
    branches: [ 'master' ]
  pull_request:
    # The branches below must be a subset of the branches above
    branches: [ 'master' ]
  schedule:
    - cron: '38 19 * * 5'

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write

    strategy:
      fail-fast: false
      matrix:
        language: [ 'python' ]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
        # Use only 'java' to analyze code written in Java, Kotlin or both
        # Use only 'javascript' to analyze code written in JavaScript, TypeScript or both
        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v2
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
        # By default, queries listed here will override any specified in a config file.
        # Prefix the list here with "+" to use these queries and those in the config file.

        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
        # queries: security-extended,security-and-quality

    # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
      uses: github/codeql-action/autobuild@v2

    # ℹ Command-line programs to run using the OS shell.
    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun

    #   If the Autobuild fails above, remove it and uncomment the following three lines.
    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.

    # - run: |
    #     echo "Run, Build Application using script"
    #     ./location_of_script_within_repo/buildscript.sh

    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v2
      with:
        category: "/language:${{matrix.language}}"


codespell

name: codespell
on: [pull_request, push]
jobs:
  codespell:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - run: python3 -m pip install codespell
      - run: codespell --ignore-words-list="ba,fo,hel,revered,womens"
          --skip="./README.*.md,*.svg,*.ai,./benchmarks/snippets.py,./tests,./tools"


issues

name: issues
on:
  issues:
    types: [closed]
jobs:
  add-comment:
    runs-on: ubuntu-latest
    permissions:
      issues: write
    steps:
      - name: Did I solve your problem?
        uses: peter-evans/create-or-update-comment@a35cf36e5301d70b76f316e867e7788a55a31dae
        with:
          issue-number: ${{ github.event.issue.number }}
          body: |
            I hope we solved your problem.

            If you like using Rich, you might also enjoy [Textual](https://textual.textualize.io)


name: issues
on:
  issues:
    types: [opened]
jobs:
  add-comment:
    runs-on: ubuntu-latest
    permissions:
      issues: write
    steps:
      - uses: actions/checkout@v4
        with:
          ref: master
      - name: Install FAQtory
        run: pip install FAQtory
      - name: Run Suggest
        env:
          TITLE: ${{ github.event.issue.title }}
        run: faqtory suggest "$TITLE" > suggest.md
      - name: Read suggest.md
        id: suggest
        uses: juliangruber/read-file-action@v1
        with:
          path: ./suggest.md
      - name: Suggest FAQ
        uses: peter-evans/create-or-update-comment@a35cf36e5301d70b76f316e867e7788a55a31dae
        with:
          issue-number: ${{ github.event.issue.number }}
          body: ${{ steps.suggest.outputs.content }}


Test Rich module

name: Test Rich module

on: [pull_request]

jobs:
  build:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [windows-latest, ubuntu-latest, macos-latest]
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
        include:
          - { os: ubuntu-latest, python-version: "3.7" }
          - { os: windows-latest, python-version: "3.7" }
          - { os: macos-12, python-version: "3.7" }
    defaults:
      run:
        shell: bash
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install and configure Poetry
        # TODO: workaround for https://github.com/snok/install-poetry/issues/94
        uses: snok/install-poetry@v1.3.3
        with:
          version: 1.3.1
          virtualenvs-in-project: true
      - name: Install dependencies
        run: poetry install
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
      - name: Format check with black
        run: |
          source $VENV
          make format-check
      - name: Typecheck with mypy
        run: |
          source $VENV
          make typecheck
      - name: Test with pytest (with coverage)
        run: |
          source $VENV
          pytest tests -v --cov=./rich --cov-report=xml:./coverage.xml --cov-report term-missing
      - name: Upload code coverage
        uses: codecov/codecov-action@v3
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          file: ./coverage.xml
          name: rich
          flags: unittests
          env_vars: OS,PYTHON


README.md Changed

name: README.md Changed

on:
  push:
    branches:
      - master
    paths:
      - 'README.md'

jobs:
  send_notification:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - name: Send notification to README Authors
      env:
        GITHUB_TOKEN: ${{ secrets.GHP_README_WORKFLOW }}
        GIT_SHA: ${{ github.sha }}
      run: |
        COMMIT=$(git rev-parse --short "$GIT_SHA")
        AUTHORS='@willmcgugan @oleksis @Adilius'
        BODY="🤓 $AUTHORS README.md changed 📝. Check the [commit $COMMIT](https://github.com/willmcgugan/rich/commit/$GIT_SHA) 👀"
        DISCUSSIONID='MDEwOkRpc2N1c3Npb24zMzI2NzM0'
        gh api graphql -H 'GraphQL-Features: discussions_api' -f body="$BODY" -F discussionId="$DISCUSSIONID" -f query='mutation($body: String!, $discussionId: ID!){addDiscussionComment(input:{body: $body , discussionId: $discussionId}){comment{id}}}'


# Run the workflow that adds new tickets that are either:
# - labelled "DEPR"
# - title starts with "[DEPR]"
# - body starts with "Proposal Date" (this is the first template field)
# to the org-wide DEPR project board

name: Add newly created DEPR issues to the DEPR project board

on:
  issues:
    types: [opened]

jobs:
  routeissue:
    uses: openedx/.github/.github/workflows/add-depr-ticket-to-depr-board.yml@master
    secrets:
      GITHUB_APP_ID: ${{ secrets.GRAPHQL_AUTH_APP_ID }}
      GITHUB_APP_PRIVATE_KEY: ${{ secrets.GRAPHQL_AUTH_APP_PEM }}
      SLACK_BOT_TOKEN: ${{ secrets.SLACK_ISSUE_BOT_TOKEN }}


# This workflow runs when a comment is made on the ticket
# If the comment starts with "label: " it tries to apply
# the label indicated in rest of comment.
# If the comment starts with "remove label: ", it tries
# to remove the indicated label.
# Note: Labels are allowed to have spaces and this script does
# not parse spaces (as often a space is legitimate), so the command
# "label: really long lots of words label" will apply the
# label "really long lots of words label"

name: Allows for the adding and removing of labels via comment

on:
  issue_comment:
    types: [created]

jobs:
  add_remove_labels:
    uses: openedx/.github/.github/workflows/add-remove-label-on-comment.yml@master


Python CI

name: Python CI

on:
  push:
    branches: [master]
  pull_request:
    branches:
    - '**'

jobs:
  run_tests:
    name: tests
    runs-on: ubuntu-20.04
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.8', '3.11', '3.12']
        toxenv: [quality, django42]

    steps:
    - uses: actions/checkout@v3
    - name: setup python
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install pip
      run: pip install -r requirements/pip.txt

    - name: Install Dependencies
      run: pip install -r requirements/ci.txt

    - name: Run Tests
      run: tox -e ${{ matrix.toxenv }}

    - name: Run Coverage
      if: matrix.python-version == '3.8' && matrix.toxenv == 'django42'
      uses: codecov/codecov-action@v4
      with:
        flags: unittests
        fail_ci_if_error: true
        token: ${{ secrets.CODECOV_TOKEN }}


Lint Commit Messages

# Run commitlint on the commit messages in a pull request.

name: Lint Commit Messages

on:
  - pull_request

jobs:
  commitlint:
    uses: openedx/.github/.github/workflows/commitlint.yml@master


Publish package to PyPI

name: Publish package to PyPI

on:
  release:
    types: [published]

jobs:

  push:
    runs-on: ubuntu-20.04

    steps:
      - name: Checkout
        uses: actions/checkout@v2
      - name: setup python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8

      - name: Install pip
        run: pip install wheel setuptools

      - name: Build package
        run: python setup.py sdist bdist_wheel

      - name: Publish to PyPI
        uses: pypa/gh-action-pypi-publish@master
        with:
          user: __token__
          password: ${{ secrets.PYPI_UPLOAD_TOKEN }}


# This workflow runs when a comment is made on the ticket
# If the comment starts with "assign me" it assigns the author to the
# ticket (case insensitive)

name: Assign comment author to ticket if they say "assign me"
on:
  issue_comment:
    types: [created]

jobs:
  self_assign_by_comment:
    uses: openedx/.github/.github/workflows/self-assign-issue.yml@master


Upgrade Python Requirements

name: Upgrade Python Requirements

on:
  schedule:
    - cron: "0 0 * * 0"
  workflow_dispatch:
    inputs:
      branch:
        description: "Target branch against which to create requirements PR"
        required: true
        default: 'master'

jobs:
  call-upgrade-python-requirements-workflow:
    uses: openedx/.github/.github/workflows/upgrade-python-requirements.yml@master
    # Do not run on forks
    if: github.repository_owner == 'openedx'
    with:
      branch: ${{ github.event.inputs.branch || 'master' }}
      # optional parameters below; fill in if you'd like github or email notifications
      # user_reviewers: ""
      team_reviewers: "axim-aximprovements"
      email_address: "aximimprovements@axim.org"
      send_success_notification: false
    secrets:
      requirements_bot_github_token: ${{ secrets.REQUIREMENTS_BOT_GITHUB_TOKEN }}
      requirements_bot_github_email: ${{ secrets.REQUIREMENTS_BOT_GITHUB_EMAIL }}
      edx_smtp_username: ${{ secrets.EDX_SMTP_USERNAME }}
      edx_smtp_password: ${{ secrets.EDX_SMTP_PASSWORD }}


Upload Python Package

name: Upload Python Package

on:
  release:
    types: [published]

jobs:
  deploy:

    runs-on: ubuntu-20.04

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.x'
    - name: Install dependencies
      run: |
        pip install -U pip
        pip install setuptools wheel twine
    - name: Build and publish
      env:
        TWINE_USERNAME: __token__
        TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
      run: |
        python setup.py sdist bdist_wheel
        twine upload dist/*


Build and test GAM

name: Build and test GAM

on:
  push:
  pull_request:
  schedule:
    - cron: '37 22 * * *'

permissions:
  contents: read
  id-token: write
  attestations: write

defaults:
  run:
    shell: bash
    working-directory: src

env:
  SCRATCH_COUNTER: 1
  OPENSSL_CONFIG_OPTS: no-fips --api=3.0.0
  OPENSSL_INSTALL_PATH: ${{ github.workspace }}/bin/ssl
  OPENSSL_SOURCE_PATH: ${{ github.workspace }}/src/openssl
  PYTHON_INSTALL_PATH: ${{ github.workspace }}/bin/python
  PYTHON_SOURCE_PATH: ${{ github.workspace }}/src/cpython

jobs:
  build:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: ubuntu-20.04
            jid: 1
            goal: build
            arch: x86_64
            openssl_archs: linux-x86_64
          - os: [self-hosted, linux, arm64]
            jid: 2
            goal: build
            arch: aarch64
            openssl_archs: linux-aarch64
          - os: ubuntu-20.04
            jid: 3
            goal: build
            arch: x86_64
            openssl_archs: linux-x86_64
            staticx: yes
          - os: [self-hosted, linux, arm64]
            jid: 4
            goal: build
            arch: aarch64
            openssl_archs: linux-aarch64
            staticx: yes
          - os: macos-12
            jid: 5
            goal: build
            arch: x86_64
            openssl_archs: darwin64-x86_64
          - os: macos-14
            jid: 6
            goal: build
            arch: aarch64
            openssl_archs: darwin64-arm64
          - os: macos-14
            jid: 7
            goal: build
            arch: universal2
            openssl_archs: darwin64-arm64 darwin64-x86_64
          - os: windows-2022
            jid: 8
            goal: build
            arch: Win64
            openssl_archs: VC-WIN64A
          - os: ubuntu-22.04
            goal: test
            python: "3.8"
            jid: 9
            arch: x86_64
          - os: ubuntu-22.04
            goal: test
            python: "3.9"
            jid: 10
            arch: x86_64
          - os: ubuntu-22.04
            goal: test
            python: "3.10"
            jid: 11
            arch: x86_64
          - os: ubuntu-22.04
            goal: test
            python: "3.11"
            jid: 12
            arch: x86_64

    steps:

      - uses: actions/checkout@v4
        with:
          persist-credentials: false
          fetch-depth: 0

      - id: auth
        name: Authenticate to Google Cloud
        uses: google-github-actions/auth@v2
        with:
          workload_identity_provider: projects/297925809119/locations/global/workloadIdentityPools/gha-pool/providers/gha-provider
          service_account: github-actions-testing-for-gam@gam-project-wyo-lub-ivl.iam.gserviceaccount.com

      - name: Cache multiple paths
        if: matrix.goal == 'build'
        uses: actions/cache@v4
        id: cache-python-ssl
        with:
          path: |
            cache.tar.xz
          key: gam-${{ matrix.jid }}-20240620

      - name: Untar Cache archive
        if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit == 'true'
        working-directory: ${{ github.workspace }}
        run: |
          tar xvvf cache.tar.xz

      - name: Use pre-compiled Python for testing
        if: matrix.python != ''
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
          allow-prereleases: true

      - name: common variables for all runs
        env:
          arch: ${{ matrix.arch }}
          JID: ${{ matrix.jid }}
          ACTIONS_CACHE: ${{ steps.cache-python-ssl.outputs.cache-hit }}
          ACTIONS_GOAL: ${{ matrix.goal }}
        run: |
          echo "arch=${arch}" >> $GITHUB_ENV
          echo "JID=${JID}" >> $GITHUB_ENV
          echo "ACTIONS_CACHE=${ACTIONS_CACHE}" >> $GITHUB_ENV
          echo "ACTIONS_GOAL=${ACTIONS_GOAL}" >> $GITHUB_ENV
          curl_version=$(curl --version | head -n 1 | awk '{ print $2 }')
          echo "cURL is ${curl_version}"
          if [ "$curl_version" == "7.68.0" ]; then
            export curl_retry="--retry 5 --retry-connrefused"
          else
            export curl_retry="--retry 5 --retry-all-errors"
          fi
          echo "curl_retry=${curl_retry}" >> $GITHUB_ENV
          # GAMCFGDIR should be recreated on every run
          GAMCFGDIR="${RUNNER_TEMP}/.gam"
          if [ "$arch" == "Win64" ]; then
            GAMCFGDIR=$(cygpath -u "$GAMCFGDIR")
          fi
          echo "GAMCFGDIR=${GAMCFGDIR}" >> $GITHUB_ENV
          echo "GAMCFGDIR is: ${GAMCFGDIR}"
          if [[ "${RUNNER_OS}" == "macOS" ]]; then
            GAMOS="macos"
          elif [[ "${RUNNER_OS}" == "Linux" ]]; then
            GAMOS="linux"
          elif [[ "${RUNNER_OS}" == "Windows" ]]; then
            GAMOS="windows"
          else
           GAMOS='unknown'
          fi
          echo "GAMOS=${GAMOS}" >> $GITHUB_ENV
          echo "GAMOS is: ${GAMOS}"

      - name: Set env variables for test
        if: matrix.goal == 'test'
        run: |
          export PYTHON=$(which python3)
          export PIP=$(which pip3)
          export gam="${PYTHON} -m gam"
          export gampath="$(readlink -e .)"
          echo -e "PYTHON: ${PYTHON}\nPIP: ${PIP}\gam: ${gam}\ngampath: ${gampath}"
          echo "PYTHON=${PYTHON}" >> $GITHUB_ENV
          echo "PIP=${PIP}" >> $GITHUB_ENV
          echo "gam=${gam}" >> $GITHUB_ENV
          echo "gampath=${gampath}" >> $GITHUB_ENV

      - name: Install necessary Github-hosted Linux packages
        if: runner.os == 'Linux' && runner.arch == 'X64'
        run: |
          echo "RUNNING: apt update..."
          sudo apt-get -qq --yes update
          sudo apt-get -qq --yes install swig libpcsclite-dev libxslt1-dev

      - name: MacOS install tools
        if: runner.os == 'macOS'
        run: |
          # Install latest Rust
          curl $curl_retry -fsS -o rust.sh https://sh.rustup.rs
          bash ./rust.sh -y
          source $HOME/.cargo/env
          # Install needed packages
          #brew update
          #brew install gpg
          #brew install swig
          #brew install ncurses

      - name: Windows Configure VCode
        uses: ilammy/msvc-dev-cmd@v1
        if: runner.os == 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
        with:
          arch: ${{ matrix.arch }}

      - name: Set Env Variables for build
        if: matrix.goal == 'build'
        env:
          openssl_archs: ${{ matrix.openssl_archs }}
          staticx: ${{ matrix.staticx }}
        run: |
          echo "We are running on ${RUNNER_OS}"
          LD_LIBRARY_PATH="${OPENSSL_INSTALL_PATH}/lib:${PYTHON_INSTALL_PATH}/lib:/usr/local/lib"
          if [[ "${arch}" == "Win64" ]]; then
            PYEXTERNALS_PATH="amd64"
            PYBUILDRELEASE_ARCH="x64"
            GAM_ARCHIVE_ARCH="x86_64"
            WIX_ARCH="x64"
            CHOC_OPS=""
          fi
          if [[ "${RUNNER_OS}" == "macOS" ]]; then
            MAKE=make
            MAKEOPT="-j$(sysctl -n hw.logicalcpu)"
            PERL=perl
            MACOSX_DEPLOYMENT_TARGET=$(sw_vers -productVersion | awk -F '.' '{print $1 "." $2}')
            echo "MACOSX_DEPLOYMENT_TARGET=${MACOSX_DEPLOYMENT_TARGET}" >> $GITHUB_ENV
            echo "We are running on and targetting MacOS ${MACOSX_DEPLOYMENT_TARGET}"
            echo "PYTHON=${PYTHON_INSTALL_PATH}/bin/python3" >> $GITHUB_ENV
          elif [[ "${RUNNER_OS}" == "Linux" ]]; then
            MAKE=make
            MAKEOPT="-j$(nproc)"
            PERL=perl
            echo "PYTHON=${PYTHON_INSTALL_PATH}/bin/python3" >> $GITHUB_ENV
          elif [[ "${RUNNER_OS}" == "Windows" ]]; then
            MAKE=nmake
            MAKEOPT=""
            PERL="c:\strawberry\perl\bin\perl.exe"
            LD_LIBRARY_PATH="${LD_LIBRARY_PATH}:${PYTHON_SOURCE_PATH}/PCbuild/${PYEXTERNALS_PATH}"
            echo "PYTHON=${PYTHON_SOURCE_PATH}/PCbuild/${PYEXTERNALS_PATH}/python.exe" >> $GITHUB_ENV
            echo "GAM_ARCHIVE_ARCH=${GAM_ARCHIVE_ARCH}" >> $GITHUB_ENV
            echo "WIX_ARCH=${WIX_ARCH}" >> $GITHUB_ENV
          fi
          echo "We'll run make with: ${MAKEOPT}"
          echo "staticx=${staticx}" >> $GITHUB_ENV
          echo "LD_LIBRARY_PATH=${LD_LIBRARY_PATH}" >> $GITHUB_ENV
          echo "MAKE=${MAKE}" >> $GITHUB_ENV
          echo "MAKEOPT=${MAKEOPT}" >> $GITHUB_ENV
          echo "PERL=${PERL}" >> $GITHUB_ENV
          echo "PYEXTERNALS_PATH=${PYEXTERNALS_PATH}" >> $GITHUB_ENV
          echo "PYBUILDRELEASE_ARCH=${PYBUILDRELEASE_ARCH}" >> $GITHUB_ENV
          echo "openssl_archs=${openssl_archs}" >> $GITHUB_ENV

      - name: Get latest stable OpenSSL source
        if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
        run: |
          mkdir -vp "${GITHUB_WORKSPACE}/src"
          cd "${GITHUB_WORKSPACE}/src"
          git clone https://github.com/openssl/openssl.git
          cd "${OPENSSL_SOURCE_PATH}"
          export LATEST_STABLE_TAG=$(git tag --list openssl-* | grep -v alpha | grep -v beta | sort -Vr | head -n1)
          echo "Checking out version ${LATEST_STABLE_TAG}"
          git checkout "${LATEST_STABLE_TAG}"
          export COMPILED_OPENSSL_VERSION=${LATEST_STABLE_TAG:8} # Trim the openssl- prefix
          echo "COMPILED_OPENSSL_VERSION=${COMPILED_OPENSSL_VERSION}" >> $GITHUB_ENV
          if ([ "${RUNNER_OS}" == "macOS" ] && [ "$arch" == "universal2" ]); then
            for openssl_arch in $openssl_archs; do
              ssldir="${OPENSSL_SOURCE_PATH}-${openssl_arch}"
              mkdir -v "${ssldir}"
              cp -vrf ${OPENSSL_SOURCE_PATH}/* "${ssldir}/"
            done
            rm -vrf "${OPENSSL_SOURCE_PATH}"
          else
            mv -v "${OPENSSL_SOURCE_PATH}" "${OPENSSL_SOURCE_PATH}-${openssl_archs}"
          fi

      - name: Windows NASM Install
        uses: ilammy/setup-nasm@v1
        if: matrix.goal == 'build' && runner.os == 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'

      - name: Config OpenSSL
        if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
        run: |
          for openssl_arch in $openssl_archs; do
            cd "${GITHUB_WORKSPACE}/src/openssl-${openssl_arch}"
            # --libdir=lib is needed so Python can find OpenSSL libraries
            "${PERL}" ./Configure "${openssl_arch}" --libdir=lib --prefix="${OPENSSL_INSTALL_PATH}" $OPENSSL_CONFIG_OPTS
          done

      - name: Rename GNU link on Windows
        if: matrix.goal == 'build' && runner.os == 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
        shell: bash
        run: mv /usr/bin/link /usr/bin/gnulink

      - name: Make OpenSSL
        if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
        run: |
          for openssl_arch in $openssl_archs; do
            cd "${GITHUB_WORKSPACE}/src/openssl-${openssl_arch}"
            $MAKE "${MAKEOPT}"
          done

      - name: Install OpenSSL
        if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
        run: |
          if ([ "${RUNNER_OS}" == "macOS" ] && [ "$arch" == "universal2" ]); then
            for openssl_arch in $openssl_archs; do
              cd "${GITHUB_WORKSPACE}/src/openssl-${openssl_arch}"
              # install_sw saves us ages processing man pages :-)
              $MAKE install_sw
              mv "${OPENSSL_INSTALL_PATH}" "${GITHUB_WORKSPACE}/bin/ssl-${openssl_arch}"
            done
            mkdir -vp "${OPENSSL_INSTALL_PATH}/lib"
            mkdir -vp "${OPENSSL_INSTALL_PATH}/bin"
            for archlib in libcrypto.3.dylib libssl.3.dylib libcrypto.a libssl.a; do
              lipo -create "${GITHUB_WORKSPACE}/bin/ssl-darwin64-x86_64/lib/${archlib}" \
                           "${GITHUB_WORKSPACE}/bin/ssl-darwin64-arm64/lib/${archlib}" \
                   -output "${GITHUB_WORKSPACE}/bin/ssl/lib/${archlib}"
            done
            mv ${GITHUB_WORKSPACE}/bin/ssl-darwin64-x86_64/include ${GITHUB_WORKSPACE}/bin/ssl/
            lipo -create "${GITHUB_WORKSPACE}/bin/ssl-darwin64-x86_64/bin/openssl" \
                         "${GITHUB_WORKSPACE}/bin/ssl-darwin64-arm64/bin/openssl" \
                 -output "${GITHUB_WORKSPACE}/bin/ssl/bin/openssl"
            rm -rf ${GITHUB_WORKSPACE}/bin/ssl-darwin64-x86_64
            rm -rf ${GITHUB_WORKSPACE}/bin/ssl-darwin64-arm64
          else
            cd "${GITHUB_WORKSPACE}/src/openssl-${openssl_archs}"
            # install_sw saves us ages processing man pages :-)
            $MAKE install_sw
          fi
          echo "LDFLAGS=-L${OPENSSL_INSTALL_PATH}/lib" >> $GITHUB_ENV
          echo "CRYPTOGRAPHY_SUPPRESS_LINK_FLAGS=1" >> $GITHUB_ENV
          case $arch in
            universal2)
              echo "CFLAGS=-I${OPENSSL_INSTALL_PATH}/include -arch arm64 -arch x86_64 ${CFLAGS}" >> $GITHUB_ENV
              echo "ARCHFLAGS=-arch x86_64 -arch arm64" >> $GITHUB_ENV
              ;;
            x86_64)
              echo "CFLAGS=-I${OPENSSL_INSTALL_PATH}/include ${CFLAGS}" >> $GITHUB_ENV
              echo "ARCHFLAGS=-arch x86_64" >> $GITHUB_ENV
              ;;
            aarch64)
              echo "CFLAGS=-I${OPENSSL_INSTALL_PATH}/include ${CFLAGS}" >> $GITHUB_ENV
              echo "ARCHFLAGS=-arch arm64" >> $GITHUB_ENV
              ;;
          esac

      - name: Run OpenSSL
        if: matrix.goal == 'build'
        run: |
          "${OPENSSL_INSTALL_PATH}/bin/openssl" version
          "${OPENSSL_INSTALL_PATH}/bin/openssl" version -f
          file "${OPENSSL_INSTALL_PATH}/bin/openssl"

      - name: Get latest stable Python source
        if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
        run: |
          cd "${GITHUB_WORKSPACE}/src"
          git clone https://github.com/python/cpython.git
          cd "${PYTHON_SOURCE_PATH}"
          # Pin Windows to 3.11.6 for the moment
          # if [[ "${RUNNER_OS}" == "Windows" ]]; then
          #   export LATEST_STABLE_TAG="v3.11.6"
          # else
          export LATEST_STABLE_TAG=$(git tag --list | grep -v a | grep -v rc | grep -v b | sort -Vr | head -n1)
          # fi
          git checkout "${LATEST_STABLE_TAG}"
          export COMPILED_PYTHON_VERSION=${LATEST_STABLE_TAG:1} # Trim the "v" prefix
          echo "COMPILED_PYTHON_VERSION=${COMPILED_PYTHON_VERSION}" >> $GITHUB_ENV

      - name: Mac/Linux Configure Python
        if: matrix.goal == 'build' && runner.os != 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
        run: |
          cd "${PYTHON_SOURCE_PATH}"
          if ([ "${RUNNER_OS}" == "macOS" ] && [ "$arch" == "universal2" ]); then
            extra_args=( "--enable-universalsdk" "--with-universal-archs=universal2" )
          else
            extra_args=( )
          fi
          ./configure --with-openssl="${OPENSSL_INSTALL_PATH}" \
                      --prefix="${PYTHON_INSTALL_PATH}" \
                      --enable-shared \
                      --with-ensurepip=upgrade \
                      --enable-optimizations \
                      --with-lto \
                      "${extra_args[@]}" || : # exit 0
          cat config.log

      - name: Windows Get External Python deps
        if: matrix.goal == 'build' && runner.os == 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
        shell: powershell
        run: |
          cd "${env:PYTHON_SOURCE_PATH}"
          PCBuild\get_externals.bat

      - name: Windows overwrite external OpenSSL with local
        if: matrix.goal == 'build' && runner.os == 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
        shell: powershell
        run: |
          cd "${env:PYTHON_SOURCE_PATH}"
          $env:OPENSSL_EXT_PATH = "$(Get-Item externals\openssl-bin-* | Select -exp FullName)\"
          echo "External OpenSSL was downloaded to ${env:OPENSSL_EXT_PATH}"
          Remove-Item -recurse -force "${env:OPENSSL_EXT_PATH}*"
          # Emulate what this script does:
          # https://github.com/python/cpython/blob/main/PCbuild/openssl.vcxproj
          $env:OPENSSL_EXT_TARGET_PATH = "${env:OPENSSL_EXT_PATH}${env:PYEXTERNALS_PATH}"
          echo "Copying our OpenSSL to ${env:OPENSSL_EXT_TARGET_PATH}"
          mkdir "${env:OPENSSL_EXT_TARGET_PATH}\include\openssl\"
          Copy-Item -Path "${env:GITHUB_WORKSPACE}/src/openssl-${env:openssl_archs}\LICENSE.txt" -Destination "${env:OPENSSL_EXT_TARGET_PATH}\LICENSE" -Verbose
          cp -v "$env:OPENSSL_INSTALL_PATH\lib\*" "${env:OPENSSL_EXT_TARGET_PATH}"
          cp -v "$env:OPENSSL_INSTALL_PATH\bin\*" "${env:OPENSSL_EXT_TARGET_PATH}"
          cp -v "$env:OPENSSL_INSTALL_PATH\include\openssl\*" "${env:OPENSSL_EXT_TARGET_PATH}\include\openssl\"
          cp -v "$env:OPENSSL_INSTALL_PATH\include\openssl\applink.c" "${env:OPENSSL_EXT_TARGET_PATH}\include\"

      - name: Windows Install sphinx-build
        if: matrix.goal == 'build' && runner.os == 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
        shell: powershell
        run: |
          pip install --upgrade pip
          pip install --upgrade sphinx
          sphinx-build --version

      - name: Windows Config/Build Python
        if: matrix.goal == 'build' && runner.os == 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
        shell: powershell
        run: |
          cd "${env:PYTHON_SOURCE_PATH}"
          # We need out custom openssl.props which uses OpenSSL 3 DLL names
          Copy-Item -Path "${env:GITHUB_WORKSPACE}\src\tools\openssl.props" -Destination PCBuild\ -Verbose
          echo "Building for ${env:PYBUILDRELEASE_ARCH}..."
          PCBuild\build.bat -m --pgo -c Release -p "${env:PYBUILDRELEASE_ARCH}"

      - name: Mac/Linux Build Python
        if: matrix.goal == 'build' && runner.os != 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
        run: |
          cd "${PYTHON_SOURCE_PATH}"
          echo "Running: ${MAKE} ${MAKEOPT}"
          $MAKE $MAKEOPT

      - name: Mac/Linux Install Python
        if: matrix.goal == 'build' && runner.os != 'Windows' && steps.cache-python-ssl.outputs.cache-hit != 'true'
        run: |
          cd "${PYTHON_SOURCE_PATH}"
          $MAKE altinstall
          $MAKE bininstall
          export PATH="${PATH}:${PYTHON_INSTALL_PATH}/bin"
          echo "PATH=${PATH}" >> $GITHUB_ENV
          echo "PATH: ${PATH}"

      - name: Run Python
        run: |
          "${PYTHON}" -V

      - name: Upgrade pip, wheel, etc
        run: |
          curl $curl_retry -O https://bootstrap.pypa.io/get-pip.py
          "${PYTHON}" get-pip.py
          "${PYTHON}" -m pip install --upgrade pip
          "${PYTHON}" -m pip install --upgrade wheel
          "${PYTHON}" -m pip install --upgrade setuptools

      - name: Install pip requirements
        run: |
          echo "before anything..."
          "${PYTHON}" -m pip list
          if ([ "${RUNNER_OS}" == "macOS" ] && [ "$arch" == "universal2" ]); then
            # cffi is a dep of cryptography and doesn't ship
            # a universal2 wheel so we must build one ourself :-/
            export CFLAGS="-arch x86_64 -arch arm64"
            export ARCHFLAGS="-arch x86_64 -arch arm64"
            "${PYTHON}" -m pip install --upgrade --force-reinstall --no-binary :all: \
                              --no-cache-dir --no-deps --use-pep517 \
                              --use-feature=no-binary-enable-wheel-cache \
                              cffi
            echo "before cryptography..."
            "${PYTHON}" -m pip list
            # cryptography has a universal2 wheel but getting it installed
            # on x86-64 MacOS is a royal pain in the keester.
            "${PYTHON}" -m pip download --only-binary :all: \
                                          --dest . \
                                          --no-cache \
                                          --no-deps \
                                          --platform macosx_10_15_universal2 \
                                          cryptography
            "${PYTHON}" -m pip install --force-reinstall --no-deps cryptography*.whl
            echo "after cryptography..."
            "${PYTHON}" -m pip list
            "${PYTHON}" -m pip install --upgrade --no-binary :all: -r requirements.txt
          else
            "${PYTHON}" -m pip install --upgrade -r requirements.txt
            echo "after requirements..."
            "${PYTHON}" -m pip list
            "${PYTHON}" -m pip install --force-reinstall --no-deps --upgrade cryptography
          fi
          echo "after everything..."
          "${PYTHON}" -m pip list

      - name: Install PyInstaller
        if: matrix.goal == 'build'
        run: |
          git clone https://github.com/pyinstaller/pyinstaller.git
          cd pyinstaller
          export latest_release=$(git tag --list | grep -v dev | grep -v rc | sort -Vr | head -n1)
          #V6.0.0 causes errors on staticx
          #if [[ "${staticx}" == "yes" ]]; then
          #  git checkout "v5.13.2"
          #elif [[ "${RUNNER_OS}" == "Windows" ]]; then
          #  git checkout "v5.13.2"
          #elif [[ "${RUNNER_OS}" == "macOS" ]]; then
          #  git checkout "v5.13.2"
          #else
          git checkout "${latest_release}"
          #fi
          # remove pre-compiled bootloaders so we fail if bootloader compile fails
          rm -rvf PyInstaller/bootloader/*-*/*
          cd bootloader
          export PYINSTALLER_BUILD_ARGS=""
          case "${arch}" in
            "Win64")
              export PYINSTALLER_BUILD_ARGS="--target-arch=64bit"
              ;;
          esac
          echo "PyInstaller build arguments: ${PYINSTALLER_BUILD_ARGS}"
          "${PYTHON}" ./waf all $PYINSTALLER_BUILD_ARGS
          cd ..
          echo "---- Installing PyInstaller ----"
          "${PYTHON}" -m pip install .

      - name: Build GAM with PyInstaller
        if: matrix.goal != 'test'
        run: |
          if [[ "${staticx}" == "yes" ]]; then
            export distpath="./dist/gam"
            export gampath="${distpath}"
          else
            export distpath="./dist"
            export gampath="${distpath}/gam"
          fi
          mkdir -p -v "${gampath}"
          if [[ "${RUNNER_OS}" == "macOS" ]]; then
            export gampath=$($PYTHON -c "import os; print(os.path.realpath('$gampath'))")
          elif [[ "${RUNNER_OS}" == "Windows" ]]; then
            # Work around issue where PyInstaller picks up python3.dll from other Python versions
            # https://github.com/pyinstaller/pyinstaller/issues/7102
            export PATH="$(dirname ${PYTHON}):/usr/bin"
          else
            export gampath=$(realpath "${gampath}")
          fi
          export gam="${gampath}/gam"
          echo "gampath=${gampath}" >> $GITHUB_ENV
          # TEMP force everything back to one file.
          export PYINSTALLER_BUILD_ONEFILE="yes"
          export distpath="./dist/gam"
          export gampath="${distpath}"
          "${PYTHON}" -m PyInstaller --clean --noconfirm --distpath="${distpath}" gam.spec
          cat build/gam/warn-gam.txt
          if [ -x "$(command -v realpath)" ]; then
               realpath=realpath
             else
               brew install coreutils
               realpath=grealpath
             fi
          export gam=$(realpath "$gam")
          if [[ "${RUNNER_OS}" == "Windows" ]]; then
            export gam=$(cygpath -w "$gam")
            echo "GAM on Windows at ${gam}"
          fi
          echo "gam=${gam}" >> $GITHUB_ENV
          echo -e "GAM: ${gam}\nGAMPATH: ${gampath}"

      - name: Copy extra package files
        if: matrix.goal == 'build'
        run: |
          cp -v cacerts.pem $gampath
          cp -v LICENSE $gampath
          cp -v GamCommands.txt $gampath
          cp -v GamUpdate.txt $gampath
          if [[ "${RUNNER_OS}" == "Windows" ]]; then
              cp -v gam-setup.bat $gampath
          fi

      - name: Install StaticX
        if: matrix.staticx == 'yes'
        run: |
          "${PYTHON}" -m pip install --upgrade patchelf-wrapper
          "${PYTHON}" -m pip install --upgrade staticx

      - name: Make StaticX
        if: matrix.staticx == 'yes'
        run: |
          case $RUNNER_ARCH in
            X64)
              ldlib=/lib/x86_64-linux-gnu/ld-linux-x86-64.so.2
              ;;
            ARM64)
              ldlib=/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1
              ;;
          esac
          echo "ldlib=${ldlib}"
          $PYTHON -m staticx -l "${ldlib}" "${gam}" "${gam}-staticx"
          rm -v "${gam}"
          mv -v "${gam}-staticx" "${gam}"

      - name: Basic Tests all jobs
        id: basictests
        run: |
          $PYTHON -m unittest discover --start-directory ./ --pattern "*_test.py" --buffer || if [ $? != 5 ]; then exit $?; fi # exit 5 is no tests
          $gam version extended nooffseterror
          export GAMVERSION=$($gam version simple)
          echo "GAM Version ${GAMVERSION}"
          echo "GAMVERSION=${GAMVERSION}" >> $GITHUB_ENV

      - name: Attest Binary Provenance
        uses: actions/attest-build-provenance@v1
        if: matrix.goal == 'build'
        with:
          subject-path: ${{ env.gam }}

      - name: Linux/MacOS package
        if: runner.os != 'Windows' && matrix.goal == 'build'
        run: |
          if [[ "${RUNNER_OS}" == "macOS" ]]; then
            GAM_ARCHIVE="gam-${GAMVERSION}-macos-${arch}.tar.xz"
          elif [[ "${RUNNER_OS}" == "Linux" ]]; then
            if [[ "${staticx}" == "yes" ]]; then
              libver="legacy"
            else
              libver="glibc$(ldd --version | awk '/ldd/{print $NF}')"
            fi
            GAM_ARCHIVE="gam-${GAMVERSION}-linux-$(arch)-${libver}.tar.xz"
          fi
          echo "GAM Archive ${GAM_ARCHIVE}"
          tar -C dist/ --create --verbose --exclude-from "${GITHUB_WORKSPACE}/.github/actions/package_exclusions.txt" --file $GAM_ARCHIVE --xz gam

      - name: Windows package
        if: runner.os == 'Windows' && matrix.goal != 'test'
        run: |
          cd dist/
          GAM_ARCHIVE="../gam-${GAMVERSION}-windows-${GAM_ARCHIVE_ARCH}.zip"
          /c/Program\ Files/7-Zip/7z.exe a -tzip $GAM_ARCHIVE gam "-xr@${GITHUB_WORKSPACE}/.github/actions/package_exclusions.txt" -bb3
          cd ..
          /c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.14/bin/candle.exe -arch "${WIX_ARCH}" gam.wxs
          /c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.14/bin/light.exe -ext /c/Program\ Files\ \(x86\)/WiX\ Toolset\ v3.14/bin/WixUIExtension.dll gam.wixobj -o "gam-${GAMVERSION}-windows-${GAM_ARCHIVE_ARCH}.msi" || true;
          rm -v -f *.wixpdb

      - name: Basic Tests build jobs only
        if: matrix.goal != 'test' && steps.cache-python-ssl.outputs.cache-hit != 'true'
        run: |
          export voutput=$($gam version extended nooffseterror)
          export python_line=$(echo -e "${voutput}" | grep "Python ")
          export python_arr=($python_line)
          export this_python=${python_arr[1]}
          if [[ "${this_python}" != "${COMPILED_PYTHON_VERSION}" ]]; then
            echo "ERROR: Tried to compile Python ${COMPILED_PYTHON_VERSION} but ended up with ${this_python}"
            exit 1
          fi
          export openssl_line=$(echo -e "${voutput}" | grep "OpenSSL ")
          export openssl_arr=($openssl_line)
          export this_openssl="${openssl_arr[1]}"
          if [[ "${this_openssl}" != "${COMPILED_OPENSSL_VERSION}" ]]; then
            echo "ERROR: Tried to compile OpenSSL ${COMPILED_OPENSSL_VERSION} but ended up with ${this_openssl}"
            exit 1
          fi
          echo "We successfully compiled Python ${this_python} and OpenSSL ${this_openssl}"

      - name: Live API tests push only
        if: (github.event_name == 'push' || github.event_name == 'schedule')
        env:
          PASSCODE: ${{ secrets.PASSCODE }}
        run: |
          source ../.github/actions/decrypt.sh ../.github/actions/creds.tar.xz.gpg creds.tar.xz "${GAMCFGDIR}"
          mv -v "${GAMCFGDIR}/oauth2.txt-gam-gha-${JID}" "${GAMCFGDIR}/oauth2.txt"
          rm -v $GAMCFGDIR/oauth2.txt-gam*
          export gam_user="gam-gha-${JID}@pdl.jaylee.us"
          echo "gam_user=${gam_user}" >> $GITHUB_ENV
          $gam config customer_id "C03uzfv2s" save
          $gam config domain "pdl.jaylee.us" save
          $gam config admin_email "${gam_user}" save
          $gam config enable_dasa false save
          $gam oauth info
          $gam oauth refresh
          $gam config enable_dasa true save
          $gam create signjwtserviceaccount
          $gam checkconn
          $gam user "$gam_user" check serviceaccount
          $gam info domain
          $gam info user
          export tstamp=$($PYTHON -c "import time; print(time.time_ns())")
          export newbase="gha_test_${JID}_${tstamp}"
          export newuser="${newbase}@pdl.jaylee.us"
          export newgroup="${newbase}-group@pdl.jaylee.us"
          export newalias="${newbase}-alias@pdl.jaylee.us"
          export newbuilding="${newbase}-building"
          export newresource="${newbase}-resource"
          export newou="aaaGithub Actions/${newbase}"

          # cleanup old runs
          $gam config enable_dasa false save
          $gam config csv_output_row_filter "name:regex:gha_test_${JID}_" print vaultholds || if [ $? != 55 ]; then exit $?; fi | $gam csv - gam delete vaulthold "id:~~holdId~~" matter "id:~~matterId~~"
          $gam config enable_dasa true save
          $gam config csv_output_row_filter "name:regex:gha_test_${JID}_" print features | $gam csv - gam delete feature ~name
          $gam config csv_output_row_filter "name:regex:^gha_test_${JID}_" user $gam_user print shareddrives asadmin | $gam csv - gam user $gam_user delete shareddrive ~id nukefromorbit
          $gam print users query "gha.jid=$JID" | $gam csv - gam delete user ~primaryEmail
          $gam config csv_output_row_filter "name:regex:^gha_test_${JID}_" print ous fromparent "aaaGithub Actions" | $gam csv - gam delete ou ~orgUnitId
          $gam config csv_output_row_filter "email:regex:^gha_test_${JID}_" print cigroups | $gam csv - gam delete cigroup ~email
          $gam config csv_output_row_filter "resourceId:regex:^gha_test_${JID}_" print resources | $gam csv - gam delete resource ~resourceId
          $gam config csv_output_row_filter "buildingId:regex:^gha_test_${JID}_" print buildings | $gam csv - gam delete building ~buildingId
          $gam config csv_output_row_filter "Emails.1.address:regex:^gha_test-${JID}_" print contacts | $gam csv - gam delete contact ~ContactID

          echo "Creating OrgUnit ${newou}"
          $gam create ou "${newou}"
          export GAM_THREADS=5
          echo email > sample.csv;
          for i in {1..10}; do
            echo "${newbase}-bulkuser-$i" >> sample.csv;
          done
          driveid=$($gam user $gam_user add shareddrive "${newbase}" returnidonly)
          echo "Created shared drive ${driveid}"
          $gam create user $newuser firstname GHA lastname $JID displayname "Github Actions ${JID}" password random ou "${newou}" recoveryphone 12125121110 recoveryemail jay0lee@gmail.com gha.jid $JID languages en+,en-GB-
          $gam user $newuser update photo https://dummyimage.com/400x600/000/fff
          $gam user $newuser get photo
          $gam user $newuser delete photo
          $gam create alias $newalias user $newuser
          $gam create group $newgroup name "GHA $JID group" description "This is a description" isarchived true
          $gam user $gam_user sendemail recipient $newuser subject "test message $newbase" message "GHA test message"
          $gam user $gam_user sendemail recipient exchange@pdl.jaylee.us subject "test ${tstamp}" message "test message"
          $gam config enable_dasa false save
          $gam create contact firstname GHA lastname "$JID" email work "${newbase}@example.com" primary
          $gam print contacts
          $gam user $newuser add license workspaceenterpriseplus
          $gam print privileges
          $gam config enable_dasa true save
          $gam update cigroup $newgroup security memberrestriction 'member.type == 1 || member.customer_id == groupCustomerId()'
          $gam info cigroup $newgroup
          $gam update group $newgroup add owner $gam_user
          $gam update group $newgroup add member $newuser
          $gam config enable_dasa false save
          $gam create admin $newuser _GROUPS_EDITOR_ROLE CUSTOMER # condition nonsecuritygroup
          $gam create admin $newgroup _HELP_DESK_ADMIN_ROLE org_unit "${newou}"
          $gam config csv_output_row_filter "assignedToUser:regex:${newuser}" print admins | $gam csv - gam delete admin "~roleAssignmentId"
          $gam config csv_output_row_filter "assignedToGroup:regex:${newgroup}" print admins | $gam csv - gam delete admin "~roleAssignmentId"
          $gam config enable_dasa false save
          $gam csv sample.csv gam create user ~~email~~ firstname "GHA Bulk" lastname ~~email~~ gha.jid $JID ou "${newou}"
          $gam csv sample.csv gam update user ~~email~~ recoveryphone 12125121110 recoveryemail jay0lee@gmail.com password random displayname "GitHub Actions Bulk ${JID}"
          $gam csv sample.csv gam update user ~~email~~ recoveryphone "" recoveryemail ""
          $gam config enable_dasa false save
          $gam csv sample.csv gam user ~email add license workspaceenterpriseplus
          $gam config enable_dasa true save
          $gam csv sample.csv gam user $gam_user sendemail recipient ~~email~~@pdl.jaylee.us subject "test message $newbase" message "GHA test message"
          $gam csv sample.csv gam update group $newgroup add member ~email
          $gam info group $newgroup
          $gam info cigroup $newgroup membertree
          # confirm mailbox is provisoned before continuing
          $gam user $newuser waitformailbox retries 20
          $gam user $newuser imap on
          $gam user $newuser show imap
          $gam user $newuser show delegates
          #$gam user $newuser add contactdelegate "${newbase}-bulkuser-1"
          #$gam user $newuser print contactdelegates
          export biohazard=$(echo -e '\xe2\x98\xa3')
          $gam user $newuser label "$biohazard unicode biohazard $biohazard"
          $gam user $newuser show labels
          $gam user $newuser show labels > labels.txt
          $gam user $gam_user importemail subject "GHA import $newbase" message "This is a test import" labels IMPORTANT,UNREAD,INBOX,STARRED
          $gam user $gam_user insertemail subject "GHA insert $newbase" file gam.py labels INBOX,UNREAD # yep body is gam code
          $gam user $gam_user sendemail subject "GHA send $gam_user $newbase" file gam.py recipient admin@pdl.jaylee.us
          $gam user $gam_user draftemail subject "GHA draft $newbase" message "Draft message test"
          $gam csvfile sample.csv:email waitformailbox retries 20
          $gam user $newuser delegate to "${newbase}-bulkuser-1" || if [ $? != 50 ]; then exit $?; fi # expect a 50 return code (delegation failed)
          $gam users "$gam_user $newbase-bulkuser-1 $newbase-bulkuser-2 $newbase-bulkuser-3" delete messages query in:anywhere maxtodelete 99999 doit || if [ $? != 60 ]; then exit $?; fi # expect a 60 return code (no messages)
          $gam users "$newbase-bulkuser-4 $newbase-bulkuser-5 $newbase-bulkuser-6" trash messages query in:anywhere maxtotrash 99999 doit || if [ $? != 60 ]; then exit $?; fi # expect a 60 return code (no messages)
          $gam users "$newbase-bulkuser-7 $newbase-bulkuser-8 $newbase-bulkuser-9" modify messages query in:anywhere maxtomodify 99999 addlabel IMPORTANT addlabel STARRED doit || if [ $? != 60 ]; then exit $?; fi # expect a 60 return code (no messages)
          $gam user $newuser delete label --ALL_LABELS--
          $gam config csv_output_row_filter "name:regex:gha-test-${JID}" print features | $gam csv - gam delete feature ~name
          $gam create feature name VC-$newbase
          $gam create feature name Whiteboard-$newbase
          $gam create building "My Building - $newbase" id $newbuilding floors 1,2,3,4,5,6,7,8,9,10,11,12,14,15 description "No 13th floor here..."
          $gam create resource $newresource "Resource Calendar $tstamp" capacity 25 features Whiteboard-$newbase,VC-$newbase building $newbuilding floor 15 type Room
          $gam info resource $newresource
          $gam user $newuser add drivefile drivefilename "TPS Reports" mimetype gfolder
          $gam user $newuser show filelist
          $gam calendar $gam_user printacl | $gam csv - gam calendar $gam_user delete ~id # clear ACLs
          $gam calendar $gam_user add read domain
          $gam calendar $gam_user add freebusy default
          $gam calendar $gam_user add editor $newuser
          $gam calendar $gam_user showacl
          $gam calendar $gam_user printacl | $gam csv - gam calendar $gam_user delete ~id
          $gam calendar $gam_user addevent summary "GHA test event" start +1h end +2h attendee $newgroup hangoutsmeet guestscanmodify true sendupdates all
          $gam calendar $gam_user printevents after -0d
          $gam config enable_dasa false save
          matterid=uid:$($gam create vaultmatter name "GHA matter $newbase" description "test matter" collaborators $newuser returnidonly)
          $gam create vaulthold matter $matterid name "GHA hold $newbase" corpus mail accounts $newuser
          $gam print vaultmatters matterstate open
          $gam print vaultholds matter $matterid
          $gam print vaultcount matter $matterid corpus mail everyone todrive tdnobrowser
          $gam create vaultexport matter $matterid name "GHA export $newbase" corpus mail accounts $newuser
          $gam print exports matter $matterid | $gam csv - gam info export $matterid id:~~id~~
          $gam config enable_dasa true save
          $gam csv sample.csv gam user ~email add calendar id:$newresource
          $gam delete resource $newresource
          $gam delete feature Whiteboard-$newbase
          $gam delete feature VC-$newbase
          $gam delete building $newbuilding
          $gam delete group $newgroup
          $gam config enable_dasa false save
          echo start
          $gam user $newuser delete license workspaceenterpriseplus
          echo finish
          $gam config enable_dasa true save
          $gam whatis $newuser || if [ $? != 20 ]; then exit $?; fi # expect a 20 return code (is a user)
          $gam user $gam_user show tokens
          $gam config enable_dasa false save
          download_dir="${RUNNER_TEMP}/TEMP_DELETE_ME"
          mkdir -v "$download_dir"
          $gam print exports matter $matterid | $gam csv - gam download export $matterid id:~~id~~ targetfolder "$download_dir"
          rm -rvf "$download_dir"
          $gam delete hold "GHA hold $newbase" matter $matterid
          $gam update matter $matterid action close
          $gam update matter $matterid action delete
          # shakes off vault hold on user so we can delete
          $gam print users query "email:${newuser}" orgunitpath | $gam csv - gam update user ~primaryEmail ou ~orgUnitPath
          $gam user $newuser show holds || if [ $? != 55 ]; then exit $?; fi # expect a 55 return code
          export sn="$JID$JID$JID$JID-$(openssl rand -base64 32 | sed 's/[^a-zA-Z0-9]//g')"
          $gam create device serialnumber $sn devicetype android
          $gam config enable_dasa true save
          $gam print users query "gha.jid=$JID" | $gam csv - gam delete user ~primaryEmail  || if [ $? != 50 ]; then exit $?; fi # expect a 50 return code (vault hold on user)
          $gam delete contacts emailmatchpattern "^${newbase}@example.com$"
          $gam print mobile
          $gam print devices
          $gam print browsers
          $gam print cros allfields orderby serialnumber
          $gam show crostelemetry storagepercentonly
          $gam report usageparameters customer
          $gam report usage customer parameters gmail:num_emails_sent,accounts:num_1day_logins
          $gam report customer todrive tdnobrowser
          #$gam report users fields accounts:is_less_secure_apps_access_allowed,gmail:last_imap_time,gmail:last_pop_time filters "accounts:last_login_time>2019-01-01T00:00:00.000Z" todrive tdnobrowser
          $gam report users todrive tdnobrowser
          $gam report admin start -3d todrive tdnobrowser
          $gam print devices nopersonaldevices nodeviceusers filter "serial:$JID$JID$JID$JID-" | $gam csv - gam delete device id ~name
          $gam config enable_dasa false save
          $gam print userinvitations
          $gam print userinvitations | $gam csv - gam send userinvitation ~name
          $gam config enable_dasa false save
          $gam create caalevel "zzz_${newbase}" basic condition ipsubnetworks 1.1.1.1/32,2.2.2.2/32 endcondition
          $gam print caalevels
          $gam delete caalevel "zzz_${newbase}"
          $gam user $gam_user add drivefile localfile gam.py parentid "${driveid}"
          $gam user $gam_user update shareddrive "${driveid}" ou "${newou}"
          $gam user $gam_user show shareddrives asadmin
          $gam user $gam_user update shareddrive "${driveid}" ou "aaaGithub Actions" # so we can delete our OU...
          $gam user $gam_user delete shareddrive "${driveid}" nukefromorbit
          echo "printer model count:"
          ssoprofile=$($gam create inboundssoprofile name "El Goog ${newbase}" loginurl https://www.google.com logouturl https://www.google.com changepasswordurl https://www.google.com entityid ElGoog return_name_only)
          $gam create inboundssocredential profile "id:${ssoprofile}" generate_key
          #$gam create inboundssoassignment profile "id:${ssoprofile}" orgunit "${newou}" mode SAML_SSO
          #$gam delete inboundssoassignment "orgunit:${newou}"
          $gam delete inboundssoprofile "id:${ssoprofile}"
          $gam print printermodels | wc -l
          $gam print printers
          printerid=$($gam create printer displayname "${newbase}" uri ipp://localhost:631 driverless description "made by $(gam_user)" ou "${newou}" nodetails | awk '{print substr($2, 1, length($2)-1)}')
          $gam info printer "$printerid"
          $gam delete printer "$printerid"
          $gam delete ou "${newou}"

      - name: Tar Cache archive
        if: matrix.goal == 'build' && steps.cache-python-ssl.outputs.cache-hit != 'true'
        working-directory: ${{ github.workspace }}
        run: |
          if [[ "${RUNNER_OS}" == "Windows" ]]; then
            tar_folders="src/cpython/ bin/ssl"
          else
            tar_folders="bin/"
          fi
          tar cJvvf cache.tar.xz $tar_folders

      - name: Attest Build Archive Provenance
        uses: actions/attest-build-provenance@v1
        if: (github.event_name == 'push' || github.event_name == 'schedule') && matrix.goal == 'build'
        with:
          subject-path: |
            src/gam*.tar.xz
            src/gam*.zip
            src/gam*.msi

      - name: Archive production artifacts
        uses: actions/upload-artifact@v4
        if: (github.event_name == 'push' || github.event_name == 'schedule') && matrix.goal != 'test'
        with:
          name: gam-binaries-${{ env.GAMOS }}-${{ env.arch }}-${{ matrix.jid }}
          path: |
            src/gam*.tar.xz
            src/gam*.zip
            src/gam*.msi

  merge:
    if: (github.event_name == 'push' || github.event_name == 'schedule')
    runs-on: ubuntu-latest
    needs: build
    permissions:
      contents: write
      packages: write
    steps:
      - name: Merge Artifacts
        uses: actions/upload-artifact/merge@v4
        with:
          name: gam-binaries
          pattern: gam-binaries-*

  publish:
    if: github.event_name == 'push'
    runs-on: ubuntu-latest
    needs: merge
    permissions:
      contents: write
      packages: write
      pull-requests: read

    steps:

      - uses: actions/checkout@v4
        with:
          persist-credentials: false
          fetch-depth: 0

      - name: Download artifacts
        uses: actions/download-artifact@v4

      - name: VirusTotal Scan
        uses: crazy-max/ghaction-virustotal@v4
        with:
          vt_api_key: ${{ secrets.VT_API_KEY }}
          files: |
            gam-binaries/*

      - name: Set datetime version string
        id: dateversion
        run: |
          export dateversion="$(date  +'%Y%m%d.%H%M%S')"
          echo "Date version: ${dateversion}"
          echo "dateversion=${dateversion}" >> $GITHUB_OUTPUT

      - uses: "marvinpinto/action-automatic-releases@latest"
        name: Publish draft release
        with:
          repo_token: "${{ secrets.GITHUB_TOKEN }}"
          automatic_release_tag: "${{ steps.dateversion.outputs.dateversion }}"
          prerelease: false
          draft: true
          files: |
            gam-binaries/*


CodeQL

# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"

on:
  push:
    branches: [ main ]
  pull_request:
    # The branches below must be a subset of the branches above
    branches: [ main ]
  schedule:
    - cron: '25 10 * * 1'

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write

    strategy:
      fail-fast: false
      matrix:
        language: [ 'python' ]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
        # Learn more about CodeQL language support at https://git.io/codeql-language-support

    steps:
    - name: Checkout repository
      uses: actions/checkout@v3

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v2
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
        # By default, queries listed here will override any specified in a config file.
        # Prefix the list here with "+" to use these queries and those in the config file.
        # queries: ./path/to/local/query, your-org/your-repo/queries@main

    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
      uses: github/codeql-action/autobuild@v2

    # ℹ Command-line programs to run using the OS shell.
    # 📚 https://git.io/JvXDl

    # ✏ If the Autobuild fails above, remove it and uncomment the following three lines
    #    and modify them (or add more) to build your code if your project
    #    uses a compiled language

    #- run: |
    #   make bootstrap
    #   make release

    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v2


Check for Google Root CA Updates

name: Check for Google Root CA Updates

on:
  push:
  pull_request:
  schedule:
    - cron: '23 23 * * *'

defaults:
  run:
    shell: bash
    working-directory: src

jobs:
  check-apis:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@master
        with:
          persist-credentials: false # otherwise, the token used is the GITHUB_TOKEN, instead of your personal token
          fetch-depth: 0 # otherwise, you will failed to push refs to dest repo

      - name: Check for updates
        run: curl -o ./cacerts.pem -vvvv https://pki.goog/roots.pem

      - name: Commit file
        run: |
          git config --local user.email "action@github.com"
          git config --local user.name "GitHub Action"
          git add cacerts.pem
          git diff --quiet && git diff --staged --quiet || git commit -am '[ci skip] Updated cacerts.pem'

      - name: Push changes
        uses: ad-m/github-push-action@master
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}


name: Apply 'untriaged' label during issue lifecycle

on:
  issues:
    types: [opened, reopened, transferred]

jobs:
  apply-label:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/github-script@v6
        with:
          script: |
            github.rest.issues.addLabels({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              labels: ['untriaged']
            })


name: Publish OpenSearch Benchmark Docker Image
on:
  push:
    tags:
      - "v*.*.*"
  workflow_dispatch:
    inputs:
      logLevel:
        description: Log level
        required: true
        default: warning
        type: choice
        options:
          - info
          - warning
          - debug

jobs:
  docker:
    runs-on: ubuntu-latest
    steps:
      -
        name: Set up QEMU
        uses: docker/setup-qemu-action@v2
      -
        name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1
      -
        name: Login to DockerHub
        uses: docker/login-action@v1
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      -
        name: Build and push
        uses: docker/build-push-action@v2
        with:
          platforms: linux/amd64,linux/arm64
          file: docker/Dockerfile
          context: "{{defaultContext}}:."
          push: true
          tags: opensearchstaging/opensearch-benchmark:latest,opensearchstaging/opensearch-benchmark:0.2.0


dockerfile-lint

---
name: dockerfile-lint

on:
  pull_request:
    paths:
      - 'docker/**'

jobs:
  test_docker:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: jbergstroem/hadolint-gh-action@v1
        with:
          dockerfile: 'docker/*Dockerfile'
          annotate: false


CI Actions

name: CI Actions
on: [pull_request]
jobs:
  Unit-Tests:
    strategy:
      matrix:
        os:
          - ubuntu-latest
          - macos-latest
    runs-on: ${{ matrix.os }}
    steps:
      - name: Check out repository code
        uses: actions/checkout@v2
      - name: Clone pyenv
        run: git clone https://github.com/pyenv/pyenv.git ~/.pyenv
      - name: Run the CI build script
        run: bash .ci/build.sh build
        env:
          BENCHMARK_HOME: env.GITHUB_WORKSPACE


Integ Actions

name: Integ Actions
on: [workflow_dispatch, pull_request]
jobs:
  Integration-Tests:
    strategy:
      matrix:
        python-version:
          - "3.8"
          - "3.9"
          - "3.10"
          - "3.11"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Check out repository code
        uses: actions/checkout@v2
      - name: Clone pyenv
        run: git clone https://github.com/pyenv/pyenv.git ~/.pyenv
      - name: Clone pyenv-virtualenv
        run: git clone https://github.com/pyenv/pyenv-virtualenv.git ~/.pyenv/plugins/pyenv-virtualenv
      - name: Install JDK 8
        uses: actions/setup-java@v3
        with:
          distribution: 'adopt'
          java-version: '8'
      - run: echo "JAVA8_HOME=$JAVA_HOME" >> $GITHUB_ENV
      - name: Install JDK 11
        uses: actions/setup-java@v3
        with:
          distribution: 'adopt'
          java-version: '11'
      - run: echo "JAVA11_HOME=$JAVA_HOME" >> $GITHUB_ENV
      - name: Install JDK 15
        uses: actions/setup-java@v3
        with:
          distribution: 'adopt'
          java-version: '15'
      - run: echo "JAVA15_HOME=$JAVA_HOME" >> $GITHUB_ENV
      - name: Install JDK 16
        uses: actions/setup-java@v3
        with:
          distribution: 'adopt'
          java-version: '16'
      - run: echo "JAVA16_HOME=$JAVA_HOME" >> $GITHUB_ENV
      - name: Install JDK 17
        uses: actions/setup-java@v3
        with:
          distribution: 'adopt'
          java-version: '17'
      - run: echo "JAVA17_HOME=$JAVA_HOME" >> $GITHUB_ENV
      - name: Run the CI build_it script
        run: bash .ci/build.sh build_it ${{ matrix.python-version }}
        env:
          BENCHMARK_HOME: env.GITHUB_WORKSPACE


Release drafter

name: Release drafter

on:
  push:
    tags:
      - "*"

jobs:
  draft-a-release:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout Repository
        uses: actions/checkout@v3
      - id: get_approvers
        run: |
          echo "approvers=$(cat .github/CODEOWNERS | grep @ | tr -d '* ' | sed 's/@/,/g' | sed 's/,//1')" >> $GITHUB_OUTPUT
      - uses: trstringer/manual-approval@v1
        with:
          secret: ${{ github.TOKEN }}
          approvers: ${{ steps.get_approvers.outputs.approvers }}
          minimum-approvals: 1
          issue-title: 'Release opensearch-benchmark'
          issue-body: "Please approve or deny the release of opensearch-benchmark. **Tag**: ${{ github.ref_name }}  **Commit**: ${{ github.sha }}"
          exclude-workflow-initiator-as-approver: true
      - name: Set up Python 3
        uses: actions/setup-python@v3
        with:
          python-version: '3.x'
      - name: Install build tools
        run: |
          python -m pip install --upgrade build
      - name: Build project for distribution
        run: |
          python -m build
          tar -zvcf artifacts.tar.gz dist
      - name: Release
        uses: softprops/action-gh-release@v1
        with:
          draft: true
          generate_release_notes: true
          files: |
            artifacts.tar.gz


CodeQL

# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"

on:
  push:
    branches: [ main ]
  pull_request:
    # The branches below must be a subset of the branches above
    branches: [ main ]
  schedule:
    - cron: '41 20 * * 6'

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write

    strategy:
      fail-fast: false
      matrix:
        language: [ 'python' ]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
        # Learn more:
        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed

    steps:
    - name: Checkout repository
      uses: actions/checkout@v2

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v1
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
        # By default, queries listed here will override any specified in a config file.
        # Prefix the list here with "+" to use these queries and those in the config file.
        # queries: ./path/to/local/query, your-org/your-repo/queries@main

    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
      uses: github/codeql-action/autobuild@v1

    # ℹ Command-line programs to run using the OS shell.
    # 📚 https://git.io/JvXDl

    # ✏ If the Autobuild fails above, remove it and uncomment the following three lines
    #    and modify them (or add more) to build your code if your project
    #    uses a compiled language

    #- run: |
    #   make bootstrap
    #   make release

    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v1


Documentation Check

name: Documentation Check

on: [push, pull_request]

jobs:
  markdown-link-check:
    name: Check markdown links
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@main
    - uses: gaurav-nelson/github-action-markdown-link-check@v1


Static Analysis

name: Static Analysis

on: [push, pull_request]

jobs:
  analyze:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest]
        python-version: [3.9]
    env:
      PYTHONPATH: ./src:./tests
      MYPYPATH: ./src:./tests:./src/stubs
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -e '.[dev]'
      - name: Lint with flake8
        run: |
          # stop the build if there are Python syntax errors or undefined names
          flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
          # exit-zero treats all errors as warnings.
          flake8 . --count --exit-zero --max-complexity=10 --max-line-length=160 --statistics
      - name: Lint with pylint
        run: |
          pylint -r y src tests/*.py
      - name: Security check with bandit
        run: |
          bandit -r src/
      - name: Type check with mypy
        run: |
          mypy src/ tests/


Unix Unit Tests

name: Unix Unit Tests

on: [push, pull_request]

jobs:
  test:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-latest, macos-latest]
        python-version: [3.8, 3.9, "3.10", 3.11, 3.12]
    env:
      PYTHONPATH: ./src:./test
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -e '.[dev]'
      - name: Test with pytest
        run: |
          pytest --tb=native --verbose


Windows Unit Tests

name: Windows Unit Tests

on: [push, pull_request]

jobs:
  test:
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [windows-latest]
        python-version: [3.8, 3.9, "3.10", 3.11, 3.12]
    env:
      PYTHONPATH: .\src;.\test
    steps:
      - uses: actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install -e '.[dev]'
      - name: Test with pytest
        run: |
          pytest --tb=native --verbose


Tests

name: Tests

on:
  workflow_dispatch:
  push:
    branches:
      - main
      - v1
    paths:
      - tests/*
      - hydromt/*
      - data/*
      - pyproject.toml
      - pixi.lock
      - pixi.toml
      - .github/workflows/tests.yml
  pull_request:
    branches:
      - main
      - v1
    paths:
      - tests/*
      - hydromt/*
      - data/*
      - pyproject.toml
      - pixi.lock
      - pixi.toml
      - .github/workflows/tests.yml

jobs:
  build:
    defaults:
      run:
        shell: bash -e -l {0}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest]
        python-version: ['39','310','311']

    name: py ${{ matrix.python-version }} (${{ matrix.os }})
    runs-on: ${{ matrix.os }}
    timeout-minutes: 30
    concurrency:
      group: ${{ github.workflow }}-${{ matrix.os }}-${{ matrix.python-version }}-${{ github.ref }}
      cancel-in-progress: true
    steps:

      - uses: actions/checkout@v4
      - uses: prefix-dev/setup-pixi@v0.6.0
        with:
          pixi-version: "v0.21.1"
          environments: full-py${{ matrix.python-version }}
      - name: Prepare pixi
        run: |
          pixi run --locked -e full-py${{ matrix.python-version }} install-hydromt

      # Enable tmate debugging of manually-triggered workflows if the input option was provided
      - name: Setup tmate session
        uses: mxschmitt/action-tmate@v3
        if: ${{ github.event_name == 'workflow_dispatch' && inputs.debug_enabled }}

      - name: Test
        run: |
          export NUMBA_DISABLE_JIT=1
          pixi run --locked -e full-py${{ matrix.python-version }} test-cov

      # enable for CI debugging
      # - name: Setup tmate session
      #   if: ${{ failure() }}
      #   uses: mxschmitt/action-tmate@v3


Build Documentation

---
name: Build Documentation
on:
  workflow_dispatch:
  push:
    branches: [main]
    paths:
      - tests/*
      - hydromt/*
      - data/*
      - docs/*
      - examples/*
      - pyproject.toml
      - .github/workflows/docs.yml
  pull_request:
    branches: [main]
    paths:
      - tests/*
      - hydromt/*
      - data/*
      - docs/*
      - examples/*
      - pyproject.toml
      - .github/workflows/docs.yml

jobs:
  test-docs:
    defaults:
      run:
        shell: bash -e -l {0}
    timeout-minutes: 30
    runs-on: ubuntu-latest
    steps:
      - name: checkout code
        uses: actions/checkout@v3
      - uses: prefix-dev/setup-pixi@v0.6.0
        with:
          pixi-version: "v0.21.1"
          environments: full-py311
      - name: Prepare pixi
        run: |
          pixi run --locked -e full-py311 install-hydromt

      # if we're not publishing we don't have to write them, so we might as well
      # save ourself a bunch of IO time
      - name: Generate dummy docs
        if: ${{ github.event_name == 'pull_request' }}
        run: |
          pixi run --locked -e full-py311 sphinx-build ./docs ./docs/_build -b dummy -W

      - name: Generate docs
        if: ${{ github.event_name != 'pull_request' && !github.event.act }}
        run: |
          pixi run --locked -e full-py311 html
          echo "DOC_VERSION=$(pixi run --locked -e full-py311 python -c 'from hydromt import __version__ as v; print("dev" if "dev" in v else "v"+v.replace(".dev",""))')" >> $GITHUB_ENV

      - name: Upload to GitHub Pages
        if: ${{ github.event_name != 'pull_request' && !github.event.act }}
        uses: peaceiris/actions-gh-pages@v3.9.3
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}
          publish_dir: ./docs/_build/html
          exclude_assets: .buildinfo,_sources/*,_examples/*.ipynb
          destination_dir: ./${{ env.DOC_VERSION }}
          keep_files: false
          full_commit_message: Deploy ${{ env.DOC_VERSION }} to GitHub Pages


Linting

---
name: Linting

on:
  pull_request:
  push:
    branches: [main]

jobs:
  pre-commit:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - uses: actions/setup-python@v3
      - uses: pre-commit/action@v3.0.0


---
name: Test Docker images and Binder integration

on:
  push:
    branches: [main]
    paths:
      - .binder/*
      - Dockerfile
      - pyproject.toml
  pull_request:
    branches: [main]
    paths:
      - .binder/*
      - Dockerfile
      - pyproject.toml

jobs:
  test-docker-image:
    runs-on: ubuntu-latest
    concurrency:
      group: test-docker-${{ github.ref }}
      cancel-in-progress: true
    timeout-minutes: 30
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Build and test
        uses: docker/build-push-action@v4
        with:
          context: .
          target: full
          tags: hydromt
          load: true
          build-args: |
            PIXIENV=full-py311

      - name: Run Tests
        run: docker run --env NUMBA_DISABLE_JIT=1 --rm hydromt pytest
      - name: Test Binder integration with repo2docker
        run: |
          pip install jupyter-repo2docker
          repo2docker . echo 'success!'


cleanup all caches

name: cleanup all caches
on:
  workflow_dispatch:
  workflow_call:

jobs:
  cleanup:
    runs-on: ubuntu-latest
    steps:
      - name: Check out code
        uses: actions/checkout@v4

      - name: Cleanup caches
        run: |
          if gh cache list | grep -vq "No cache"; then
            gh cache delete -a
          else
            echo "No caches found. skipping..."
          fi
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}


Tests with pip install

name: Tests with pip install

on:
  push:
    branches: [main]
    paths:
      - tests/*
      - hydromt/*
      - data/*
      - pyproject.toml
  pull_request:
    branches:
      - main
    paths:
      - tests/*
      - hydromt/*
      - data/*
      - pyproject.toml

jobs:
  build:
    defaults:
      run:
        shell: bash -l {0}
    strategy:
      fail-fast: false
    runs-on: ubuntu-latest
    timeout-minutes: 30
    # concurrency:
    #   group: ${{ github.workflow }}-${{ matrix.python-version }}-${{ github.ref }}
    #   cancel-in-progress: true
    steps:

      - uses: actions/checkout@v3

      - uses: actions/setup-python@v4
        id: py310
        with:
          python-version: '3.10'
          # caching, see https://github.com/actions/setup-python/blob/main/docs/advanced-usage.md#caching-packages
          cache: 'pip'
          cache-dependency-path: pyproject.toml

      # true if cache-hit occurred on the primary key
      - name: Cache hit
        run: echo '${{ steps.py310.outputs.cache-hit }}'

      # build environment with pip
      - name: Install hydromt
        run: |
          pip install --upgrade pip
          pip install .[test,io,extra]

      # run test
      - name: Test
        run: |
          export NUMBA_DISABLE_JIT=1
          python -m pytest --verbose


Validate Data Catalog

name: Validate Data Catalog

on:
  workflow_dispatch:
  push:
    branches: [main]
    paths:
      - data/catalogs/*.yml
  pull_request:
    branches:
      - main
    paths:
      - data/catalogs/*.yml

jobs:
  build:
    defaults:
      run:
        shell: bash -l {0}
    env:
      DOC_PYTHON_VERSION: '3.11'
    runs-on: ubuntu-latest
    timeout-minutes: 30
    concurrency:
      group: ${{ github.workflow }}-data-catalog-validation-${{ github.ref }}
      cancel-in-progress: true

    steps:
      - uses: actions/checkout@v3
      - uses: prefix-dev/setup-pixi@v0.6.0
        with:
          pixi-version: "v0.21.1"
          environments: min-py311
      - name: Prepare pixi
        run: |
          pixi run --locked -e min-py311 install-hydromt
      - name: Validate the catalogs
        run: |
          export NUMBA_DISABLE_JIT=1
          for d in $(ls data/catalogs/*.yml); do
            echo "Checking $d"
            PYTHONPYCACHEPREFIX=~/pycache pixi run --locked -e min-py311 hydromt check -d $d
          done


Pixi auto update

name: Pixi auto update

on:
  schedule:
  # At 03:00 on day 3 of the month
    - cron: "0 3 3 * *"
  # on demand
  workflow_dispatch:

jobs:
  auto-update:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          ssh-key: ${{ secrets.SSH_PRIVATE_KEY }}
      - uses: prefix-dev/setup-pixi@v0.6.0
        with:
          pixi-version: "v0.21.1"
          cache: false
      - name: Update pixi lock file
        run: |
          rm pixi.lock
          pixi install
      - uses: peter-evans/create-pull-request@v6
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          branch: update/pixi-lock
          title: Update pixi lock file
          commit-message: "Update `pixi.lock`"
          body: Update pixi dependencies to the latest version.
          author: "GitHub <noreply@github.com>"


Conditional Bump Version

name: Conditional Bump Version

on:
  push:
    branches:
      - master
    paths-ignore:
      - .*
      - .github/*/*.md
      - .github/*/*.yml
      - AUTHORS.rst
      - CHANGES.rst
      - Dockerfile
      - Makefile
      - docker-compose.yml
      - docs/*/*.ipynb
      - docs/*/*.py
      - docs/*/*.rst
      - docs/Makefile
      - docs/_static/*
      - environment-docs.yml
      - finch/__version__.py
      - requirements*.txt
      - setup.cfg
      - setup.py

permissions:
  contents: read

jobs:
  bump_patch_version:
    name: Bumpversion
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: write
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
        with:
          disable-sudo: true
          egress-policy: block
          allowed-endpoints: >
            files.pythonhosted.org:443
            github.com:443
            pypi.org:443
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
      - uses: actions/setup-python@v5
        with:
          python-version: "3.x"
      - name: Config Commit Bot
        run: |
          git config --local user.email "bumpversion[bot]@ouranos.ca"
          git config --local user.name "bumpversion[bot]"
      - name: Current Version
        run: |
          CURRENT_VERSION="$(grep -E '__version__' finch/__version__.py | cut -d ' ' -f3)"
          echo "current_version=${CURRENT_VERSION}"
          echo "CURRENT_VERSION=${CURRENT_VERSION}" >> $GITHUB_ENV
      - name: Bump Version
        if: ${{ github.event.inputs.tag == 'false' }}
        run: |
          pip install bump-my-version
          echo "running `bump-my-version bump  --no-tag ${{ github.event.inputs.bumpversion }}`"
          NEW_VERSION="$(grep -E '__version__' finch/__version__.py | cut -d ' ' -f3)"
          echo "new_version=${NEW_VERSION}"
      - name: Install bump-my-version
        run: |
          python -m pip install bump-my-version>=0.18.3
      - name: Conditional Bump
        id: bump
        run: |
          if [[ ${{ env.CURRENT_VERSION }} =~ -dev(\.\d+)? ]]; then
            echo "Development version (ends in 'dev(\.\d+)?'), bumping 'build' version"
            bump-my-version bump build
          else
            echo "Version is stable, bumping 'patch' version"
            bump-my-version bump patch
          fi
          NEW_VERSION="$(grep -E '__version__'  finch/__version__.py | cut -d ' ' -f3)"
          echo "new_version=${NEW_VERSION}"
          echo "NEW_VERSION=${NEW_VERSION}" >> $GITHUB_ENV
      - name: Push Changes
        uses: ad-m/github-push-action@master
        with:
          force: false
          github_token: ${{ secrets.BUMPVERSION_TOKEN }}
          branch: ${{ github.ref }}


Docker-based Testing Suite

name: Docker-based Testing Suite

on:
  push:
    branches:
      - master
  pull_request:

concurrency:
  # For a given workflow, if we push to the same branch, cancel all previous builds on that branch except on master.
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}

jobs:
  build:
    name: Build and Test Docker image
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: docker/setup-buildx-action@v3
      - uses: docker/build-push-action@v5
        with:
          context: .
          file: "Dockerfile"
          tags: localfinch:latest
          load: true
          cache-from: type=gha
          cache-to: type=gha,mode=max
          push: false
      - uses: addnab/docker-run-action@v3
        with:
          image: localfinch:latest
          options: -p 5000:5000
          run: |
            finch start -d
            sleep 2s
            finch status
            finch stop


Labeler

# This workflow will triage pull requests and apply a label based on the
# paths that are modified in the pull request.
#
# To use this workflow, you will need to set up a .github/labeler.yml
# file with configuration.  For more information, see:
# https://github.com/actions/labeler/blob/master/README.md

name: Labeler
on:
  # Note: potential security risk from this action using pull_request_target.
  # Do not add actions in here which need a checkout of the repo, and do not use any caching in here.
  # See: https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target
  pull_request_target:
    types:
      - opened
      - reopened
      - synchronize

permissions:
  contents: read

jobs:
  label:
    runs-on: ubuntu-latest
    permissions:
      checks: write
      contents: read
      pull-requests: write
    steps:
      - name: Harden Runner
        uses: step-security/harden-runner@17d0e2bd7d51742c71671bd19fa12bdc9d40a3d6 # v2.8.1
        with:
          disable-sudo: true
          egress-policy: block
          allowed-endpoints: >
            api.github.com:443
      - uses: actions/labeler@8558fd74291d67161a8a78ce36a881fa63b766a9 # v5.0.0
        with:
          repo-token: "${{ secrets.GITHUB_TOKEN }}"


Finch Testing Suite ⚙

name: Finch Testing Suite ⚙

on:
  push:
    branches:
      - master
  pull_request:

concurrency:
  # For a given workflow, if we push to the same branch, cancel all previous builds on that branch except on master.
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: ${{ github.ref != 'refs/heads/master' }}

jobs:
  lint:
    name: Code linting
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.8"
      - name: Install flake8
        run: |
          python -m pip install flake8
      - name: Run linting suite
        run: |
          flake8

  conda:
    name: Build ⚙ and test 🧪
    needs: lint
    runs-on: ubuntu-latest
    defaults:
      run:
        shell: bash -l {0}
    strategy:
        matrix:
            python-version: ["3.8", "3.9", "3.10", "3.11"]
    steps:
      - uses: actions/checkout@v4
      - name: Setup conda and environment
        uses: mamba-org/setup-micromamba@v1
        with:
          cache-downloads: true
          cache-environment: true
          environment-file: environment.yml
          create-args: >-
            conda
            python=${{ matrix.python-version }}
      - name: Conda and Mamba versions
        run: |
          conda --version
          echo "micromamba: $(micromamba --version)"
      - name: Install finch-wps
        run: |
          make develop
      - name: Check versions
        run: |
          conda list
          python -m pip check || true
      - name: Run tests
        run: |
          make start
          sleep 2
          make test-notebooks
          make test


odxtools CI pipeline

# SPDX-License-Identifier: MIT
# Copyright (c) 2023 MBition GmbH

name: odxtools CI pipeline

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]
  workflow_dispatch:

jobs:
  lint-coding-style:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.12"

      - name: Setup up YaPF formatting linter
        run: |
          pip install toml yapf

      - name: Lint coding style (yapf)
        # make yapf fail if any file needs to be re-formatted
        run: |
          yapf -r --diff examples/ odxtools/ tests/

  linting:
    runs-on: ubuntu-latest

    strategy:
      matrix:
        # restrict the matrix to the oldest and the latest Python
        # version being supported by odxtools
        python-version: ["3.8", "3.12"]

    steps:
      - uses: actions/checkout@v3

      - name: Set up Python ${{matrix.python-version}}
        uses: actions/setup-python@v4
        with:
          python-version: ${{matrix.python-version}}
          cache: "pip"

      - name: Install odxtools development and testing dependencies
        run: |
          pip install --upgrade pip
          pip install .
          pip install .[test]

      - name: Static type checking (mypy)
        run: |
          # remove '# type:' comments in version.py which mypy-3.8
          # stumbles over. for some reason, mypy cannot be
          # instructed to ignore 'version.py'
          sed -i "s/# type:.*//" odxtools/version.py
          python -m mypy .

      - name: Lint code quality (ruff)
        run: |
          ruff .

  test:
    strategy:
      matrix:
        os: [ubuntu-latest, windows-latest]
        # restrict the matrix to the oldest and the latest Python
        # version being supported by odxtools
        python-version: ["3.8", "3.12"]

        # due to the slow windows runners, we refrain from testing every python
        # version on windows-latest
        exclude:
          - os: windows-latest
            python-version: "3.8"

    runs-on: ${{matrix.os}}

    steps:
      - uses: actions/checkout@v3

      - name: Set up Python ${{matrix.python-version}}
        uses: actions/setup-python@v4
        with:
          python-version: ${{matrix.python-version}}
          cache: "pip"

      # dependency are likely cached on ubuntu-latest
      - name: Install odxtools development and testing dependencies
        run: |
          pip install --upgrade pip
          pip install .
          pip install .[test]

      - name: Create somersault.pdx
        run: |
          python -m examples.mksomersaultpdx examples/somersault.pdx

      - name: Run unit tests and report line coverage
        # use `python -m pytest ...` instead of `pytest ...` to avoid setting the
        # PYTHONPATH environment variable. This works independently of the OS.

        # For some reason, pytest-cov reports invalid line coverage. As a remedy,
        # we utilize the `coverage` module instead of `pytest-cov`. Line coverage
        # shall be only reported for the odxtools package, i.e., the tests package
        # should be ignored.
        run: |
          coverage run --source=odxtools --omit=tests -m pytest
          coverage report


CodeQL

name: "CodeQL"

on:
  push:
    branches: [ "master" ]
  pull_request:
    branches: [ "master" ]
  schedule:
    - cron: "31 5 * * 5"

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write

    strategy:
      fail-fast: false
      matrix:
        language: [ python ]

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Initialize CodeQL
        uses: github/codeql-action/init@v3
        with:
          languages: ${{ matrix.language }}
          queries: +security-and-quality
          config-file: codeql.yml

      - name: Autobuild
        uses: github/codeql-action/autobuild@v3

      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v3
        with:
          category: "/language:${{ matrix.language }}"


Build Docs

name: Build Docs

on:
  push:
    branches:
      - master
  pull_request:

jobs:
  docs:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python:
          - 3.8
          #- 3.9
          #- 3.10
          #- 3.11
    steps:
    - name: Check out repository
      uses: actions/checkout@v4
      with:
        # Fetch all commits so that versioneer will return something compatible
        # with semantic-version
        fetch-depth: 0

    - name: Set up Python ${{ matrix.python }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python }}

    - name: Install hdf5 (Ubuntu)
      if: matrix.python == '3.9' && startsWith(matrix.os, 'ubuntu')
      run: sudo apt-get update && sudo apt-get install -y libhdf5-dev

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip wheel
        python -m pip install --upgrade tox

    - name: Build docs
      run: tox -e docs


Linters

name: Linters

on:
  - push
  - pull_request

jobs:
  lint:
    runs-on: ubuntu-latest
    steps:
    - name: Set up environment
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: '3.8'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install --upgrade tox
    # Annotate codespell within PR
    - uses: codespell-project/codespell-problem-matcher@v1
    - name: Run linters
      run: |
        tox -e lint


Auto-release on PR merge

name: Auto-release on PR merge

on:
  # ATM, this is the closest trigger to a PR merging
  push:
    branches:
      - master

jobs:
  auto-release:
    runs-on: ubuntu-latest
    if: "!contains(github.event.head_commit.message, 'ci skip') && !contains(github.event.head_commit.message, 'skip ci')"
    steps:
      - name: Checkout source
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Download latest auto
        run: |
          auto_download_url="$(curl -fsSL https://api.github.com/repos/intuit/auto/releases/latest | jq -r '.assets[] | select(.name == "auto-linux.gz") | .browser_download_url')"
          wget -O- "$auto_download_url" | gunzip > ~/auto
          chmod a+x ~/auto

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: 3.8

      - name: Install build & twine
        run: python -m pip install build twine

      - name: Create release
        run: ~/auto shipit -vv
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          TWINE_USERNAME: __token__
          TWINE_PASSWORD: ${{ secrets.PYPI_TOKEN }}

# vim:set sts=2:


Tests

name: Tests

on:
  push:
    branches:
      - master
  pull_request:
  schedule:
    - cron: '0 6 * * *'

concurrency:
  group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.ref_name }}
  cancel-in-progress: true

defaults:
  run:
    shell: bash

jobs:
  test:
    runs-on: ${{ matrix.os }}
    env:
      NO_ET: 1
      DANDI_ALLOW_LOCALHOST_URLS: "1"
      DANDI_PAGINATION_DISABLE_FALLBACK: "1"
      DANDI_TESTS_PERSIST_DOCKER_COMPOSE: "1"
    strategy:
      fail-fast: false
      matrix:
        os:
          - windows-2019
          - ubuntu-latest
          - macos-latest
        python:
          - 3.8
          - 3.9
          - '3.10'  # Needs quotes so YAML doesn't think it's 3.1
          - '3.11'
          - '3.12'
        mode:
          - normal
        include:
          - os: ubuntu-latest
            python: 3.8
            mode: dandi-api
          - os: ubuntu-latest
            python: 3.8
            mode: dev-deps
          - os: ubuntu-latest
            python: 3.8
            mode: nfs

    steps:
    - name: Set up environment
      uses: actions/checkout@v4
      with:
        # Fetch all commits so that versioneer will return something compatible
        # with semantic-version
        fetch-depth: 0

    - name: Set up Python ${{ matrix.python }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python }}

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip wheel
        pip install ".[extras,test]"

    - name: Install dev versions of select dependencies
      if: matrix.mode == 'dev-deps'
      run: |
        pip install git+https://github.com/jaraco/keyring
        pip install git+https://github.com/NeurodataWithoutBorders/nwbinspector
        pip install git+https://github.com/NeurodataWithoutBorders/pynwb
        pip install git+https://github.com/hdmf-dev/hdmf

    - name: Create NFS filesystem
      if: matrix.mode == 'nfs'
      run: |
        mkdir /tmp/nfsmount_ /tmp/nfsmount
        mkdir /tmp/nfsmount_/tmp /tmp/nfsmount_/home
        echo "/tmp/nfsmount_ localhost(rw)" | sudo tee /etc/exports
        sudo apt-get install -y nfs-kernel-server
        sudo exportfs -a
        sudo mount -t nfs localhost:/tmp/nfsmount_ /tmp/nfsmount
        echo TMPDIR=/tmp/nfsmount/tmp >> "$GITHUB_ENV"
        echo HOME=/tmp/nfsmount/home >> "$GITHUB_ENV"
        echo DANDI_DEVEL_INSTRUMENT_REQUESTS_SUPERLEN=1 >> "$GITHUB_ENV"

    - name: Use scheduled test configuration
      if: github.event_name == 'schedule'
      run: echo PYTEST_ADDOPTS=--scheduled >> "$GITHUB_ENV"

    - name: Run all tests
      if: matrix.mode != 'dandi-api'
      run: |
        python -m pytest -s -v --cov=dandi --cov-report=xml dandi

    - name: Smoke test example code in docs
      if: matrix.mode != 'dandi-api' && github.event_name == 'schedule'
      run: |
        set -ex
        cd docs/source/examples
        for f in *.py
        do python "$f"
        done

    - name: Run Dandi API tests only
      if: matrix.mode == 'dandi-api'
      run: |
        python -m pytest -s -v --cov=dandi --cov-report=xml --dandi-api dandi

    - name: Dump Docker Compose logs
      if: failure() && startsWith(matrix.os, 'ubuntu')
      run: |
        docker-compose \
          -f dandi/tests/data/dandiarchive-docker/docker-compose.yml \
          logs --timestamps

    - name: Shut down Docker Compose
      if: startsWith(matrix.os, 'ubuntu')
      run: |
        docker-compose \
          -f dandi/tests/data/dandiarchive-docker/docker-compose.yml \
          down -v

    - name: Upload coverage to Codecov
      uses: codecov/codecov-action@v4
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        file: ./coverage.xml
        flags: unittests
        # name: codecov-umbrella
        # yml: ./codecov.yml
        fail_ci_if_error: false


Type-check

name: Type-check

on:
  - push
  - pull_request

jobs:
  typing:
    runs-on: ubuntu-latest
    steps:
      - name: Check out repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.8'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install --upgrade tox

      - name: Run type checker
        run: tox -e typing