Codespeed Benchmarks

name: Codespeed Benchmarks

on:
  push:
    branches:
      - master
      - release
  pull_request:
    branches:
      - master
  schedule:
    - cron: '0 4 * * *'

jobs:
  # Special job which automatically cancels old runs for the same branch, prevents runs for the
  # same file set which has already passed, etc.
  pre_job:
    name: Skip Duplicate Jobs Pre Job
    runs-on: ubuntu-latest
    if: ${{ ! vars.UNIT_TESTS_DISABLED }}
    permissions:
      actions: write  # Needed for skip-duplicate-jobs job
      contents: read
    outputs:
      should_skip: ${{ steps.skip_check.outputs.should_skip }}
    steps:
      - id: skip_check
        uses: fkirc/skip-duplicate-actions@12aca0a884f6137d619d6a8a09fcc3406ced5281 # v4.0.0
        with:
          cancel_others: 'true'
          github_token: ${{ github.token }}

  codespeed-micro-benchmarks:
    name: Micro Benchmarks
    runs-on: ubuntu-latest
    needs: pre_job
    if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref_name == 'master' }}
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "3.11.0"

    steps:
      - name: Checkout Repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4
        with:
          submodules: true

      - name: Setup Python
        uses: actions/setup-python@v4
        id: setup-python
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install Deps
        env:
          TOX_VERSION: 3.20.1
          TOX_GH_ACTIONS_VERSION: 2.9.1
        run: |
          python -m pip install --upgrade pip
          pip install "tox==$TOX_VERSION" "tox-gh-actions==$TOX_GH_ACTIONS_VERSION" "cffi==1.15.1" "udatetime==0.0.17" "requests"
          # Needed for snappy library
          sudo apt-get update
          sudo apt-get install -y libsnappy-dev

      - name: Run Benchmarks
        run: tox -e micro-benchmarks

      - name: Process and submit results to CodeSpeed
        # We only want to submit results for master branch runs and not for scheduled runs
        if: ${{ github.ref_name == 'master' && github.event_name != 'schedule' }}
        env:
          PYTHONPATH: .
          CODESPEED_URL: "https://scalyr-agent-codespeed.herokuapp.com/"
          CODESPEED_PROJECT: "scalyr-agent-2-microbenchmarks"
          CODESPEED_EXECUTABLE: Python ${{ steps.setup-python.outputs.python-version }}
          CODESPEED_ENVIRONMENT: GitHub Hosted Action Runner
          CODESPEED_AUTH: ${{ secrets.CODESPEED_AUTH }}
        run: |
          python benchmarks/scripts/send_microbenchmarks_data_to_codespeed.py --data-path="benchmark_results/*.json" --debug

      - name: Store artifacts
        uses: actions/upload-artifact@v3
        with:
          name: codespeed-micro-benchmarks-${{ matrix.python-version }}
          path: |
            benchmark_results/
            benchmark_histograms/

      - name: Notify Slack on Failure
        if: ${{ failure() && github.ref_name == 'master' }}
        uses: act10ns/slack@ed1309ab9862e57e9e583e51c7889486b9a00b0f # v2.0.0
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
        with:
          status: ${{ job.status }}
          steps: ${{ toJson(steps) }}
          channel: '#eng-dataset-cloud-tech'

  benchmarks-idle-agent-py-311:
    name: Process Benchmark
    needs: pre_job
    if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref_name == 'master' }}
    uses: ./.github/workflows/reusable-codespeed-agent-benchmarks.yaml
    secrets: inherit
    with:
      job_name: benchmarks-idle-agent-py-311
      python-version: 3.11.0
      codespeed_executable: "Python 3.11.0 - idle conf 1"
      agent_config: "benchmarks/configs/agent_no_monitored_logs.json"
      agent_server_host: "ci-codespeed-benchmarks-py311-idle-conf-1"
      capture_line_counts: true
      # We only want to send data to CodeSpeed on master branch runs
      dry_run: ${{ github.ref_name != 'master' }}

  benchmarks-idle-agent-no-monitors-py-311:
    name: Process Benchmark
    needs: pre_job
    if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref_name == 'master' }}
    uses: ./.github/workflows/reusable-codespeed-agent-benchmarks.yaml
    secrets: inherit
    with:
      job_name: benchmarks-idle-agent-no-monitors-py-311
      python-version: 3.11.0
      codespeed_executable: "Python 3.11.0 - idle conf 2"
      agent_config: "benchmarks/configs/agent_no_monitored_logs_no_monitors.json"
      agent_server_host: "ci-codespeed-benchmarks-py311-idle-conf-2"
      capture_line_counts: true
      # We only want to send data to CodeSpeed on master branch runs
      dry_run: ${{ github.ref_name != 'master' }}

  benchmarks-loaded-agent-single-50mb-log-file-with-parser-py-311:
    name: Process Benchmark
    needs: pre_job
    if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref_name == 'master' }}
    uses: ./.github/workflows/reusable-codespeed-agent-benchmarks.yaml
    secrets: inherit
    with:
      job_name: benchmarks-loaded-agent-single-50mb-log-file-with-parser-py-311
      python-version: 3.11.0
      codespeed_executable: "Python 3.11.0 - loaded conf 1"
      agent_config: "benchmarks/configs/agent_single_50mb_access_log_file.json"
      agent_pre_run_command: "wget --directory-prefix=/tmp https://github.com/scalyr/codespeed-agent-fixtures/raw/master/fixtures/logs/access_log_50_mb.log"
      agent_server_host: "ci-codespeed-benchmarks-py311-loaded-conf-1"
      run_time: 140
      capture_line_counts: true
      # We only want to send data to CodeSpeed on master branch runs
      dry_run: ${{ github.ref_name != 'master' }}

  # NOTE: For the benchmarks below to work correctly "/tmp/random.log" file
  # which is being written to during the benchmark must existing before the
  # agent process is started.
  benchmarks-loaded-agent-single-growing-log-file-20mb-py-311:
    name: Process Benchmark
    needs: pre_job
    if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref_name == 'master' }}
    uses: ./.github/workflows/reusable-codespeed-agent-benchmarks.yaml
    secrets: inherit
    with:
      job_name: benchmarks-loaded-agent-single-growing-log-file-20mb-py-311
      python-version: 3.11.0
      codespeed_executable: "Python 3.11.0 - loaded conf 2"
      agent_config: "benchmarks/configs/agent_single_growing_log_file_with_shell_and_url_monitor.json"
      run_time: 140
      agent_pre_run_command: "touch /tmp/random.log"
      agent_post_run_command: "benchmarks/scripts/write-random-lines.sh /tmp/random.log 2M 10 100 1"
      agent_server_host: "ci-codespeed-benchmarks-py311-loaded-conf-2"
      capture_line_counts: true
      # We only want to send data to CodeSpeed on master branch runs
      dry_run: ${{ github.ref_name != 'master' }}

  benchmarks-loaded-agent_2-growing_logs-monitors-multiprocess-2-worker-20mb-py-311:
    name: Process Benchmark
    needs: pre_job
    if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref_name == 'master' }}
    uses: ./.github/workflows/reusable-codespeed-agent-benchmarks.yaml
    secrets: inherit
    with:
      job_name: benchmarks-loaded-agent_2-growing_logs-monitors-multiprocess-2-worker-20mb-py-31
      python-version: 3.11.0
      codespeed_executable: "Python 3.11.0 - loaded conf 4"
      agent_config: "benchmarks/configs/agent_2-growing_logs-monitors-multiprocess-2-worker.json"
      run_time: 140
      agent_pre_run_command: "touch /tmp/random.log & touch /tmp/random2.log"
      agent_post_run_command: "benchmarks/scripts/write-random-lines.sh /tmp/random.log 2M 10 100 1 & benchmarks/scripts/write-random-lines.sh /tmp/random2.log 2M 10 100 1 &"
      agent_server_host: "ci-codespeed-benchmarks-py311-loaded-conf-4"
      capture_line_counts: true
      # We only want to send data to CodeSpeed on master branch runs
      dry_run: ${{ github.ref_name != 'master' }}

  benchmarks-loaded-agent-single-50mb-log-file-with-500k-lines-py-311:
    name: Process Benchmark
    needs: pre_job
    if: ${{ needs.pre_job.outputs.should_skip != 'true' || github.ref_name == 'master' }}
    uses: ./.github/workflows/reusable-codespeed-agent-benchmarks.yaml
    secrets: inherit
    with:
      job_name: benchmarks-loaded-agent-single-50mb-log-file-with-500k-lines-py-311
      python-version: 3.11.0
      codespeed_executable: "Python 3.11.0 - loaded conf 5"
      agent_config: "benchmarks/configs/agent_single_50mb_genlog_500k_line_file.json"
      agent_pre_run_command: "wget --directory-prefix=/tmp https://github.com/scalyr/codespeed-agent-fixtures/raw/master/fixtures/logs/scalyr_genlog_500k_line_50_mb.log"
      agent_server_host: "ci-codespeed-benchmarks-py311-loaded-conf-5"
      run_time: 140
      capture_agent_status_metrics: true
      capture_line_counts: true
      # We only want to send data to CodeSpeed on master branch runs
      dry_run: ${{ github.ref_name != 'master' }}


Codespeed Agent Benchmark

# Workflow which performs end to end benchmarking of the agent process - it relies on starting up
# the agent and exercising various code with different config option.
name: Codespeed Agent Benchmark

on:
  workflow_call:
    inputs:
      job_name:
        description: "Github Action job name. Used to display user friendly job name in the UI."
        type: string
        required: true
      python-version:
        description: Python Version
        type: string
        required: true
      agent_config:
        description: "Path to the agent config file to use."
        type: string
        required: true
      codespeed_executable:
        description: "CodeSpeed executable name."
        type: string
        required: true
      codespeed_environment:
        description: "CodeSpeed environment name."
        type: string
        default: GitHub Hosted Action Runner
        required: false
      run_time:
        description: "How long to run the capture for (in seconds)."
        type: number
        default: 120
        required: false
      capture_interval:
        description: "How often to capture agent process level metrics during the process run time (in seconds)."
        type: number
        default: 5
        required: false
      agent_pre_run_command:
        description: "Optional bash command / script to run before starting the agent and the metrics capture script."
        type: string
        default: ""
        required: false
      agent_post_run_command:
        description: "Optional bash command / script to run after starting the agent and the metrics capture script."
        type: string
        default: ""
        required: false
      agent_server_host:
        description: "Value for the server_attributes.serverHost agent configuration option."
        type: string
        default: "ci-codespeed-benchmarks"
        required: false
      capture_agent_status_metrics:
        description: "True to capture additional metrics exposed via agent status command."
        type: boolean
        default: false
        required: false
      capture_line_counts:
        description: "True to submit log line counts for each log level to CodeSpeed."
        type: boolean
        default: false
        required: false
      dry_run:
        description: "True to enable dry run which runs the benchmarks without submitting data to CodeSpeed."
        type: boolean
        default: false
        required: false

jobs:
  codespeed-agent-benchmark:
    name: ${{ inputs.job_name }}
    runs-on: ubuntu-20.04
    steps:
      - name: Checkout Repository
        uses: actions/checkout@8ade135a41bc03ea155e62e844d188df1ea18608 # v4

      - name: Setup Python
        uses: actions/setup-python@v4
        id: setup-python
        with:
          python-version: ${{ inputs.python-version }}

      - name: Install Deps
        run: |
          python -m pip install --upgrade pip
          pip install -r benchmarks/scripts/requirements.txt

      - name: Capture commit date
        id: commit-date
        env:
          TZ: UTC
        run: |
          echo "commit-date=$(git show --quiet --date='format-local:%Y-%m-%d %H:%M:%S' --format='%cd' ${{ github.sha }})" >> $GITHUB_OUTPUT

      - name: Pre-Run Command
        if: inputs.agent_pre_run_command != ''
        run: ${{ inputs.agent_pre_run_command }}

      - name: Run Agent And Capture Resource Utilization
        id: run-agent
        env:
          CODESPEED_AUTH: ${{ secrets.CODESPEED_AUTH }}
          CODESPEED_URL: "https://scalyr-agent-codespeed.herokuapp.com/"
          CODESPEED_PROJECT: "scalyr-agent-2-procbenchmarks"
          CODESPEED_EXECUTABLE: ${{ inputs.codespeed_executable }}
          CODESPEED_ENVIRONMENT: ${{ inputs.codespeed_environment }}
          CODESPEED_BRANCH: ${{ github.head_ref || github.ref_name }}
          # NOTE: "idle" agent process (which monitors no logs but just runs the linux process
          # monitor for the agent process) should stabilize in a couple of minutes so it makes
          # no sense to run that benchmark longer.
          RUN_TIME: ${{ inputs.run_time }}
          CAPTURE_INTERVAL: ${{ inputs.capture_interval }}
          AGENT_CONFIG_FILE: ${{ inputs.agent_config }}
          SCALYR_SERVER: https://agent.scalyr.com
          SCALYR_API_KEY: ${{ secrets.CT_SCALYR_TOKEN_PROD_US_CLOUDTECH_TESTING_WRITE }}
          SCALYR_SERVER_ATTRIBUTES: "{\"serverHost\": \"${{ inputs.agent_server_host }}\"}"
          CAPTURE_AGENT_STATUS_METRICS: ${{ inputs.capture_agent_status_metrics }}
          DRY_RUN: ${{ inputs.dry_run }}
          COMMIT_DATE: ${{ steps.commit-date.outputs.commit-date }}
          PYTHONPATH: .
        run: |
          # Create directories which are needed by the agent process
          mkdir -p ~/scalyr-agent-dev/{log,config,data}
          # Run the agent process and capture the metrics
          ./benchmarks/scripts/start-agent-and-capture-metrics.sh "${{ github.sha }}" &
          bg_pid=$!
          echo "bg-pid=${bg_pid}" >> $GITHUB_OUTPUT

      - name: Run any post agent run script
        if: inputs.agent_post_run_command != ''
        run: |
          sleep 2
          ${{ inputs.agent_post_run_command }}

      - name: Wait for agent to finish
        timeout-minutes: 10
        run: tail --pid ${{ steps.run-agent.outputs.bg-pid }} -f /dev/null || true

      - name: Send line count values for various log levels
        if: inputs.capture_line_counts && ! inputs.dry_run
        env:
          CODESPEED_AUTH: ${{ secrets.CODESPEED_AUTH }}
          CODESPEED_URL: "https://scalyr-agent-codespeed.herokuapp.com/"
          CODESPEED_PROJECT: "scalyr-agent-2-procbenchmarks"
          CODESPEED_EXECUTABLE: ${{ inputs.codespeed_executable }}
          CODESPEED_ENVIRONMENT: ${{ inputs.codespeed_environment }}
          CODESPEED_BRANCH: ${{ github.head_ref || github.ref_name }}
          COMMIT_DATE: ${{ steps.commit-date.outputs.commit-date }}
          CAPTURE_AGENT_STATUS_METRICS: ${{ inputs.capture_agent_status_metrics }}
          DRY_RUN: ${{ inputs.dry_run }}
          PYTHONPATH: .
        run: |
          ./benchmarks/scripts/send-log-level-counts-to-codespeed.sh "${{ github.sha }}"

      - name: Upload log artifacts
        uses: actions/upload-artifact@v3
        with:
          name: agent-logs-${{ inputs.agent_server_host }}
          path: |
            ~/scalyr-agent-dev/log
        if: ${{ success() || failure() }}

      - name: Notify Slack on Failure
        if: ${{ failure() && github.ref_name == 'master' }}
        uses: act10ns/slack@ed1309ab9862e57e9e583e51c7889486b9a00b0f # v2.0.0
        env:
          SLACK_WEBHOOK_URL: ${{ secrets.SLACK_WEBHOOK_URL }}
        with:
          status: ${{ job.status }}
          steps: ${{ toJson(steps) }}
          channel: '#eng-dataset-cloud-tech'


CI

name: CI

on:
  push:
  pull_request:
  workflow_dispatch:

jobs:

  Run:
    strategy:
      fail-fast: false
      matrix:
        library:
          - sky130_fd_sc_hd
          - sky130_fd_sc_hdll
          - sky130_fd_sc_hs
          - sky130_fd_sc_ms
          - sky130_fd_sc_ls
          - sky130_fd_sc_hvl
          - all
    env:
      STD_CELL_LIBRARY: ${{ matrix.library }}

    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v3
      with:
        fetch-depth: 0
        submodules: true

    - name: Build
      run: |
        bash .github/build.sh

    - name: Test
      run: |
        bash .github/test.sh

    - name: Capture
      if: ${{ always() }}
      run: |
        bash .github/capture.sh

    - name: Upload
      if: ${{ always() }}
      uses: actions/upload-artifact@v3
      with:
        name: ${{ matrix.library }}
        path: ${{ github.workspace }}/output/**


---
name: CI

on:
    push:
        branches:
            - development
            - stable
    pull_request:
    schedule:
        - cron: 0 0 * * 0 # weekly

jobs:
    build-linux:
        runs-on: ubuntu-latest
        steps:
            - uses: actions/checkout@v3

            - uses: actions/setup-python@v4
              with:
                  python-version: '3.10'

            - name: Install build dependencies
              uses: ./.github/actions/install-build-dependencies

            - name: Build Python wheel
              run: make bdist_wheel bdist_wheel-linux-rename
              env:
                  CC: clang
                  CXX: clang++
                  BAZEL_OPTS: --batch
                  BAZEL_FETCH_OPTS: --config=ci
                  BAZEL_BUILD_OPTS: --config=ci

            - name: Upload Python wheel
              uses: actions/upload-artifact@v3
              with:
                  name: linux-wheel
                  path: dist/*.whl
                  if-no-files-found: error
                  retention-days: 7

    build-linux-cmake:
        runs-on: ubuntu-latest
        steps:
            - uses: actions/checkout@v3

            - uses: actions/setup-python@v4
              with:
                  python-version: '3.10'

            - name: Install build dependencies
              uses: ./.github/actions/install-cmake-build-dependencies

            - name: Download LLVM 10.0.0 release
              run: |
                  if [ "$(uname)" = "Darwin" ]; then
                    wget --quiet https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/clang+llvm-10.0.0-x86_64-apple-darwin.tar.xz -O ~/llvm.tar.xz
                  else
                    wget --quiet https://github.com/llvm/llvm-project/releases/download/llvmorg-10.0.0/clang+llvm-10.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz -O ~/llvm.tar.xz
                  fi
                  mkdir ~/llvm && tar xf ~/llvm.tar.xz --strip-components 1 -C ~/llvm
                  rm ~/llvm.tar.xz
                  echo "Unpacked, testing for expected file:"
                  test -d ~/llvm/lib/cmake/llvm

            - name: CMake Build
              run: |
                  cmake \
                      -GNinja \
                      -DCMAKE_C_COMPILER=clang \
                      -DCMAKE_CXX_COMPILER=clang++ \
                      -DCOMPILER_GYM_LLVM_PROVIDER=external \
                      -DLLVM_DIR=$HOME/llvm/lib/cmake/llvm \
                      -DPython3_FIND_UNVERSIONED_NAMES=FIRST \
                      -DCOMPILER_GYM_BUILD_TESTS=ON \
                      -DCOMPILER_GYM_BUILD_EXAMPLES=ON \
                      -S . \
                      -B ~/cmake_build
                  cmake --build ~/cmake_build
              shell: bash

            - name: Upload Python wheel
              uses: actions/upload-artifact@v3
              with:
                  name: linux-wheel-cmake
                  path: ~/cmake_build/py_pkg/dist/*.whl
                  if-no-files-found: error
                  retention-days: 7

            - name: Upload compute_programl binary
              uses: actions/upload-artifact@v3
              with:
                  name: linux-compute-programl-cmake
                  path: compiler_gym/third_party/programl/compute_programl
                  if-no-files-found: error
                  retention-days: 7

            - name: Upload loop_unroller binary
              uses: actions/upload-artifact@v3
              with:
                  name: linux-loop-unroller-cmake
                  path: examples/example_unrolling_service/loop_unroller/loop_unroller
                  if-no-files-found: error
                  retention-days: 7

            - name: Upload opt_loops binary
              uses: actions/upload-artifact@v3
              with:
                  name: linux-opt-loops-cmake
                  path: examples/loop_optimizations_service/opt_loops/opt_loops
                  if-no-files-found: error
                  retention-days: 7

    build-mlir-env-linux-cmake:
        runs-on: ubuntu-latest
        steps:
            - uses: actions/checkout@v3

            - uses: actions/setup-python@v4
              with:
                  python-version: '3.10'

            - name: Install build dependencies
              uses: ./.github/actions/install-cmake-build-dependencies

            - name: Download LLVM 14.0.0 release
              run: |
                  if [ "$(uname)" = "Darwin" ]; then
                    wget --quiet https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.0/clang+llvm-14.0.0-x86_64-apple-darwin.tar.xz -O ~/llvm.tar.xz
                  else
                    wget --quiet https://github.com/llvm/llvm-project/releases/download/llvmorg-14.0.0/clang+llvm-14.0.0-x86_64-linux-gnu-ubuntu-18.04.tar.xz -O ~/llvm.tar.xz
                  fi
                  mkdir ~/llvm && tar xf ~/llvm.tar.xz --strip-components 1 -C ~/llvm
                  rm ~/llvm.tar.xz
                  echo "Unpacked, testing for expected file:"
                  test -d ~/llvm/lib/cmake/llvm

            - name: CMake Build
              run: |
                  cmake \
                      -GNinja \
                      -DCMAKE_C_COMPILER=clang \
                      -DCMAKE_CXX_COMPILER=clang++ \
                      -DCOMPILER_GYM_LLVM_PROVIDER=external \
                      -DLLVM_DIR=$HOME/llvm/lib/cmake/llvm \
                      -DMLIR_DIR=$HOME/llvm/lib/cmake/mlir \
                      -DClang_DIR=$HOME/llvm/lib/cmake/clang \
                      -DPython3_FIND_UNVERSIONED_NAMES=FIRST \
                      -DCOMPILER_GYM_ENABLE_MLIR_ENV=ON \
                      -DCOMPILER_GYM_ENABLE_LLVM_ENV=OFF \
                      -DCOMPILER_GYM_BUILD_TESTS=ON \
                      -DCOMPILER_GYM_BUILD_EXAMPLES=ON \
                      -S . \
                      -B ~/cmake_build
                  cmake --build ~/cmake_build
              shell: bash

            - name: Upload Python wheel
              uses: actions/upload-artifact@v3
              with:
                  name: mlir-env-linux-wheel-cmake
                  path: ~/cmake_build/py_pkg/dist/*.whl
                  if-no-files-found: error
                  retention-days: 7

    build-macos:
        runs-on: macos-latest
        steps:
            - uses: actions/checkout@v3

            - uses: actions/setup-python@v4
              with:
                  python-version: '3.10'

            - name: Install build dependencies
              uses: ./.github/actions/install-build-dependencies

            - name: Build Python wheel
              run: make bdist_wheel
              env:
                  BAZEL_OPTS: --batch
                  BAZEL_FETCH_OPTS: --config=ci
                  BAZEL_BUILD_OPTS: --config=ci

            - name: Upload Python wheel
              uses: actions/upload-artifact@v3
              with:
                  name: macos-wheel
                  path: dist/*.whl
                  if-no-files-found: error
                  retention-days: 7

    test-linux:
        needs: build-linux
        runs-on: ubuntu-latest
        strategy:
            fail-fast: false
            matrix:
                python: [3.7, 3.8, 3.9, '3.10']
        steps:
            - uses: actions/checkout@v3

            - name: Set up Python ${{ matrix.python }}
              uses: actions/setup-python@v4
              with:
                  python-version: ${{ matrix.python }}

            - name: Download Python wheel
              uses: actions/download-artifact@v3
              with:
                  name: linux-wheel

            - name: Install wheel
              run: python -m pip install *.whl

            - name: Install runtime dependencies
              uses: ./.github/actions/install-runtime-dependencies

            - name: Install test dependencies
              run: python -m pip install -r tests/requirements.txt

            - name: Run the test suite
              run: |
                  make test-cov \
                  PYTEST_ARGS="--ignore tests/llvm --ignore tests/gcc --ignore tests/loop_tool"

            - name: Upload coverage report to Codecov
              uses: codecov/codecov-action@v2

    test-linux-cmake:
        needs: build-linux-cmake
        runs-on: ubuntu-latest
        strategy:
            fail-fast: false
            matrix:
                python: ['3.10']
        steps:
            - uses: actions/checkout@v3

            - name: Set up Python ${{ matrix.python }}
              uses: actions/setup-python@v4
              with:
                  python-version: ${{ matrix.python }}

            - name: Download Python wheel
              uses: actions/download-artifact@v3
              with:
                  name: linux-wheel-cmake

            - name: Install wheel
              run: python -m pip install *.whl

            - name: Install runtime dependencies
              uses: ./.github/actions/install-runtime-dependencies

            - name: Install test dependencies
              run: python -m pip install -r tests/requirements.txt

            - name: Download compute_programl
              uses: actions/download-artifact@v3
              with:
                  name: linux-compute-programl-cmake

            - name: Download loop_unroller
              uses: actions/download-artifact@v3
              with:
                  name: linux-loop-unroller-cmake

            - name: Download opt_loops
              uses: actions/download-artifact@v3
              with:
                  name: linux-opt-loops-cmake

            - name: Move examples and utility binaries
              run: |
                  chmod +x compute_programl
                  chmod +x loop_unroller
                  chmod +x opt_loops
                  mv compute_programl ./compiler_gym/third_party/programl/compute_programl
                  mv loop_unroller ./examples/example_unrolling_service/loop_unroller/loop_unroller
                  mv opt_loops ./examples/loop_optimizations_service/opt_loops/opt_loops
                  test ./compiler_gym/third_party/programl/compute_programl
                  test ./examples/example_unrolling_service/loop_unroller/loop_unroller
                  test ./examples/loop_optimizations_service/opt_loops/opt_loops
              shell: bash

            - name: Run the test suite
              run: make test CI=1 PYTEST_ARGS="--ignore tests/mlir"

    test-mlir-env-linux-cmake:
        needs: build-mlir-env-linux-cmake
        runs-on: ubuntu-latest
        strategy:
            fail-fast: false
            matrix:
                python: [3.7, 3.8, 3.9, '3.10']
        steps:
            - uses: actions/checkout@v3

            - name: Set up Python ${{ matrix.python }}
              uses: actions/setup-python@v4
              with:
                  python-version: ${{ matrix.python }}

            - name: Download Python wheel
              uses: actions/download-artifact@v3
              with:
                  name: mlir-env-linux-wheel-cmake

            - name: Install wheel
              run: python -m pip install *.whl

            - name: Install runtime dependencies
              uses: ./.github/actions/install-mlir-env-runtime-dependencies

            - name: Install test dependencies
              run: python -m pip install -r tests/requirements.txt

            - name: Run the test suite
              run: make test-cov CI=1 TEST_TARGET="tests/mlir"

    test-macos:
        needs: build-macos
        runs-on: macos-latest
        strategy:
            fail-fast: false
            matrix:
                python: [3.8, 3.9, '3.10']
        steps:
            - uses: actions/checkout@v3

            - name: Set up Python ${{ matrix.python }}
              uses: actions/setup-python@v4
              with:
                  python-version: ${{ matrix.python }}

            - name: Download Python wheel
              uses: actions/download-artifact@v3
              with:
                  name: macos-wheel

            - name: Install wheel
              run: python -m pip install *.whl

            - name: Install runtime dependencies
              uses: ./.github/actions/install-runtime-dependencies

            - name: Install test dependencies
              run: python -m pip install -r tests/requirements.txt

            - name: Run the test suite
              run: |
                  make test-cov \
                  CI=1 \
                  PYTEST_ARGS="--ignore tests/llvm --ignore tests/gcc --ignore tests/loop_tool"

            - name: Upload coverage report to Codecov
              uses: codecov/codecov-action@v2
    test-llvm-env-linux:
        needs: build-linux
        runs-on: ubuntu-latest
        strategy:
            fail-fast: false
            matrix:
                python: [3.7, 3.8, 3.9, '3.10']
        steps:
            - uses: actions/checkout@v3

            - name: Set up Python ${{ matrix.python }}
              uses: actions/setup-python@v4
              with:
                  python-version: ${{ matrix.python }}

            - name: Download Python wheel
              uses: actions/download-artifact@v3
              with:
                  name: linux-wheel

            - name: Install wheel
              run: python -m pip install *.whl

            - name: Install runtime dependencies
              uses: ./.github/actions/install-runtime-dependencies

            - name: Install test dependencies
              run: python -m pip install -r tests/requirements.txt

            - name: Run the test suite
              run: make test-cov CI=1 TEST_TARGET="tests/llvm"

            - name: Upload coverage report to Codecov
              uses: codecov/codecov-action@v2
    test-llvm-env-macos:
        needs: build-macos
        runs-on: macos-latest
        strategy:
            fail-fast: false
            matrix:
                python: ['3.10']
        steps:
            - uses: actions/checkout@v3

            - name: Set up Python ${{ matrix.python }}
              uses: actions/setup-python@v4
              with:
                  python-version: ${{ matrix.python }}

            - name: Download Python wheel
              uses: actions/download-artifact@v3
              with:
                  name: macos-wheel

            - name: Install wheel
              run: python -m pip install *.whl

            - name: Install runtime dependencies
              uses: ./.github/actions/install-runtime-dependencies

            - name: Install test dependencies
              run: python -m pip install -r tests/requirements.txt

            - name: Run the test suite
              run: make test-cov CI=1 TEST_TARGET="tests/llvm"

            - name: Upload coverage report to Codecov
              uses: codecov/codecov-action@v2
    test-gcc-env-linux:
        needs: build-linux
        runs-on: ubuntu-latest
        strategy:
            fail-fast: false
            matrix:
                python: ['3.10']
        steps:
            - uses: actions/checkout@v3

            - name: Set up Python ${{ matrix.python }}
              uses: actions/setup-python@v4
              with:
                  python-version: ${{ matrix.python }}

            - name: Download Python wheel
              uses: actions/download-artifact@v3
              with:
                  name: linux-wheel

            - name: Install wheel
              run: python -m pip install *.whl

            - name: Install runtime dependencies
              uses: ./.github/actions/install-runtime-dependencies

            - name: Install test dependencies
              run: python -m pip install -r tests/requirements.txt

            - name: Run the test suite
              run: make test-cov CI=1 TEST_TARGET="tests/gcc"

            - name: Upload coverage report to Codecov
              uses: codecov/codecov-action@v2
    test-gcc-env-linux-no-docker:
        needs: build-linux
        runs-on: ubuntu-latest
        steps:
            - name: Stop docker
              run: |
                  docker ps -q | xargs --no-run-if-empty sudo docker kill
                  sudo systemctl stop docker
                  sudo apt-get remove -y docker docker-engine docker.io containerd runc

            - name: Set up Python
              uses: actions/setup-python@v4
              with:
                  python-version: '3.10'

            - uses: actions/checkout@v3

            - name: Download Python wheel
              uses: actions/download-artifact@v3
              with:
                  name: linux-wheel

            - name: Install wheel
              run: python -m pip install *.whl

            - name: Install runtime dependencies
              uses: ./.github/actions/install-runtime-dependencies

            - name: Install test dependencies
              run: python -m pip install -r tests/requirements.txt

            - name: Run the test suite
              run: make test-cov CI=1 TEST_TARGET="tests/gcc"

            - name: Upload coverage report to Codecov
              uses: codecov/codecov-action@v2

    test-gcc-env-macos:
        needs: build-macos
        runs-on: macos-latest
        steps:
            - uses: actions/checkout@v3

            - name: Set up Python
              uses: actions/setup-python@v4
              with:
                  python-version: '3.10'

            - name: Download Python wheel
              uses: actions/download-artifact@v3
              with:
                  name: macos-wheel

            - name: Install wheel
              run: python -m pip install *.whl

            - name: Install runtime dependencies
              uses: ./.github/actions/install-runtime-dependencies

            - name: Install test dependencies
              run: python -m pip install -r tests/requirements.txt

            - name: Run the test suite
              run: make test-cov CI=1 TEST_TARGET="tests/gcc"

            - name: Upload coverage report to Codecov
              uses: codecov/codecov-action@v2

    test-loop_tool-env-linux:
        needs: build-linux
        runs-on: ubuntu-latest
        steps:
            - name: Set up Python
              uses: actions/setup-python@v4
              with:
                  python-version: '3.10'

            - uses: actions/checkout@v3

            - name: Download Python wheel
              uses: actions/download-artifact@v3
              with:
                  name: linux-wheel

            - name: Install wheel
              run: python -m pip install *.whl

            - name: Install runtime dependencies
              uses: ./.github/actions/install-runtime-dependencies

            - name: Install test dependencies
              run: python -m pip install -r tests/requirements.txt

            - name: Run the test suite
              run: make test-cov CI=1 TEST_TARGET="tests/loop_tool"

            - name: Upload coverage report to Codecov
              uses: codecov/codecov-action@v2

    test-loop_tool-env-macos:
        needs: build-macos
        runs-on: macos-latest
        steps:
            - uses: actions/checkout@v3

            - name: Set up Python
              uses: actions/setup-python@v4
              with:
                  python-version: '3.10'

            - name: Download Python wheel
              uses: actions/download-artifact@v3
              with:
                  name: macos-wheel

            - name: Install wheel
              run: python -m pip install *.whl

            - name: Install runtime dependencies
              uses: ./.github/actions/install-runtime-dependencies

            - name: Install test dependencies
              run: python -m pip install -r tests/requirements.txt

            - name: Run the test suite
              run: make test-cov CI=1 TEST_TARGET="tests/loop_tool"

            - name: Upload coverage report to Codecov
              uses: codecov/codecov-action@v2

    test-examples-linux:
        needs: build-linux
        runs-on: ubuntu-latest
        strategy:
            fail-fast: false
            matrix:
                python: [3.7, 3.8, 3.9, '3.10']
        steps:
            - uses: actions/checkout@v3

            - name: Set up Python ${{ matrix.python }}
              uses: actions/setup-python@v4
              with:
                  python-version: ${{ matrix.python }}

            - name: Download Python wheel
              uses: actions/download-artifact@v3
              with:
                  name: linux-wheel

            - name: Install wheel
              run: python -m pip install *.whl

            - name: Install runtime dependencies
              uses: ./.github/actions/install-runtime-dependencies

            - name: Install test dependencies
              run: make -C examples install

            - name: Run the test suite
              run: make -C examples test PYTEST_ARGS="--ignore example_unrolling_service --ignore loop_optimizations_service"

            - name: Upload coverage report to Codecov
              uses: codecov/codecov-action@v2

    test-examples-macos:
        needs: build-macos
        runs-on: macos-latest
        strategy:
            fail-fast: false
            matrix:
                python: ['3.10']
        steps:
            - uses: actions/checkout@v3

            - name: Set up Python ${{ matrix.python }}
              uses: actions/setup-python@v4
              with:
                  python-version: ${{ matrix.python }}

            - name: Download Python wheel
              uses: actions/download-artifact@v3
              with:
                  name: macos-wheel

            - name: Install wheel
              run: python -m pip install *.whl

            - name: Install runtime dependencies
              uses: ./.github/actions/install-runtime-dependencies

            - name: Install test dependencies
              run: make -C examples install

            - name: Run the test suite
              run: make -C examples test PYTEST_ARGS="--ignore example_unrolling_service --ignore loop_optimizations_service"

            - name: Upload coverage report to Codecov
              uses: codecov/codecov-action@v2

    build-asan-llvm-service:
        runs-on: ubuntu-latest
        steps:
            - uses: actions/checkout@v3

            - name: Setup python
              uses: actions/setup-python@v4
              with:
                  python-version: '3.10'

            - name: Install build dependencies
              uses: ./.github/actions/install-build-dependencies

            - name: Build address sanitized LLVM compiler service
              run: |
                  make bazel-build BAZEL_BUILD_OPTS=--config=asan BUILD_TARGET=//compiler_gym/envs/llvm/service:compiler_gym-llvm-service
                  tar -C bazel-bin/compiler_gym/envs/llvm/service -cjvf compiler_gym-llvm-service.tar.bz2 compiler_gym-llvm-service
              env:
                  CC: clang
                  CXX: clang++
                  BAZEL_OPTS: --batch
                  BAZEL_FETCH_OPTS: --config=ci
                  BAZEL_BUILD_OPTS: --config=ci

            - name: Upload address sanitized LLVM compiler service
              uses: actions/upload-artifact@v3
              with:
                  name: compiler_gym-llvm-service-linux-asan-build
                  path: compiler_gym-llvm-service.tar.bz2
                  if-no-files-found: error
                  retention-days: 7

    test-llvm-env-asan-linux:
        runs-on: ubuntu-latest
        needs: [build-linux, build-asan-llvm-service]
        steps:
            - uses: actions/checkout@v3

            - name: Setup python
              uses: actions/setup-python@v4
              with:
                  python-version: '3.10'

            - name: Download Python wheel
              uses: actions/download-artifact@v3
              with:
                  name: linux-wheel

            - name: Download address sanitized LLVM compiler service
              uses: actions/download-artifact@v3
              with:
                  name: compiler_gym-llvm-service-linux-asan-build

            - name: Repack wheel
              run: |
                  tar xjvf compiler_gym-llvm-service.tar.bz2
                  rm compiler_gym-llvm-service.tar.bz2
                  chmod +x compiler_gym-llvm-service
                  python -m pip install -U wheel
                  wheel unpack *.whl
                  rm *.whl
                  chmod +x compiler_gym-*/compiler_gym-*.data/purelib/compiler_gym/third_party/csmith/csmith/bin/csmith
                  chmod +x compiler_gym-*/compiler_gym-*.data/purelib/compiler_gym/envs/llvm/service/compute_observation
                  chmod +x compiler_gym-*/compiler_gym-*.data/purelib/compiler_gym/envs/llvm/service/llvm-extract-one
                  mv compiler_gym-llvm-service compiler_gym-*/compiler_gym-*.data/purelib/compiler_gym/envs/llvm/service/compiler_gym-llvm-service
                  wheel pack compiler_gym-*

            - name: Install wheel
              run: python -m pip install *.whl

            - name: Install runtime dependencies
              uses: ./.github/actions/install-runtime-dependencies

            - name: Install test dependencies
              run: python -m pip install -r tests/requirements.txt

            - name: Test
              run: make test CI=1 TEST_TARGET=tests/llvm
              env:
                  ASAN_OPTIONS: detect_leaks=1
                  CC: clang
                  CXX: clang++

    www-build:
        runs-on: ${{ matrix.os }}
        strategy:
            fail-fast: false
            matrix:
                os: [ubuntu-latest, macos-latest]
                node: [17]
        steps:
            - uses: actions/checkout@v3

            - uses: actions/setup-python@v4
              with:
                  python-version: '3.10'

            - uses: actions/setup-node@v3
              with:
                  node-version: ${{ matrix.node }}
                  cache: npm
                  cache-dependency-path: www/frontends/compiler_gym/package-lock.json

            - name: Build web frontend
              run: make www-build


Pre-commit

---
name: Pre-commit

on:
    push:
        branches:
            - development
    pull_request:

jobs:
    all:
        name: all
        runs-on: ubuntu-latest

        steps:
            - uses: actions/checkout@v3
            - name: Install clang-format
              run: |
                  wget --quiet -O - https://apt.llvm.org/llvm-snapshot.gpg.key | sudo apt-key add -
                  sudo apt-get update
                  sudo apt-get install clang-format
                  clang-format --version

            - name: Install go
              uses: actions/setup-go@v3
              with:
                  go-version: ^1.13.1

            - name: Install buildifier
              run: |
                  go install github.com/bazelbuild/buildtools/buildifier@latest
                  buildifier --version

            - name: Install prototool
              run: |
                  GO111MODULE=on go install github.com/uber/prototool/cmd/prototool@dev
                  prototool version

            - name: Install hadolint
              run: |
                  wget --quiet -O hadolint https://github.com/hadolint/hadolint/releases/download/v1.19.0/hadolint-Linux-x86_64
                  chmod +x hadolint
                  sudo mv hadolint /usr/local/bin

            - name: Set up Python
              uses: actions/setup-python@v4
              with:
                  python-version: 3.9

            - name: Install Python dependencies
              run: |
                  python3 -m pip install --upgrade wheel
                  python3 -m pip install -r requirements_pre_commit.txt
                  python3 -m isort --version
                  python3 -m pre_commit --version

            - name: Run pre-commit checks
        # TODO(github.com/facebookresearch/CompilerGym/issues/1): Disable
        # isort due to inconsistent results when run locally versus in
        # the GitHub Actions environment.
              run: SKIP=isort python3 -m pre_commit run --all-files --show-diff-on-failure


Release Fuzz

---
# This file defines a GitHub actions job that runs the fuzzing test suite for
# 10 minutes every weekday against the latest CompilerGym release. The job can
# also be scheduled manually using the GitHub actions UI.
#
# TODO(cummins): Set up https://github.com/JasonEtco/create-an-issue
# to create a boilerplate issue if `make fuzz` fails.
name: Release Fuzz

on:
    workflow_dispatch:
    schedule:
        - cron: 0 9 * * 1-5 # every weekday at 9am

jobs:
    fuzz:
        runs-on: ubuntu-latest

        steps:
            - uses: actions/checkout@v3

            - name: Persist the bazel cache
              uses: actions/cache@v2
              with:
                  path: ~/.cache/bazel/_bazel_runner
                  key: bazel-${{ hashFiles('WORKSPACE') }}-${{ hashFiles('**/BUILD') }}
                  restore-keys: |
                      bazel-${{ hashFiles('WORKSPACE') }}-
                      bazel-

            - name: Set up Python
              uses: actions/setup-python@v4
              with:
                  python-version: 3.9

            - name: Install build dependencies
              uses: ./.github/actions/install-build-dependencies

            - name: Build and install Python wheel
              run: make install
              env:
                  CC: clang
                  CXX: clang++
                  BAZEL_OPTS: --batch
                  BAZEL_FETCH_OPTS: --config=ci
                  BAZEL_BUILD_OPTS: --config=ci

            - name: Install runtime dependencies
              uses: ./.github/actions/install-runtime-dependencies

            - name: Install test dependencies
              run: python -m pip install -r tests/requirements.txt

            - name: Fuzz test
              run: FUZZ_TIME=600 make fuzz


Build Binaries

---
# This workflow is used to build the binary wheels that can be uploaded to pypi
# for releases. It produces an artifact containing a macOS and manylinux binary
# wheel from the requested branch. It also runs the full test suite against
# these wheels.
name: Build Binaries

on:
    workflow_dispatch:
    pull_request:
        branches: [stable]
    push:
        branches: [stable]

jobs:
    build:
        runs-on: ${{ matrix.os }}
        strategy:
            matrix:
                os: [macos-latest, ubuntu-latest]
        steps:
            - uses: actions/checkout@v3

            - name: Evaluate artifact name
              run: |
                  echo "ARTIFACT=compiler_gym-$(cat VERSION)" >> $GITHUB_ENV
                  echo "Artifact name is $ARTIFACT"

            - uses: actions/setup-python@v4
              with:
                  python-version: 3.9

            - name: Install build dependencies
              uses: ./.github/actions/install-build-dependencies

            - name: Build Python wheel
              run: |
                  if [ "$(uname)" = "Darwin" ]; then
                    make bdist_wheel
                  else
                    make bdist_wheel-linux
                  fi
              env:
                  BAZEL_OPTS: --batch
                  BAZEL_FETCH_OPTS: --config=ci
                  BAZEL_BUILD_OPTS: --config=ci -c opt

            - name: Upload artifact
              uses: actions/upload-artifact@v3
              with:
                  name: ${{ env.ARTIFACT }}
                  path: dist/*.whl
                  if-no-files-found: error
                  retention-days: 14

    examples-test:
        needs: build
        runs-on: ${{ matrix.os }}
        strategy:
            fail-fast: false
            matrix:
                os: [macos-latest, ubuntu-latest]
                python: [3.7, 3.8, 3.9, '3.10']
        steps:
            - uses: actions/checkout@v3

            - name: Evaluate artifact name
              run: |
                  echo "ARTIFACT=compiler_gym-$(cat VERSION)" >> $GITHUB_ENV
                  echo "Artifact name is $ARTIFACT"

            - name: Set up Python ${{ matrix.python }}
              uses: actions/setup-python@v4
              with:
                  python-version: ${{ matrix.python }}

            - name: Download artifact
              uses: actions/download-artifact@v3
              with:
                  name: ${{ env.ARTIFACT }}

            - name: Install wheel
              run: |
                  if [ "$(uname)" = "Darwin" ]; then
                    python -m pip install compiler_gym-*-macosx*.whl
                  else
                    python -m pip install compiler_gym-*-manylinux*.whl
                  fi

            - name: Install test dependencies
              run: python -m pip install -r examples/requirements.txt -r tests/requirements.txt

            - name: Install runtime dependencies
              uses: ./.github/actions/install-runtime-dependencies

            - name: Run the examples tests
              run: make -C examples test PYTEST_ARGS="--ignore loop_optimizations_service --ignore example_unrolling_service"


Pip install test

---
# This workflow pip install's the compiler_gym package and checks that the
# module can be imported.
#
# The goal of this workflow is to catch failures in the installation process
# that can occur because of a breaking change in the dependent packages, and to
# test for any import-time error.
name: Pip install test

on:
    workflow_dispatch:
    schedule:
        - cron: 0 0 * * * # daily

jobs:
    test:
        runs-on: ${{ matrix.os }}

        strategy:
            fail-fast: false
            matrix:
                os: [ubuntu-latest, macos-latest]
                python: [3.7, 3.8, 3.9]

        steps:
            - uses: actions/setup-python@v4
              with:
                  python-version: ${{ matrix.python }}

            - name: Install python wheel
              # We use the TestPyPI repository to install the compiler_gym
              # package from because we use the install count on the main
              # repository to track growth and usage of the project. We also
              # provide the main PyPI repo as a fallback for installing
              # dependencies.
              run: python -m pip install --index-url https://test.pypi.org/simple/ --extra-index-url https://pypi.org/simple/ compiler_gym

            - name: Check package version
              run: python -c 'import compiler_gym; print(compiler_gym.__version__)'


CodeQL

---
# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: CodeQL

on:
    push:
        branches: [development]
    pull_request:
        # The branches below must be a subset of the branches above
        branches: [development]
    schedule:
        - cron: 18 6 * * 1

jobs:
    analyze:
        name: Analyze
        runs-on: ubuntu-latest
        permissions:
            actions: read
            contents: read
            security-events: write

        strategy:
            fail-fast: false
            matrix:
                language: [python]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'python' ]
        # Learn more:
        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed

        steps:
            - name: Checkout repository
              uses: actions/checkout@v3

            # Initializes the CodeQL tools for scanning.
            - name: Initialize CodeQL
              uses: github/codeql-action/init@v1
              with:
                  languages: ${{ matrix.language }}
            # If you wish to specify custom queries, you can do so here or in a config file.
            # By default, queries listed here will override any specified in a config file.
            # Prefix the list here with "+" to use these queries and those in the config file.
            # queries: ./path/to/local/query, your-org/your-repo/queries@main

            # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
            # If this step fails, then you should remove it and run the build manually (see below)
            - name: Autobuild
              uses: github/codeql-action/autobuild@v1

            # ℹ Command-line programs to run using the OS shell.
            # 📚 https://git.io/JvXDl

            # ✏ If the Autobuild fails above, remove it and uncomment the following three lines
            #    and modify them (or add more) to build your code if your project
            #    uses a compiled language

            #- run: |
            #   make bootstrap
            #   make release

            - name: Perform CodeQL Analysis
              uses: github/codeql-action/analyze@v1


cffconvert

name: cffconvert

on:
  push:
    paths:
      - CITATION.cff
  pull_request:
    paths:
      - CITATION.cff

jobs:
  validate:
    name: "validate"
    runs-on: ubuntu-latest
    steps:
      - name: Check out a copy of the repository
        uses: actions/checkout@v4

      - name: Check whether the citation metadata from CITATION.cff is valid
        uses: citation-file-format/cffconvert-github-action@2.0.0
        with:
          args: "--validate"


on: [status]
jobs:
  circleci_artifacts_redirector_job:
    runs-on: ubuntu-latest
    if: "${{ startsWith(github.event.context, 'ci/circleci: docs-build') }}"
    permissions:
      statuses: write
    name: Run CircleCI artifacts redirector
    steps:
      - name: GitHub Action step
        uses: larsoner/circleci-artifacts-redirector-action@master
        with:
          repo-token: ${{ secrets.GITHUB_TOKEN }}
          api-token: ${{ secrets.CIRCLECI_TOKEN }}
          artifact-path: 0/dev/index.html
          circleci-jobs: docs-build
          job-title: Check the rendered docs here!


build

# Upload a Python Package using Twine when a release is created

name: build
on:  # yamllint disable-line rule:truthy
  release:
    types: [published]
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

permissions:
  contents: read

jobs:
  package:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: '3.12'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install build twine
      - run: python -m build --sdist --wheel
      - run: twine check --strict dist/*
      - uses: actions/upload-artifact@v4
        with:
          name: dist
          path: dist

  pypi-upload:
    needs: package
    runs-on: ubuntu-latest
    if: github.event_name == 'release'
    permissions:
      id-token: write  # for trusted publishing
    environment:
      name: pypi
      url: https://pypi.org/p/mne
    steps:
      - uses: actions/download-artifact@v4
        with:
          name: dist
          path: dist
      - uses: pypa/gh-action-pypi-publish@release/v1
        if: github.event_name == 'release'


name: test_suite

concurrency:
  group: ${{ github.workflow }}-${{ github.event.number }}-${{ github.event.ref }}
  cancel-in-progress: true

on:
  push:
    branches: ['**']
    tags: ['**']
  pull_request:
    branches: ['**']
  schedule:
    - cron: "0 4 * * *"

jobs:
  style:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.12"]
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - uses: actions/cache@v4
      with:
        path: ${{ env.pythonLocation }}
        key: style-0-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install ruff
    - name: Display versions and environment information
      run: |
        python --version
        which python
    - name: Run style & documentation tests
      run: make pep

  build:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest]
        python-version: ["3.9", "3.12"]  # Oldest and newest supported versions
    steps:
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - uses: actions/cache@v4
      with:
        path: ${{ env.pythonLocation }}
        key: build-1-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}

    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install build twine

    - uses: actions/checkout@v4
    - name: Build sdist
      run: python -m build --sdist
    - name: Check sdist
      run: twine check --strict dist/*
    - name: Install sdist
      run: python -m pip install ./dist/mne_bids-*
    - name: Clean up working directory
      run: rm -rf ./*
    - name: Try importing mne_bids
      run: python -c 'import mne_bids; print(mne_bids.__version__)'
    - name: Try cli mne_bids
      run: mne_bids --version
    - name: Remove sdist install
      run: python -m pip uninstall -y mne-bids

    - uses: actions/checkout@v4
    - name: Build wheel
      run: python -m build --wheel
    - name: Check wheel
      run: twine check --strict dist/*
    - name: Install wheel
      run: python -m pip install ./dist/mne_bids-*.whl
    - name: Clean up working directory
      run: rm -rf ./*
    - name: Try importing mne_bids
      run: python -c 'import mne_bids; print(mne_bids.__version__)'
    - name: Try cli mne_bids
      run: mne_bids --version
    - name: Remove wheel install
      run: python -m pip uninstall -y mne-bids

    - uses: actions/checkout@v4
    - name: Test extras install
      run: |
        python -m pip install .[full]
        python -c 'import mne_bids; print(mne_bids.__version__)'
        python -c 'import nibabel; print(nibabel.__version__)'
        python -c 'import pybv; print(pybv.__version__)'
        python -c 'import eeglabio; print(eeglabio.__version__)'
        python -c 'import pymatreader; print(pymatreader.__version__)'
        python -c 'import matplotlib; print(matplotlib.__version__)'
        python -c 'import pandas; print(pandas.__version__)'
        python -c 'import EDFlib; print("EDFlib has no .__version__")'
        python -c 'import edfio; print(edfio.__version__)'

  test:
    # For GitHub "required" CI checks, add in branch protection:
    #
    # 6 checks:
    # for each OS (ubuntu, macos, windows):
    # 3.9 / mne-stable / full / validator-stable
    # 3.12 / mne-stable / full / validator-stable
    #
    # 1 additional check for Apple Silicon (doesn't support Python 3.9):
    # 3.12 / mne-stable / full / validator-stable
    #
    # 5 additional checks with alternative MNE-Python and BIDS validator versions:
    # ubuntu / 3.12 / mne-main / full / validator-main
    # ubuntu / 3.9 / mne-prev / full / validator-stable
    # ubuntu / 3.12 / mne-stable / minimal / validator-stable
    timeout-minutes: 60
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, macos-latest, windows-latest]
        python-version: ["3.9", "3.12"]  # Oldest and newest supported versions
        mne-version: [mne-stable]
        mne-bids-install: [full]
        bids-validator-version: [validator-stable]

        include:
          # special test runs running only on single CI systems to save resources
          #
          # macOS-14 (Apple Silicon) only works with Python 3.10+
          # Once we drop support for Python 3.9, move it to the "proper" matrix above.
          - os: macos-14
            python-version: "3.12"
            mne-version: mne-stable
            mne-bids-install: full
            bids-validator-version: validator-stable

          # Test development versions
          - os: ubuntu-latest
            python-version: "3.12"
            mne-version: mne-main
            mne-bids-install: full
            bids-validator-version: validator-main
          # Test previous MNE stable version
          - os: ubuntu-latest
            python-version: "3.9"
            mne-version: mne-prev-stable
            mne-bids-install: full
            bids-validator-version: validator-stable
          # Test minimal mne-bids install
          - os: ubuntu-latest
            python-version: "3.12"
            mne-version: mne-stable
            mne-bids-install: minimal
            bids-validator-version: validator-stable

    env:
      TZ: Europe/Berlin
      FORCE_COLOR: true
    steps:
    - uses: actions/checkout@v4

    - name: Set up Node.js
      uses: actions/setup-node@v4
      with:
        node-version: 20

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - name: Restore cache
      uses: actions/cache@v4
      with:
        path: ${{ env.pythonLocation }}
        key: test-0-${{ env.pythonLocation }}-${{ env.os }}-${{ hashFiles('pyproject.toml') }}

    - name: Install Python dependencies using pip
      # This would be nicer once this feature is implemented: https://github.com/pypa/pip/issues/11440
      run: |
        python -m pip install --upgrade pip
        python -m pip install -e .[test]

    - name: Install MNE (stable)
      if: matrix.mne-version == 'mne-stable'
      run: |
        git clone --single-branch --branch maint/1.6 https://github.com/mne-tools/mne-python.git
        python -m pip install -e ./mne-python

    - name: Install MNE (previous stable)
      if: matrix.mne-version == 'mne-prev-stable'
      run: |
        git clone --single-branch --branch maint/1.5 https://github.com/mne-tools/mne-python.git
        python -m pip install -e ./mne-python

    - name: Install MNE (main)
      if: matrix.mne-version == 'mne-main'
      run: |
        git clone --single-branch --branch main https://github.com/mne-tools/mne-python.git
        python -m pip install -e ./mne-python

    - name: Install BIDS validator (stable)
      if: matrix.bids-validator-version == 'validator-stable'
      run: |
        npm install -g bids-validator
        echo "BIDS_VALIDATOR_BRANCH=stable" >> $GITHUB_ENV

    - name: Download BIDS validator (main)
      if: matrix.bids-validator-version == 'validator-main'
      run: |
        pushd ..
        git clone --depth 1 https://github.com/bids-standard/bids-validator
        popd
        echo "BIDS_VALIDATOR_BRANCH=main" >> $GITHUB_ENV

    - name: Install BIDS validator (main)
      if: matrix.bids-validator-version == 'validator-main'
      run: |
        pushd ..
        cd bids-validator
        # Generate the full development node_modules
        npm install
        # Build & bundle the bids-validator CLI package
        npm -w bids-validator run build
        # Generate a package to install globally
        npm -w bids-validator pack
        # Install the package globally
        bash -c "npm install -g bids-validator-*.tgz"
        popd

    - name: Display versions and environment information
      run: |
        echo $TZ
        date
        echo "npm"; npm --version
        echo "node"; node --version
        echo "bids-validator"; bids-validator --version
        echo "python"; python --version
        which python
        mne sys_info

    - name: Install MNE-BIDS (minimal)
      if: matrix.mne-bids-install == 'minimal'
      run: python -m pip install -e .

    - name: Install MNE-BIDS (full)
      if: matrix.mne-bids-install == 'full'
      run: python -m pip install -e .[full]

    # Only run on a limited set of jobs
    - name: Run pytest without testing data
      run: make test
      if: ${{ matrix.python-version == '3.12' && matrix.os == 'ubuntu-latest' && matrix.mne-version == 'mne-main' }}

    # Get testing data
    - run: ./tools/get_testing_version.sh
      working-directory: mne-python
      shell: bash
      name: 'Get testing version'

    - uses: actions/cache@v4
      with:
        key: ${{ env.TESTING_VERSION }}
        path: ~/mne_data
      name: 'Cache testing data'

    - run: ./tools/github_actions_download.sh
      shell: bash
      working-directory: mne-python

    - name: Run pytest
      shell: bash
      run: make test

    - name: Upload coverage stats to codecov
      uses: codecov/codecov-action@v4
      with:
        files: ./coverage.xml

  build_docs:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.12"]
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - uses: actions/cache@v4
      with:
        path: ${{ env.pythonLocation }}
        key: build_docs-0-${{ env.pythonLocation }}-${{ hashFiles('pyproject.toml') }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install --upgrade https://github.com/mne-tools/mne-python/archive/refs/heads/main.zip
        python -m pip install -e .[test,doc]
    - name: Build the documentation
      run: |
        make build-doc
    - name: Upload artifacts
      uses: actions/upload-artifact@v4
      with:
        name: documentation
        path: doc/_build/html


Checks

name: Checks

on: [pull_request, push]

permissions:
  contents: read

jobs:
  spelling_checker:
    name: "Spelling"
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: codespell-project/actions-codespell@master
        with:
          ignore_words_list: msdos, pullrequest
          skip: ./.git,coverity,rpmbuild,samples

  python_code_linters:
      name: "Python Linters"
      runs-on: ubuntu-latest
      steps:
      - name: "Clone Repository"
        uses: actions/checkout@v4
      - name: "Run Linters"
        uses: osbuild/containers/src/actions/privdocker@552e30cf1b4ed19c6ddaa57f96c342b3dff4227b
        with:
          image: ghcr.io/osbuild/osbuild-ci:latest-202304251412
          run: |
            make lint

  shell_linters:
    name: "Shell Linters"
    runs-on: ubuntu-latest

    steps:
    - name: "Clone Repository"
      uses: actions/checkout@v4
      with:
        fetch-depth: 0

    - name: "Differential ShellCheck"
      uses: redhat-plumbers-in-action/differential-shellcheck@v3
      with:
        severity: warning
        token: ${{ secrets.GITHUB_TOKEN }}

  packit-config-lint:
    name: "📦 Packit config lint"
    runs-on: ubuntu-latest
    container:
      image: registry.fedoraproject.org/fedora:latest
    steps:
      - name: Install Packit
        run: dnf -y install packit

      - name: Check out code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}

      - name: Validate Packit config
        run: |
          packit validate-config .packit.yaml

  snapshots:
    name: "🔍 Check for valid snapshot urls"
    runs-on: ubuntu-20.04
    steps:
      - name: Check out code into the Go module directory
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - name: Check for valid snapshot urls
        run: ./tools/check-snapshots --errors-only .

  json-fmt:
    name: "🔍 Check JSON files for formatting consistency"
    runs-on: ubuntu-20.04
    steps:
      - name: Install utils
        run: |
          sudo apt update
          sudo apt install -y jq moreutils
      - name: Check out code
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.pull_request.head.sha }}
      - name: Reformat all json files
        run: find -iname "*.json" -print -exec sh -c 'jq --indent 2 . {} | sponge {}' \;
      - name: Check diff
        run: git diff --exit-code


Coverity

name: Coverity
on:
  schedule:
    - cron: '0 5 * * *' # Daily at 05:00 UTC

jobs:
  coverity:
    name: "Test Suite"
    if: github.repository == 'osbuild/osbuild'
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: osbuild
    steps:

      - name: Clone repository
        uses: actions/checkout@v4
        with:
          path: osbuild

      - name: Install Dependencies
        run: |
          sudo apt-get install python3-setuptools

      - name: Download Coverity Tool
        run: |
          make coverity-download
        env:
         COVERITY_TOKEN: ${{ secrets.COVERITY_TOKEN }}

      - name: Coverity check
        run: |
          make coverity-check

      - name: Upload analysis results
        run: |
          make coverity-submit
        env:
         COVERITY_TOKEN: ${{ secrets.COVERITY_TOKEN }}
         COVERITY_EMAIL: ${{ secrets.COVERITY_EMAIL }}


Create and push release tag

# This action creates a release every second Wednesday
name: "Create and push release tag"

on:
  workflow_dispatch:
  schedule:
    - cron: "0 8 * * 3"

jobs:
  tag-and-push:
    runs-on: ubuntu-latest

    steps:
      - name: Even or odd week
        run: if [ `expr \`date +\%s\` / 86400 \% 2` -eq 0 ]; then echo "WEEK=odd" >> $GITHUB_ENV; else echo "WEEK=even" >> $GITHUB_ENV; fi
        shell: bash

      - name: Upstream tag
        uses: osbuild/release-action@create-tag
        if: ${{ env.WEEK == 'even' || github.event_name != 'schedule' }}
        with:
          token: "${{ secrets.SCHUTZBOT_GITHUB_ACCESS_TOKEN }}"
          username: "imagebuilder-bot"
          email: "imagebuilder-bots+imagebuilder-bot@redhat.com"


Generate

name: Generate

on: [pull_request, push]

jobs:
  generate_documentation:
    name: "Documentation"
    runs-on: ubuntu-latest
    container:
      image: docker.io/library/python:3.7
    steps:
      - name: Install Dependencies
        run: |
          pip install docutils

      - name: Clone repository
        uses: actions/checkout@v4
        with:
          path: osbuild

      - name: Generate Documentation
        run: |
          make \
            -f osbuild/Makefile \
            SRCDIR=osbuild \
            BUILDDIR=build \
            RST2MAN=rst2man.py \
            man

      - name: Verify Documentation
        working-directory: build
        run: |
          test -d docs
          test -f docs/osbuild.1

  generate_test_data:
    name: "Test Data"
    runs-on: ubuntu-latest
    env:
      OSBUILD_MPP_CACHEDIR: "/var/tmp/osbuild-mpp-cache"
    steps:
    - name: "Clone Repository"
      uses: actions/checkout@v4
    - name: Cache metadata
      uses: actions/cache@v4
      with:
        path: /var/tmp/osbuild-mpp-cache
        key: no-key-needed-here
    - name: "Regenerate Test Data"
      uses: osbuild/containers/src/actions/privdocker@552e30cf1b4ed19c6ddaa57f96c342b3dff4227b
      with:
        image: ghcr.io/osbuild/osbuild-ci:latest-202304251412
        run: |
          make test-data
          git diff --exit-code -- ./test/data


Verify PR best practices

name: "Verify PR best practices"

on:
  pull_request_target:
    branches: [ main ]

jobs:
  pr-best-practices:
    runs-on: ubuntu-latest
    steps:
      - name: PR best practice check
        uses: osbuild/pr-best-practices@main
        with:
          token: ${{ secrets.SCHUTZBOT_GITHUB_ACCESS_TOKEN }}


Update osbuild ref in manifest-db

name: Update osbuild ref in manifest-db

on:
  push:
    branches:
      - main

jobs:
  propagate:
    runs-on: ubuntu-latest
    env:
      SCHUTZBOT_GH: ${{ secrets.SCHUTZBOT_GITHUB_ACCESS_TOKEN }}
    steps:
      - name: Checkout OSBuild
        uses: actions/checkout@v3
        with:
          path: osbuild
          fetch-depth: 0

      - name: Update Manifest-db
        run: |
          # update only if manifest-db SHA was updated in the last commit
          cd osbuild
          ACTUAL_MDB_SHA=$(git show HEAD:Schutzfile | jq -r '.global.dependencies."manifest-db".commit')
          PREVIOUS_MDB_SHA=$(git show HEAD^:Schutzfile | jq -r '.global.dependencies."manifest-db".commit')
          if [[ $ACTUAL_MDB_SHA == $PREVIOUS_MDB_SHA ]] ; then
            echo "The manifest-db SHA was not updated in the last commit, ignoring the dependency update"
            exit 0
          fi
          echo "updating the dependency in manifest-db"
          OSBUILD_HEAD="${{ github.sha }}"
          echo "Latest OSBuild SHA: $OSBUILD_HEAD"

          # Clone manifest-db
          cd ..
          git clone https://github.com/osbuild/manifest-db.git

          # Load the previous osbuild REF from manifest-db's Schutzfile and
          # Generate the commit list between the two osbuild revisions for the
          # PR
          cd manifest-db
          OLD_REF=$(jq -r '.global.dependencies.osbuild.commit' Schutzfile)
          cd ../osbuild
          COMMIT_LIST=$(git log --oneline $OLD_REF..$OSBUILD_HEAD | sed 's/.*/- https:\/\/github.com\/osbuild\/osbuild\/commit\/&/')

          # Update the manifest-db's dependency to the newest OSBuild
          #
          # Login as Schutzbot
          cd ../manifest-db
          echo "${SCHUTZBOT_GH}" | gh auth login --with-token
          git config --local user.name "SchutzBot"
          git config --local user.email "imagebuilder-bots+schutzbot@redhat.com"

          # Create a branch for the PR
          now=$(date '+%Y-%m-%d-%H%M%S')
          BRANCH_NAME="osbuild-update-$now"
          git checkout -b $BRANCH_NAME

          # change the value for the commit head in the schutzfile
          jq --arg variable "$OSBUILD_HEAD" '.global.dependencies.osbuild.commit=$variable' Schutzfile > Schutzfile.tmp && mv Schutzfile.tmp Schutzfile

          # create the PR
          PR_BODY="$(cat <<-END
          This PR updates the osbuild ref dependency for manifest-db. Between the
          last time it was updated, and this new reference commit, these are the changes:

          $COMMIT_LIST
          END
          )"
          git remote add upstream https://schutzbot:"$SCHUTZBOT_GH"@github.com/schutzbot/manifest-db.git
          git add -A && \
              git commit -m "schutzfile: update osbuild ref $(date '+%Y-%m-%d')" && \
              git push upstream "$BRANCH_NAME:$BRANCH_NAME" && \
              gh pr create \
                  --title "schutzfile: update osbuild ref $(date '+%Y-%m-%d')" \
                  --body "$PR_BODY" \
                  --repo "osbuild/manifest-db" \
                  -r lavocatt


Create GitHub release

name: "Create GitHub release"

on:
  push:
    tags:
      - "v*"

jobs:
  release:
    runs-on: ubuntu-latest
    steps:
      - name: Upstream release
        uses: osbuild/release-action@main
        with:
          token: "${{ secrets.SCHUTZBOT_GITHUB_ACCESS_TOKEN }}"
          slack_webhook_url: "${{ secrets.SLACK_WEBHOOK_URL }}"


Mark and close stale issues and PRs

name: Mark and close stale issues and PRs

on:
  schedule:
    - cron: '0 4 * * *'

jobs:
  stale:
    runs-on: ubuntu-latest
    permissions:
      issues: write
      pull-requests: write
    steps:
      - uses: osbuild/common-stale-action@main
        with:
          token: ${{ secrets.GITHUB_TOKEN }}


Tests

name: Tests

on: [pull_request, push]

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  # Share the store between the workers speeds things up further
  OSBUILD_TEST_STORE: /var/tmp/osbuild-test-store

jobs:
  test_suite:
    name: "Unittest"
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        test:
          - parallel
          - normal
        environment:
          - "py36"    # RH8
          - "py39"    # RH9
          - "py312"   # latest fedora
    steps:
    - name: "Clone Repository"
      uses: actions/checkout@v4
    - name: "Run"
      uses: osbuild/containers/src/actions/privdocker@552e30cf1b4ed19c6ddaa57f96c342b3dff4227b
      with:
        image: ghcr.io/osbuild/osbuild-ci:latest-202404252004
        run: |
          # Hacky replacement of container storage driver:
          # The default overlayfs doesn't work in the runner, so let's change
          # it to vfs for the local storage skopeo stage test.
          sed -i 's/overlay/vfs/g' /usr/share/containers/storage.conf  # default system config
          sed -i 's/overlay/vfs/g' /etc/containers/storage.conf || true  # potential overrides
          if [ "${{ matrix.test }}" == "parallel" ]; then
              # 4 is a bit arbitrary
              TEST_WORKERS="-n 4"
              TEST_CATEGORY="test_stages.py"
          else
              # test_assemblers.py is run below
              TEST_CATEGORY="not test_stages.py and not test_assemblers.py"
          fi
          OSBUILD_TEST_STORE="${{ env.OSBUILD_TEST_STORE }}" \
          tox -e "${{ matrix.environment }}" -- -rs $TEST_WORKERS -k "$TEST_CATEGORY"

  v1_manifests:
    name: "Assembler test (legacy)"
    runs-on: ubuntu-latest
    steps:
      - name: "Clone Repository"
        uses: actions/checkout@v4
      - name: "Run"
        uses: osbuild/containers/src/actions/privdocker@552e30cf1b4ed19c6ddaa57f96c342b3dff4227b
        env:
          # Using 4 workers is a bit arbitrary, "auto" is probably too aggressive.
          TEST_WORKERS: "-n 4"
        with:
          image: ghcr.io/osbuild/osbuild-ci:latest-202404252004
          run: |
            OSBUILD_TEST_STORE="${{ env.OSBUILD_TEST_STORE }}" \
            tox -e "py36" -- ${{ env.TEST_WORKERS }} test.run.test_assemblers


Trigger GitLab CI

# inspired by rhinstaller/anaconda

name: Trigger GitLab CI

on:
  workflow_run:
    workflows: ["Checks"]
    types: [completed]

jobs:
  trigger-gitlab:
    if: ${{ github.event.workflow_run.conclusion == 'success' }}
    runs-on: ubuntu-latest
    env:
      IMAGEBUILDER_BOT_GITLAB_SSH_KEY: ${{ secrets.IMAGEBUILDER_BOT_GITLAB_SSH_KEY }}
    steps:
      - name: Report status
        uses: haya14busa/action-workflow_run-status@v1

      - name: Install Dependencies
        run: |
          sudo apt install -y jq

      - name: Clone repository
        uses: actions/checkout@v4
        with:
          ref: ${{ github.event.workflow_run.head_sha }}
          fetch-depth: 0

      - uses: octokit/request-action@v2.x
        id: fetch_pulls
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          route: GET /repos/${{ github.repository }}/pulls

      - name: Checkout branch
        run: |
          PR_DATA=$(mktemp)
          # use uuid as a file terminator to avoid conflicts with data content
          cat > "$PR_DATA" <<'a21b3e7f-d5eb-44a3-8be0-c2412851d2e6'
          ${{ steps.fetch_pulls.outputs.data }}
          a21b3e7f-d5eb-44a3-8be0-c2412851d2e6

          PR=$(jq -rc '.[] | select(.head.sha | contains("${{ github.event.workflow_run.head_sha }}")) | select(.state | contains("open"))' "$PR_DATA" | jq -r .number)
          if [ ! -z "$PR" ]; then
            git checkout -b PR-$PR
          else
            git checkout ${{ github.event.workflow_run.head_branch }}
          fi

      - name: Push to gitlab
        run: |
          mkdir -p ~/.ssh
          echo "${IMAGEBUILDER_BOT_GITLAB_SSH_KEY}" > ~/.ssh/id_rsa
          chmod 400 ~/.ssh/id_rsa
          touch ~/.ssh/known_hosts
          ssh-keyscan -t rsa gitlab.com >> ~/.ssh/known_hosts
          git remote add ci git@gitlab.com:redhat/services/products/image-builder/ci/osbuild.git
          git push -f ci
          git push -f --tags ci


Quality Check

name: Quality Check

on:
  push:
    branches:
      main
  pull_request:
    types: [edited, synchronize, opened, reopened]
    branches:
      main
  workflow_dispatch:

jobs:
  cancel:
    name: cancel previous runs
    runs-on: ubuntu-latest

    steps:
      - uses: styfle/cancel-workflow-action@0.11.0
        with:
          all_but_latest: true
          access_token: ${{ github.token }}

  poetry--quality-check:
    name: quality check
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]

    steps:
      - name: check out repository
        uses: actions/checkout@v3

      - name: set up Python
        id: setup-python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: load cached venv if available
        id: cached-poetry-dependencies
        uses: actions/cache@v2
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

      - name: install dependencies and test tools if cache does not exist
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction --no-root --with test

      - name: run quality check with "make check" command
        run: make check

  poetry-test:
    name: Pytest with Poetry
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.12"]

    steps:
      - name: check out repository
        uses: actions/checkout@v3

      - name: set up Python
        id: setup-python
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: load cached venv if available
        id: cached-poetry-dependencies
        uses: actions/cache@v2
        with:
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}

      - name: install dependencies and test tools if cache does not exist
        if: steps.cached-poetry-dependencies.outputs.cache-hit != 'true'
        run: poetry install --no-interaction --no-root --with test

      - name: run tests
        run: poetry run pytest tests

      - name: upload test coverage to Codecov
        uses: codecov/codecov-action@v3

  pip-test:
    name: test with pip
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]

    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: pip install and test
        run: |
          pip install pip --upgrade
          pip install .
          pip install pytest pytest-cov pytest-xdist
          pytest -v --maxfail=1 tests -n auto

  anaconda_pip:
    name: test with Anaconda environment
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: ["windows-latest"]
        python-version: ["3.12"]

    steps:
      - uses: actions/checkout@v3
      - name: Setup conda
        uses: s-weigand/setup-conda@v1
        with:
          update-conda: true
          python-version: ${{ matrix.python-version }}
          conda-channels: anaconda, conda-forge

      - name: installation of CovsirPhy
        run: |
          conda install -c conda-forge pip fiona ruptures
          pip install .

  codeql:
    name: discover vulnerabilities with CodeQL
    runs-on: ubuntu-latest
    permissions:
      security-events: write

    steps:
      - name: Checkout repository
        uses: actions/checkout@v3
        with:
          fetch-depth: 2

      - name: Initialize CodeQL
        uses: github/codeql-action/init@v2
        with:
          languages: python

      - name: Autobuild
        uses: github/codeql-action/autobuild@v2

      - name: Perform CodeQL Analysis
        uses: github/codeql-action/analyze@v2


# https://github.com/marketplace/actions/verify-linked-issue
name: Verify the pull request has linked issues

on:
  pull_request:
    types: [edited, synchronize, opened, reopened]

jobs:
  verify_linked_issue:
    name: Ensure Pull Request has a linked issue
    runs-on: ubuntu-latest

    steps:
      - name: Verify Linked Issue
        uses: hattan/verify-linked-issue-action@v1.1.1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


Update documentation

name: Update documentation

on:
  push:
    branches:
      - main
    paths:
      - ".github/workflows/**"
      - "./covsirphy/__version__.py"
      - "docs/**"
  workflow_dispatch:

jobs:
  cancel:
    name: cancel previous runs
    runs-on: ubuntu-latest

    steps:
      - uses: styfle/cancel-workflow-action@0.11.0
        with:
          all_but_latest: true
          access_token: ${{ github.token }}

  docs:
    name: update documentation with Sphinx
    runs-on: ubuntu-latest

    steps:
      - name: check out repository
        uses: actions/checkout@v3

      - name: set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.12"

      - name: install Poetry
        uses: snok/install-poetry@v1
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: install dependencies
        run: poetry install --no-interaction --with docs

      - name: create html files with "make docs" command
        run: make docs

      - name: Git add & commit
        uses: EndBug/add-and-commit@v9
        with:
          add: "."
          default_author: github_actor
          message: "update documentation with Sphinx"
          push: true


Rebase Pull Requests

name: Rebase Pull Requests

on:
  push:
    branches: [main]

jobs:
  rebase:
    runs-on: ubuntu-latest
    steps:
      - uses: linhbn123/rebase-pull-requests@v1.0.1


npm

name: npm

on:
  push:
  pull_request:
    types: [opened, synchronize]

env:
  node-version: 6.x

jobs:
  test:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v3
    - name: Use Node.js ${{ env.node-version }}
      uses: actions/setup-node@v3
      with:
        node-version: ${{ env.node-version }}
    - name: Run npm test
      timeout-minutes: 10
      run: |
        npm install -g grunt-cli@1.3
        npm install
        npm test


Quality & Tests

name: Quality & Tests

on: pull_request

permissions:
  contents: read
  pull-requests: write

jobs:
  tests:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - uses: actions/cache@v4
        id: cache
        with:
          path: ${{ env.pythonLocation }}
          key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt', 'requirements-tests.txt') }}

      - name: Install requirements
        if: steps.cache.outputs.cache-hit != 'true'
        run: python -m pip install -r requirements-tests.txt

      - name: Quality checks, and linters
        run: ./check.sh

      - name: Unit tests
        run: python -Wd -m pytest tests --doctest-modules wikidict

      - name: Automerge
        if: github.actor == 'dependabot[bot]'
        uses: pascalgn/automerge-action@v0.16.3
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          MERGE_LABELS: automerge,!DO_NOT_MERGE
          MERGE_METHOD: squash
          MERGE_DELETE_BRANCH: true


Update dictionaries

name: Update dictionaries

on:
  schedule:
    # At 01:00 (UTC) every day
    - cron: "0 1 * * *"

  # Allow to call the workflow manually
  workflow_dispatch:

jobs:
  # First job just to fill the pip cache
  warmup:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - uses: actions/cache@v4
        id: cache
        with:
          path: ${{ env.pythonLocation }}
          key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}

      - name: Install requirements
        if: steps.cache.outputs.cache-hit != 'true'
        run: python -m pip install -r requirements.txt

  update:
    runs-on: ubuntu-latest
    needs: [warmup]

    # List of locales to generate
    strategy:
      fail-fast: false
      matrix:
        locale: # XXX_LOCALES
          - "ca"
          - "da"
          - "de"
          - "el"
          - "en"
          - "es"
          - "fr"
          - "it"
          - "no"
          - "pt"
          - "ro"
          - "ru"
          - "sv"

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - uses: actions/cache@v4
        id: cache
        with:
          path: ${{ env.pythonLocation }}
          key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}

      - name: Download the latest Wiktionary dump
        run: python -Wd -m wikidict ${{ matrix.locale }} --download

      - name: Parse the dump
        run: python -Wd -m wikidict ${{ matrix.locale }} --parse

      - name: Render templates
        run: python -Wd -m wikidict ${{ matrix.locale }} --render

      - name: Convert to dictionaries
        run: python -Wd -m wikidict ${{ matrix.locale }} --convert

      - name: Generate the release description
        run: python -Wd -m wikidict ${{ matrix.locale }} --release > release.txt

      - name: Upload release data
        if: github.ref == 'refs/heads/master'
        uses: softprops/action-gh-release@v2
        with:
          tag_name: ${{ matrix.locale }}
          body_path: release.txt
          files: |
            data/${{ matrix.locale }}/dicthtml-${{ matrix.locale }}-${{ matrix.locale }}*.zip
            data/${{ matrix.locale }}/dicthtml-${{ matrix.locale }}-${{ matrix.locale }}*.zip.sha256
            data/${{ matrix.locale }}/dict-${{ matrix.locale }}-${{ matrix.locale }}*.df.bz2
            data/${{ matrix.locale }}/dict-${{ matrix.locale }}-${{ matrix.locale }}*.df.bz2.sha256
            data/${{ matrix.locale }}/dict-${{ matrix.locale }}-${{ matrix.locale }}*.zip
            data/${{ matrix.locale }}/dict-${{ matrix.locale }}-${{ matrix.locale }}*.zip.sha256


Update local-specific data

name: Update local-specific data

on:
  schedule:
    # At 00:00 (UTC) every day
    - cron: "0 0 * * *"

  # Allow to call the workflow manually
  workflow_dispatch:

jobs:
  auto-update-data:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - uses: actions/cache@v4
        id: cache
        with:
          path: ${{ env.pythonLocation }}
          key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt', 'requirements-tests.txt') }}

      - name: Install requirements
        if: steps.cache.outputs.cache-hit != 'true'
        run: python -m pip install -r requirements-tests.txt

      - name: Update data
        id: update-data
        run: |
          python -m scripts
          ./check.sh || true

      - name: Push changes
        uses: Automattic/action-commit-to-branch@master
        with:
          branch: master
          commit_message: "Update locale-specific data"
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

      - name: Fail on error
        if: steps.update-data.outputs.errors != '0'
        run: exit 1


Daily random word

name: Daily random word

on:
  schedule:
    # At 06:00 (UTC) every day
    - cron: "0 6 * * *"

  # Allow to call the workflow manually
  workflow_dispatch:

jobs:
  # First job just to fill the pip cache
  warmup:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - uses: actions/cache@v4
        id: cache
        with:
          path: ${{ env.pythonLocation }}
          key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}

      - name: Install requirements
        if: steps.cache.outputs.cache-hit != 'true'
        run: python -m pip install -r requirements.txt

  daily:
    runs-on: ubuntu-latest
    needs: [warmup]

    strategy:
      fail-fast: false
      matrix:
        locale: # XXX_LOCALES
          - "ca"
          - "da"
          - "de"
          - "el"
          - "en"
          - "es"
          - "fr"
          - "it"
          - "no"
          - "pt"
          - "ro"
          - "ru"
          - "sv"

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - uses: actions/cache@v4
        id: cache
        with:
          path: ${{ env.pythonLocation }}
          key: ${{ env.pythonLocation }}-${{ hashFiles('requirements.txt') }}

      - name: Get a random word
        id: get-word
        run: python -Wd -m wikidict ${{ matrix.locale }} --get-word=''

      - name: Check that random word
        run: python -Wd -m wikidict ${{ matrix.locale }} --check-word=${{ steps.get-word.outputs.word }}