CI

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# GH actions.
# We use it to cover windows and mac builds
# Jenkins is still the primary CI

name: CI

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
  workflow_dispatch:

concurrency:
  group: CI-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

jobs:
  MacOS:
    if: ${{ github.repository == 'apache/tvm' }}
    runs-on: macOS-latest
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: 'recursive'
      - name: Set up environment
        uses: ./.github/actions/setup
      - name: Conda Build
        shell: bash -l {0}
        run: >-
          conda build --output-folder=conda/pkg  conda/recipe &&
          conda install tvm -c ./conda/pkg
      - name: Build iOS RPC
        run: |
          IOS_VERSION="14.0"
          CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Release \
                       -DCMAKE_SYSTEM_NAME=iOS \
                       -DCMAKE_SYSTEM_VERSION=${IOS_VERSION} \
                       -DCMAKE_OSX_SYSROOT=iphonesimulator \
                       -DCMAKE_OSX_ARCHITECTURES=x86_64 \
                       -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
                       -DCMAKE_BUILD_WITH_INSTALL_NAME_DIR=ON \
                       -DUSE_IOS_RPC=ON"

          mkdir build-ios-simulator
          cd build-ios-simulator
          cmake .. ${CMAKE_FLAGS}
          cmake --build . --target ios_rpc
      - name: Test
        shell: bash -l {0}
        run: >-
          python -m pytest -v tests/python/all-platform-minimal-test
      - name: Minimal Metal Compile-Only
        shell: bash -l {0}
        run: |
          python -m pytest -v -s 'tests/python/codegen/test_gpu_codegen_allreduce.py::test_allreduce_sum_compile'
          python -m pytest -v -s 'tests/python/codegen/test_target_codegen_metal.py::test_func_with_trailing_pod_params'
      - name: Minimal Metal Compile-and-Run
        shell: bash -l {0}
        run: >-
          python -m pytest -v -s 'tests/python/codegen/test_target_codegen_metal.py'
          python -m pytest -v -s 'tests/python/codegen/test_target_codegen_gpu_common.py'
          python -m pytest -v -s 'tests/python/codegen/test_gpu_codegen_allreduce.py::test_allreduce_sum[dims0-metal]'
#      - name: Test iOS RPC
#        shell: bash -l {0}
#        run: >-
#          python -m pip install tornado psutil cloudpickle &&
#          export PYTHONPATH=tests/python/contrib:${PYTHONPATH} &&
#          export BUNDLE_ID=org.apache.tvmrpc &&
#          export BUNDLE_PATH=build-ios-simulator/apps/ios_rpc/ios_rpc/src/ios_rpc-build/Release-iphonesimulator/tvmrpc.app &&
#          python -m pytest -v tests/python/contrib/test_rpc_server_device.py

  Windows:
    if: ${{ github.repository == 'apache/tvm' }}
    runs-on: windows-2019
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: 'recursive'
      - name: Set up environment
        uses: ./.github/actions/setup
      - name: Conda Build
        shell: cmd /C call {0}
        run: >-
          conda build --output-folder=conda/pkg conda/recipe &&
          conda install tvm -c ./conda/pkg
      - name: Test
        shell: cmd /C call {0}
        run: >-
          python -m pytest -v tests/python/all-platform-minimal-test

  # Disabled due to https://github.com/apache/tvm/issues/13950
  # Windows-Static-Runtime:
  #   if: ${{ github.repository == 'apache/tvm' }}
  #   runs-on: windows-2019
  #   steps:
  #     - uses: actions/checkout@v2
  #       with:
  #         submodules: 'recursive'
  #     - name: Set up environment
  #       uses: ./.github/actions/setup
  #     - name: Build static TVM runtime
  #       shell: bash -l {0}
  #       run: |
  #         tests/scripts/task_config_build_static.sh build
  #         cd build
  #         cmake .. -A x64 -DCMAKE_CONFIGURATION_TYPES="Release"
  #         cmake --build . --config Release --target runtime

  Linux-Static-Runtime:
    if: ${{ github.repository == 'apache/tvm' }}
    runs-on: Ubuntu-20.04
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: 'recursive'
      - name: Set up environment
        uses: ./.github/actions/setup
      - name: Build static TVM runtime
        shell: bash -l {0}
        run: |
          tests/scripts/task_config_build_static.sh build
          cd build
          cmake ..
          cmake --build . --config Release --target runtime

  Android:
    if: ${{ github.repository == 'apache/tvm' }}
    runs-on: Ubuntu-20.04
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: 'recursive'
      - name: Set up environment
        uses: ./.github/actions/setup
      - name: Set up java
        uses: actions/setup-java@v3
        with:
          distribution: 'zulu'
          java-version: '11'
      - name: Build TVM
        shell: bash -l {0}
        run: |
          mkdir build
          cd build
          ../tests/scripts/task_config_build_jvm.sh .
          cmake ..
          make
      - name: Build TVM4J
        run: |
          make jvmpkg
      - name: Build android_rpc
        working-directory: apps/android_rpc
        run: |
          set -eux
          export PATH="${ANDROID_NDK_LATEST_HOME}:$PATH"
          gradle clean build
      - name: Upload android_rpc APK
        uses: actions/upload-artifact@v2
        with:
          name: android_rpc-debug.apk
          path: ./apps/android_rpc/app/build/outputs/apk/debug/app-debug.apk
      - name: Build android_deploy
        working-directory: apps/android_deploy
        run: |
          set -eux
          export PATH="${ANDROID_NDK_LATEST_HOME}:$PATH"
          gradle clean build
      - name: Upload android_deploy APK
        uses: actions/upload-artifact@v2
        with:
          name: android_deploy-debug.apk
          path: ./apps/android_deploy/app/build/outputs/apk/debug/app-debug.apk


# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# GH actions.
# We use it to cover windows and mac builds
# Jenkins is still the primary CI

name: CI

on:
  push:
    branches:
      - mlc
  pull_request:
    branches:
      - mlc
  workflow_dispatch:

concurrency:
  group: CI-${{ github.event.pull_request.number || github.sha }}
  cancel-in-progress: true

jobs:
  MacOS:
    if: ${{ github.repository == 'mlc-ai/relax' }}
    runs-on: macOS-latest
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: 'recursive'
      - name: Set up environment
        uses: ./.github/actions/setup
      - name: Conda Build
        shell: bash -l {0}
        run: >-
          conda build --output-folder=conda/pkg  conda/recipe &&
          conda install tvm -c ./conda/pkg
      - name: Build iOS RPC
        run: |
          IOS_VERSION="14.0"
          CMAKE_FLAGS="-DCMAKE_BUILD_TYPE=Release \
                       -DCMAKE_SYSTEM_NAME=iOS \
                       -DCMAKE_SYSTEM_VERSION=${IOS_VERSION} \
                       -DCMAKE_OSX_SYSROOT=iphonesimulator \
                       -DCMAKE_OSX_ARCHITECTURES=x86_64 \
                       -DCMAKE_OSX_DEPLOYMENT_TARGET=14.0 \
                       -DCMAKE_BUILD_WITH_INSTALL_NAME_DIR=ON \
                       -DUSE_IOS_RPC=ON"

          mkdir build-ios-simulator
          cd build-ios-simulator
          cmake .. ${CMAKE_FLAGS}
          cmake --build . --target ios_rpc
      - name: Test
        shell: bash -l {0}
        run: >-
          python -m pytest -v tests/python/all-platform-minimal-test

  Windows:
    if: ${{ github.repository == 'mlc-ai/relax' }}
    runs-on: windows-2019
    steps:
      - uses: actions/checkout@v2
        with:
          submodules: 'recursive'
      - name: Set up environment
        uses: ./.github/actions/setup
      - name: Conda Build
        shell: cmd /C call {0}
        run: >-
          conda build --output-folder=conda/pkg conda/recipe &&
          conda install tvm -c ./conda/pkg
      - name: Test
        shell: cmd /C call {0}
        run: >-
          python -m pytest -v tests/python/all-platform-minimal-test


Nightly Docker Update

name: Nightly Docker Update
on:
  schedule:
    - cron: "0 0 * * *"
  workflow_dispatch:

concurrency:
  group: nightly-docker-update
  cancel-in-progress: true

jobs:
  open_update_pr:
    permissions:
      actions: write
      checks: write
      contents: write
      id-token: write
      issues: write
      pull-requests: write
      statuses: write
    if: github.repository == 'apache/tvm'
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v2
      - name: Open PR to update Docker images
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -eux
          python ci/scripts/jenkins/open_docker_update_pr.py


Ping Reviewers

name: Ping Reviewers
on:
  schedule:
    - cron: "0/15 * * * *"
  workflow_dispatch:

concurrency:
  group: ping
  cancel-in-progress: true

jobs:
  ping:
    if: github.repository == 'apache/tvm'
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v2
      - name: Ping reviewers
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -eux
          python ci/scripts/github/ping_reviewers.py --wait-time-minutes 10080 || echo failed


Teams

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

name: Teams

on:
  issues:
    types: [opened, edited, reopened, labeled]

concurrency:
  group: Teams-${{ github.event.pull_request.number }}-${{ github.event.issue.number }}
  cancel-in-progress: true

jobs:
  tag-teams:
    if: github.repository == 'apache/tvm'
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: Tag people from relevant teams
        env:
          PR: ${{ toJson(github.event.pull_request) }}
          ISSUE: ${{ toJson(github.event.issue) }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -eux
          python ci/scripts/github/github_tag_teams.py || echo failed


tvm-bot

name: tvm-bot
on:
  pull_request_review:
    types:
      - submitted
  issue_comment:

concurrency:
  group: merge-${{ github.event.pull_request.number }}-${{ github.event.issue.number }}
  cancel-in-progress: true

jobs:
  run-tvm-bot:
    permissions:
      actions: write
      checks: write
      contents: write
      id-token: write
      issues: write
      pull-requests: write
      statuses: write
    if: ${{ github.event.issue.pull_request && github.repository == 'apache/tvm' }}
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v2
      - name: Run tvm-bot
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          GH_ACTIONS_TOKEN: ${{ secrets.GH_ACTIONS_TOKEN }}
          TVM_BOT_JENKINS_TOKEN: ${{ secrets.TVM_BOT_JENKINS_TOKEN }}
          PR_NUMBER: ${{ github.event.issue.number }}
          ISSUE_COMMENT: ${{ toJson(github.event.comment) }}
          RUN_URL: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}
        run: |
          set -eux
          python ci/scripts/github/github_tvmbot.py --pr "$PR_NUMBER" --run-url "$RUN_URL" --trigger-comment-json "$ISSUE_COMMENT"


Update last-successful branch

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# GH actions.
# We use it to cover windows and mac builds
# Jenkins is still the primary CI

name: Update last-successful branch

on:
  schedule:
    - cron: "0/15 * * * *"
  workflow_dispatch:

concurrency:
  group: update-last-successful-branch
  cancel-in-progress: true

jobs:
  update-last-successful-branch:
    if: github.repository == 'apache/tvm'
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v2
      - name: Update last-successful branch
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -eux
          python ci/scripts/github/update_branch.py || echo step failed


Update nightly branch

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# Update a branch for nightly test results
name: Update nightly branch

on:
  schedule:
    # 9 PM PST
    - cron: "0 5 * * *"
  workflow_dispatch:

concurrency:
  group: update-nightly-branch
  cancel-in-progress: true

jobs:
  update-nightly-branch:
    if: github.repository == 'apache/tvm'
    runs-on: ubuntu-20.04
    steps:
      - uses: actions/checkout@v2
      - name: Update nightly branch
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          set -eux
          git checkout -B nightly
          git log -5
          git push origin --force nightly


Upload CI Resource

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.
name: Upload CI Resource

on:
  workflow_dispatch:
    inputs:
      url:
        description: 'URL of the file (e.g. "https://example.com/file.zip")'
        required: true
        type: string
      sha256:
        description: 'SHA256 of the file'
        required: true
        type: string
      upload_path:
        description: 'Path of the file in S3 (e.g. "my_folder/something.zip")'
        required: true
        type: string

concurrency:
  group: upload-ci-resource
  cancel-in-progress: true

jobs:
  upload-ci-resource:
    if: github.repository == 'apache/tvm'
    runs-on: ubuntu-20.04
    steps:
      - name: Download item and upload to S3
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          AWS_ACCESS_KEY_ID: ${{ secrets.CI_RESOURCES_AWS_ACCESS_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_RESOURCES_AWS_SECRET_ACCESS_KEY }}
          URL: ${{ inputs.url }}
          SHA256: ${{ inputs.sha256 }}
          UPLOAD_PATH: ${{ inputs.upload_path }}
          AWS_DEFAULT_REGION: us-west-2
        run: |
          set -eux
          curl -L -o downloaded_file "$URL"
          echo "$SHA256 downloaded_file" | sha256sum --check
          aws s3 cp downloaded_file "s3://tvm-ci-resources/$UPLOAD_PATH"
          echo "The item is available at https://tvm-ci-resources.s3.us-west-2.amazonaws.com/$UPLOAD_PATH"
          echo "Add this line to tests/scripts/request_hook/request_hook.py"
          echo "    \"$URL\": f\"{BASE}/$UPLOAD_PATH\","


Build and Test

# Copyright The Caikit Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Build and Test

on:
  push:
    branches: [ "main" ]
  pull_request:
    branches: [ "main" ]

jobs:
  build:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version:
          - setup: '3.8'
            tox: 'py38'
          - setup: '3.9'
            tox: 'py39'
          - setup: '3.10'
            tox: 'py310'
          - setup: '3.11'
            tox: 'py311'
          # Special matrix entry to ensure protobuf 3 compatibility
          - setup: '3.8'
            tox: 'proto3'
          # Special matrix entry to ensure core tests pass without optional deps
          - setup: '3.8'
            tox: 'core'

    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version.setup }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version.setup }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install -r setup_requirements.txt
      - name: Build and test with tox
        run: tox -e ${{ matrix.python-version.tox }} -- tests


Lint and Format

# Copyright The Caikit Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Lint and Format

on:
  push:
    branches: [ "main" ]
  pull_request:
    branches: [ "main" ]

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python 3.9
        uses: actions/setup-python@v4
        with:
          python-version: 3.9
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install -r setup_requirements.txt
      - name: Check Formatting
        run: tox -e fmt
      - name: Linting
        run: tox -e lint
      - name: Setup Graphviz # `graphviz/dot` is required for the import checker
        uses: ts-graphviz/setup-graphviz@v1
      - name: Enforce import rules
        run: tox -e imports


Publish

# Copyright The Caikit Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

name: Publish

on:
  release:
    types: [published]

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python
        uses: actions/setup-python@v3
      - name: Build and check package
        run: |
          pip install tox
          tox -e build,twinecheck
      - name: Upload package
        if: github.event_name == 'release'
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.PYPI_TOKEN }}


TestNDeploy

name: TestNDeploy

on: [push, pull_request]

jobs:
  cache:
    name: Caching
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [ 3.8, 3.9, "3.10", "3.11", "3.12" ]

    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Get pip cache dir
        id: pip-cache-dir
        run: |
          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
      - name: pip cache
        id: pip-cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.pip-cache-dir.outputs.dir }}
          key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
      - name: Update pip
        if: ${{ steps.pip-cache.outputs.cache-hit != 'true' }}
        run: |
          python -m pip install --upgrade pip
      - name: Install project dependencies
        if: ${{ steps.pip-cache.outputs.cache-hit != 'true' }}
        run: |
          pip install -r requirements-dev.txt

  lint:
    name: Linting
    runs-on: ubuntu-latest
    needs: cache
    strategy:
      fail-fast: false
      matrix:
        python-version: [3.8, 3.9, "3.10", "3.11", "3.12"]
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    # Retrieve the previously cached dependencies
    - name: Get pip cache dir
      id: pip-cache
      run: |
        echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
    - name: pip cache
      uses: actions/cache@v4
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
    # Update PIP
    - name: Update pip
      run: |
        python -m pip install --upgrade pip
    # Still need to properly install the dependencies - it will only skip the download part
    - name: Install project dependencies
      run: |
        pip install -r requirements-dev.txt
    - name: Lint
      run: |
        mkdir .mypy_cache
        make lint

  javatest:
    needs: lint
    uses: ./.github/workflows/tests_sdk_java.yml

  dotnettest:
    needs: lint
    uses: ./.github/workflows/tests_sdk_dotnet.yml

  rubytest:
    needs: lint
    uses: ./.github/workflows/tests_sdk_ruby.yml

  terraformexamplestest:
    needs: lint
    uses: ./.github/workflows/tests_terraform_examples.yml

  test:
    needs: [lint]
    if: "!contains(github.event.pull_request.labels.*.name, 'java')"
    uses: ./.github/workflows/tests_decoratormode.yml
    secrets: inherit

  testserver:
    needs: [lint]
    if: "!contains(github.event.pull_request.labels.*.name, 'java')"
    uses: ./.github/workflows/tests_servermode.yml
    secrets: inherit

  testproxy:
    needs: [lint]
    if: "!contains(github.event.pull_request.labels.*.name, 'java')"
    uses: ./.github/workflows/tests_proxymode.yml

  release:
    name: Release
    runs-on: ubuntu-latest
    permissions:
      id-token: write
      packages: write
      pull-requests: write
    needs: [dotnettest, javatest, test, testserver ]
    if: ${{ github.event_name == 'push' && github.ref == 'refs/heads/master' && github.repository == 'getmoto/moto' }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Update & Build
        run: |
          pip install build
          python update_version_from_git.py
          python -m build
      - name: Publish to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          verbose: true
          print-hash: true
      - name: Build Docker release
        run: |
          docker build -t motoserver/moto . --tag moto:latest
      # Required to get the correct Digest
      # See https://github.com/docker/build-push-action/issues/461
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to DockerHub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Login to GHCR
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}
      - name: Push to DockerHub and GHCR
        uses: docker/build-push-action@v6
        with:
          push: true
          platforms: linux/amd64,linux/arm64
          tags: |
            motoserver/moto:latest
            ghcr.io/getmoto/motoserver:latest
      - name: Get version number
        run: |
          version=$(grep -oP '(?<=__version__ = ")[0-9.a-z]+(?=")' moto/__init__.py)
          echo "moto_version=$version" >> $GITHUB_ENV
      - uses: octokit/graphql-action@v2.x
        name: Get PR info
        id: get_pr
        with:
          query: |
            query get_pr($owner:String!,$repo:String!,$commit:GitObjectID) {
              repository(owner:$owner,name:$repo) {
                object(oid:$commit) {
                  ... on Commit {
                    associatedPullRequests(last: 1){
                      edges {
                        node {
                          baseRepository {
                            nameWithOwner
                          }
                          merged
                          number
                        }
                      }
                    }
                  }
                }
              }
            }
          owner: ${{ github.event.repository.owner.name }}
          repo: ${{ github.event.repository.name }}
          commit: "${{ github.sha }}"
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Get PR number
        run: |
          nr="${{ fromJSON(steps.get_pr.outputs.data).repository.object.associatedPullRequests.edges[0].node.number }}"
          repo="${{ fromJSON(steps.get_pr.outputs.data).repository.object.associatedPullRequests.edges[0].node.baseRepository.nameWithOwner }}"
          if [ -z "$nr" ]
          then
            echo "PR nr not found in $msg"
            echo "pr_found=false" >> $GITHUB_ENV
          else
            echo "PR NR: $nr"
            echo "pr_nr=$nr" >> $GITHUB_ENV
            echo "pr_repo=$repo" >> $GITHUB_ENV
            echo "pr_found=true" >> $GITHUB_ENV
          fi
      - name: Leave PR comment with Moto version
        uses: peter-evans/create-or-update-comment@v4
        if: env.pr_found == 'true' && env.pr_repo == 'getmoto/moto'
        with:
          issue-number: ${{ env.pr_nr }}
          body: |
            This is now part of moto >= ${{ env.moto_version }}


CodeQL

# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"

on:
  schedule:
    - cron: '00 12 * * 4'

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write

    strategy:
      fail-fast: false
      matrix:
        language: [ 'python' ]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
        # Learn more:
        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v3
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
        # By default, queries listed here will override any specified in a config file.
        # Prefix the list here with "+" to use these queries and those in the config file.
        # queries: ./path/to/local/query, your-org/your-repo/queries@main

    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
      uses: github/codeql-action/autobuild@v3

    # ℹ Command-line programs to run using the OS shell.
    # 📚 https://git.io/JvXDl

    # ✏ If the Autobuild fails above, remove it and uncomment the following three lines
    #    and modify them (or add more) to build your code if your project
    #    uses a compiled language

    #- run: |
    #   make bootstrap
    #   make release

    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v3


# Data Update:
#   Config Managed Rules
#
# This Github Action:
#   - executes the script that updates the Config Managed Rules that come bundled with Moto
#   - creates a PR
#
name: "DataUpdate_ConfigManagedRules"

on:
  schedule:
    - cron: '00 6 1 * *'
  workflow_dispatch:

jobs:
  update:
    name: Update Config Managed Rules
    runs-on: ubuntu-latest
    if: ${{ github.ref == 'refs/heads/master' && github.repository == 'getmoto/moto' }}
    permissions:
      id-token: write
      contents: write
      pull-requests: write

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    - name: Set up Python 3.9
      uses: actions/setup-python@v5
      with:
        python-version: 3.9

    - name: Pull Config managed rules from AWS
      run: |
        pip install -r requirements-dev.txt
        cd scripts
        ./pull_down_aws_managed_rules.py
    - name: Create PR
      uses: peter-evans/create-pull-request@v6
      with:
        branch: 'chore-update-config-managed-rules-${{ github.run_number }}'
        title: 'chore: update Config Managed Rules'
        commit-message: 'chore: update Config Managed Rules'


# Data Update:
#   EC2 Instance Offerings
#
# This Github Action:
#   - executes the script that updates the EC2 instance offerings that come bundled with Moto
#   - creates a PR
#
name: "DataUpdate_EC2InstanceOfferings"

on:
  schedule:
    - cron: '00 6 * * 0'
  workflow_dispatch:

jobs:
  update:
    name: Update EC2 Instance Offerings
    runs-on: ubuntu-latest
    if: ${{ github.ref == 'refs/heads/master' && github.repository == 'getmoto/moto' }}
    permissions:
      id-token: write
      contents: write
      pull-requests: write

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    - name: Set up Python 3.9
      uses: actions/setup-python@v5
      with:
        python-version: 3.9

    - name: Configure AWS
      uses: aws-actions/configure-aws-credentials@v4
      with:
        aws-region: us-east-1
        role-to-assume: arn:aws:iam::486285699788:role/GithubActionsRole

    - name: Pull EC2 instance types from AWS
      run: |
        pip install boto3
        scripts/ec2_get_instance_type_offerings.py
    - name: Create PR
      uses: peter-evans/create-pull-request@v6
      with:
        branch: 'chore-update-ec2-offerings-${{ github.run_number }}'
        title: 'chore: update EC2 Instance Offerings'
        commit-message: 'chore: update EC2 Instance Offerings'


# Data Update:
#   EC2 Instance Types
#
# This Github Action:
#   - executes the script that updates the EC2 instance types that come bundled with Moto
#   - creates a PR
#
name: "DataUpdate_EC2InstanceTypes"

on:
  schedule:
    - cron: '00 8 * * 0'

jobs:
  update:
    name: Update EC2 Instance Types
    runs-on: ubuntu-latest
    if: ${{ github.ref == 'refs/heads/master' && github.repository == 'getmoto/moto' }}
    permissions:
      id-token: write
      contents: write
      pull-requests: write

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    - name: Set up Python 3.9
      uses: actions/setup-python@v5
      with:
        python-version: 3.9

    - name: Configure AWS
      uses: aws-actions/configure-aws-credentials@v4
      with:
        aws-region: us-east-1
        role-to-assume: arn:aws:iam::486285699788:role/GithubActionsRole

    - name: Pull EC2 instance types from AWS
      run: |
        pip install boto3
        scripts/get_instance_info.py
    - name: Create PR
      uses: peter-evans/create-pull-request@v6
      with:
        branch: 'chore-update-ec2-types-${{ github.run_number }}'
        title: 'chore: update EC2 Instance Types'
        commit-message: 'chore: update EC2 Instance Types'


# Data Update:
#   IAM Managed Policies
#
# This Github Action:
#   - executes the script that updates the IAM Managed Policies that come bundled with Moto
#   - creates a PR
#
name: "DataUpdate_IAMManagedPolicies"

on:
  schedule:
    - cron: '00 6 1 * *'
  workflow_dispatch:

jobs:
  update:
    name: Update IAM Managed Policies
    runs-on: ubuntu-latest
    if: ${{ github.ref == 'refs/heads/master' && github.repository == 'getmoto/moto' }}
    permissions:
      id-token: write
      contents: write
      pull-requests: write

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    - name: Set up Python 3.9
      uses: actions/setup-python@v5
      with:
        python-version: 3.9

    - name: Configure AWS
      uses: aws-actions/configure-aws-credentials@v4
      with:
        aws-region: us-east-1
        role-to-assume: arn:aws:iam::486285699788:role/GithubActionsRole

    - name: Pull IAM managed policies from AWS
      run: |
        pip install -r requirements-dev.txt
        make aws_managed_policies
    - name: Create PR
      uses: peter-evans/create-pull-request@v6
      with:
        branch: 'chore-update-iam-managed-policies-${{ github.run_number }}'
        title: 'chore: update IAM Managed Policies'
        commit-message: 'chore: update IAM Managed Policies'


# Data Update:
#   SSM default AMIs
#
# This Github Action:
#   - executes the script that updates the SSM default AMI's that come bundled with Moto
#   - creates a PR
#
name: "DataUpdate_SSMdefaultAMIs"

on:
  schedule:
    - cron: '00 10 1 * *'

jobs:
  update:
    name: Update SSM default AMIs
    runs-on: ubuntu-latest
    if: ${{ github.ref == 'refs/heads/master' && github.repository == 'getmoto/moto' }}
    permissions:
      id-token: write
      contents: write
      pull-requests: write

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    - name: Set up Python 3.9
      uses: actions/setup-python@v5
      with:
        python-version: 3.9

    - name: Configure AWS
      uses: aws-actions/configure-aws-credentials@v4
      with:
        aws-region: us-east-1
        role-to-assume: arn:aws:iam::486285699788:role/GithubActionsRole

    - name: Pull SSM default AMIs from AWS
      run: |
        pip install boto3
        pip install moto[all]
        python scripts/ssm_get_default_amis.py
    - name: Create PR
      uses: peter-evans/create-pull-request@v6
      with:
        branch: 'chore-update-ssm-amis-${{ github.run_number }}'
        title: "chore: update SSM Instance AMI's"
        commit-message: "chore: update SSM default AMI's"


# Data Update:
#   SSM default parameters
#
# This Github Action:
#   - executes the script that updates the SSM default parameters that come bundled with Moto
#   - creates a PR
#
name: "DataUpdate_SSMdefaultParameters"

on:
  schedule:
    - cron: '00 12 * * 0'
  workflow_dispatch:

jobs:
  update:
    name: Update SSM default parameters
    runs-on: ubuntu-latest
    if: ${{ github.ref == 'refs/heads/master' && github.repository == 'getmoto/moto' }}
    permissions:
      id-token: write
      contents: write
      pull-requests: write

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    - name: Set up Python 3.9
      uses: actions/setup-python@v5
      with:
        python-version: 3.9

    - name: Configure AWS
      uses: aws-actions/configure-aws-credentials@v4
      with:
        aws-region: us-east-1
        role-to-assume: arn:aws:iam::486285699788:role/GithubActionsRole

    - name: Pull SSM default Parameters from AWS
      run: |
        pip install boto3
        pip install moto[all]
        python scripts/ssm_get_default_params.py
    - name: Create PR
      uses: peter-evans/create-pull-request@v6
      with:
        branch: 'chore-update-ssm-params-${{ github.run_number }}'
        title: "chore: update SSM Instance parameters"
        commit-message: "chore: update SSM default parameters"


Service-specific Dependencies Test

name: "Service-specific Dependencies Test"

on:
  workflow_dispatch:
  schedule:
    - cron: '0 0 * * *'  # every day at midnight

jobs:
  prepare_list:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - id: set-matrix
      run: echo "matrix=[$(ls -A1 moto | grep -v '.py' | awk '{ printf "%s\047%s\047", (NR==1?"":", "), $0 } END{ print "" }')]" >> $GITHUB_OUTPUT
    - run: echo "matrix=[$(ls -A1 moto | grep -v '.py' | awk '{ printf "%s\047%s\047", (NR==1?"":", "), $0 } END{ print "" }')]"
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
  runtest:
    name: Run Dependency Test
    runs-on: ubuntu-latest
    needs: prepare_list
    strategy:
      matrix:
        python-version: [ 3.8 ]
        service: ${{ fromJson(needs.prepare_list.outputs.matrix) }}

    steps:
    - name: Checkout repo
      uses: actions/checkout@v4
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Run test
      env:
        AWS_ACCESS_KEY_ID: key
        AWS_SECRET_ACCESS_KEY: secret
      run: |
        scripts/dependency_test.sh ${{ matrix.service }}


DockerTests

name: DockerTests

on: [push, pull_request]

jobs:
  cache:
    name: Caching
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: [ "3.11" ]

    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
      - name: Get pip cache dir
        id: pip-cache-dir
        run: |
          echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
      - name: pip cache
        id: pip-cache
        uses: actions/cache@v4
        with:
          path: ${{ steps.pip-cache-dir.outputs.dir }}
          key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
      - name: Update pip
        if: ${{ steps.pip-cache.outputs.cache-hit != 'true' }}
        run: |
          python -m pip install --upgrade pip
      - name: Install project dependencies
        if: ${{ steps.pip-cache.outputs.cache-hit != 'true' }}
        run: |
          pip install -r requirements-dev.txt

  test_custom_port:
    name: Test Custom Port
    runs-on: ubuntu-latest
    needs: cache
    strategy:
      matrix:
        python-version: ["3.11"]

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Start MotoServer on an unusual port
      run: |
        pip install --upgrade build
        python -m build
        docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e MOTO_PORT=4555 -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 4555:4555 -v /var/run/docker.sock:/var/run/docker.sock python:3.11-slim /moto/scripts/ci_moto_server.sh &
        MOTO_PORT=4555 python scripts/ci_wait_for_server.py
    - name: Get pip cache dir
      id: pip-cache
      run: |
        echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
    - name: pip cache
      uses: actions/cache@v4
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
    - name: Update pip
      run: |
        python -m pip install --upgrade pip
    - name: Install project dependencies
      run: |
        pip install -r requirements-dev.txt
    - name: Test
      env:
        TEST_SERVER_MODE: ${{ true }}
        MOTO_PORT: 4555
      run: |
        pytest -sv tests/test_awslambda/test_lambda_invoke.py::test_invoke_lambda_using_environment_port tests/test_cloudformation/test_cloudformation_custom_resources.py
    - name: Collect Logs
      if: always()
      run: |
        mkdir serverlogs1
        cp server_output.log serverlogs1/server_output.log
        docker stop motoserver
    - name: Archive Logs
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: custom_port
        path: |
          serverlogs1/*

  test_custom_name:
    name: Test Custom Network Name
    runs-on: ubuntu-latest
    needs: cache
    strategy:
      matrix:
        python-version: ["3.11"]

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Start MotoServer on a custom Docker network bridge
      run: |
        pip install --upgrade build
        python -m build
        docker network create -d bridge my-custom-network
        docker run --rm -t -e TEST_SERVER_MODE=true -e MOTO_DOCKER_NETWORK_NAME=my-custom-network -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 --network my-custom-network -v /var/run/docker.sock:/var/run/docker.sock python:3.11-slim /moto/scripts/ci_moto_server.sh &
        python scripts/ci_wait_for_server.py
    - name: Get pip cache dir
      id: pip-cache
      run: |
        echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
    - name: pip cache
      uses: actions/cache@v4
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
    - name: Update pip
      run: |
        python -m pip install --upgrade pip
    - name: Install project dependencies
      run: |
        pip install -r requirements-dev.txt
    - name: Test
      env:
        TEST_SERVER_MODE: ${{ true }}
      run: |
        pytest -sv tests/test_awslambda/test_lambda_invoke.py::test_invoke_lambda_using_environment_port tests/test_cloudformation/test_cloudformation_custom_resources.py
    - name: Collect Logs
      if: always()
      run: |
        mkdir serverlogs2
        cp server_output.log serverlogs2/server_output.log
    - name: Archive logs
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: custom_name
        path: |
          serverlogs2/*

  test_custom_networkmode:
    name: Pass NetworkMode to AWSLambda
    runs-on: ubuntu-latest
    needs: cache
    strategy:
      matrix:
        python-version: ["3.11"]

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Start MotoServer on an unusual port
      run: |
        pip install --upgrade build
        python -m build
        docker run --rm -t -e MOTO_DOCKER_NETWORK_MODE=host -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e MOTO_PORT=4555 -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 4555:4555 -v /var/run/docker.sock:/var/run/docker.sock python:3.11-slim /moto/scripts/ci_moto_server.sh &
        MOTO_PORT=4555 python scripts/ci_wait_for_server.py
    - name: Get pip cache dir
      id: pip-cache
      run: |
        echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
    - name: pip cache
      uses: actions/cache@v4
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
    - name: Update pip
      run: |
        python -m pip install --upgrade pip
    - name: Install project dependencies
      run: |
        pip install -r requirements-dev.txt
    - name: Test
      env:
        TEST_SERVER_MODE: ${{ true }}
        MOTO_PORT: 4555
        MOTO_DOCKER_NETWORK_MODE: host
      run: |
        pytest -sv tests/test_awslambda/test_lambda_invoke.py::test_invoke_lambda_using_networkmode tests/test_cloudformation/test_cloudformation_custom_resources.py
    - name: Collect Logs
      if: always()
      run: |
        mkdir serverlogs3
        cp server_output.log serverlogs3/server_output.log
    - name: Archive Logs
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: custom_mode
        path: |
          serverlogs3/*


Release

name: Release

on:
  workflow_dispatch:
    inputs:
      version:
        description: 'Version'
        required: true
      current_release:
        description: 'True to release Docker and update version info in moto/__init__.py'
        required: true
        type: boolean

jobs:
  release-moto-job:
    runs-on: ubuntu-latest
    name: Release Moto
    permissions:
      contents: write
      id-token: write
      packages: write
    env:
      VERSION: 0.0.0
    steps:
    - name: Set Env
      run: |
        echo "VERSION=${{ github.event.inputs.version }}" >> $GITHUB_ENV
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python 3.8
      uses: actions/setup-python@v5
      with:
        python-version: 3.8
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install wheel setuptools packaging twine build --upgrade
        pip install -r requirements-dev.txt
    - name: Verify Tag does not exist
      run: |
        ! git rev-parse ${{ env.VERSION }} || { echo "Ensure that no tag exists for ${{ env.VERSION }}" ; exit 1; }
    - name: Verify supplied version exists in the CHANGELOG
      run: |
        grep ${{ env.VERSION }} CHANGELOG.md || { echo "Ensure that the CHANGELOG contains an entry for ${{ env.VERSION }}" ; exit 1; }
    - name: Set version number
      run: python update_version_from_git.py ${{ env.VERSION }}
    - name: Build Python
      run: python -m build
    - name: Build Docker release
      if: ${{ inputs.current_release }}
      run: |
        docker build -t motoserver/moto . --tag moto:${{ env.VERSION }}
    - name: Test Docker release
      if: ${{ inputs.current_release }}
      run: |
        docker run -p 5000:5000 --name moto motoserver/moto &
        TEST_SERVER_MODE=true pytest -sv tests/test_core tests/test_s3/test_s3.py
        docker stop moto
    - name: Publish to PyPI
      uses: pypa/gh-action-pypi-publish@release/v1
    - name: Tag version on Github
      run: |
        git tag ${{ env.VERSION }}
        git push origin ${{ env.VERSION }}
    - name: Create GitHub release
      uses: softprops/action-gh-release@v2
      with:
        name: ${{ env.VERSION }}
        tag_name: ${{ env.VERSION }}
        files: dist/*
    # Required to get the correct Digest
    # See https://github.com/docker/build-push-action/issues/461
    - name: Set up QEMU
      if: ${{ inputs.current_release }}
      uses: docker/setup-qemu-action@v3
    - name: Set up Docker Buildx
      if: ${{ inputs.current_release }}
      uses: docker/setup-buildx-action@v3
    - name: Login to DockerHub
      if: ${{ inputs.current_release }}
      uses: docker/login-action@v3
      with:
        username: ${{ secrets.DOCKERHUB_USERNAME }}
        password: ${{ secrets.DOCKERHUB_TOKEN }}
    - name: Login to GHCR
      if: ${{ inputs.current_release }}
      uses: docker/login-action@v3
      with:
        registry: ghcr.io
        username: ${{ github.actor }}
        password: ${{ secrets.GITHUB_TOKEN }}
    - name: Push to DockerHub and GHCR
      if: ${{ inputs.current_release }}
      id: build_and_push
      uses: docker/build-push-action@v6
      with:
        platforms: linux/amd64,linux/arm64
        push: true
        tags: |
          motoserver/moto:${{ env.VERSION }}
          ghcr.io/getmoto/motoserver:${{ env.VERSION }}
    - name: Increase patch version number
      if: ${{ inputs.current_release }}
      run: |
        python update_version_from_git.py patch
        sed -i 's/Docker Digest for ${{ env.VERSION }}: <autopopulateddigest>/Docker Digest for ${{ env.VERSION }}: _${{ steps.build_and_push.outputs.digest }}_/' CHANGELOG.md
        git config --local user.email "admin@getmoto.org"
        git config --local user.name "Moto Admin"
        git add moto/__init__.py
        git add CHANGELOG.md
        git commit -m "Post-release steps"
        git push


Outdated Dependency Tests

# Run separate test cases to verify Moto works with older versions of dependencies
#
name: "Outdated Dependency Tests"

on:
  pull_request:
    types: [ labeled ]

jobs:
  test:
    if: ${{ github.event.label.name == 'moto-core' }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        botocore: ["--upgrade boto3 botocore", "--upgrade boto3 botocore && pip install urllib3==2.2.1", "boto3==1.11.0 botocore==1.14.0"]
        python-version: [ "3.11" ]
        responses-version: ["0.15.0", "0.17.0", "0.19.0", "0.20.0" ]
        werkzeug-version: ["2.0.1", "2.1.1", "2.2.2"]
        openapi-spec-validator-version: ["0.5.0"]
        cryptography-version: ["39.0.0"]

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - name: Update pip
      run: |
        python -m pip install --upgrade pip

    - name: Install project dependencies
      run: |
        pip install -r requirements-dev.txt
        pip install responses==${{ matrix.responses-version }}
        pip install flask==${{ matrix.werkzeug-version }}
        pip install werkzeug==${{ matrix.werkzeug-version }}
        pip install openapi-spec-validator==${{ matrix.openapi-spec-validator-version }}
        pip install ${{ matrix.botocore }}
        pip install cryptography==${{ matrix.cryptography-version }}

    - name: Run tests
      run: |
        pytest -sv tests/test_core ./tests/test_apigateway/test_apigateway_integration.py tests/test_iotdata ./tests/test_s3/test_server.py

    - name: Start MotoServer
      run: |
        python -m build
        docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:3.9-slim /moto/scripts/ci_moto_server.sh &
        python scripts/ci_wait_for_server.py
    - name: Test ServerMode/Coverage
      env:
        TEST_SERVER_MODE: ${{ true }}
      run: |
        pytest -sv tests/test_core tests/test_awslambda tests/test_cloudformation tests/test_iotdata
    - name: "Stop MotoServer"
      if: always()
      run: |
        mkdir serverlogs
        cp server_output.log serverlogs/server_output.log
        docker stop motoserver
    - name: Archive TF logs
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: test-${{ matrix.responses-version }}-${{ matrix.werkzeug-version }}-${{ matrix.werkzeug-version }}-${{ matrix.openapi-spec-validator-version }}-${{ matrix.botocore }}
        path: |
          serverlogs/*


SQS Tests

# Run separate test cases to verify SQS works with multiple botocore versions (/multiple response-types, QUERY and JSON)
#
name: "SQS Tests"

on:
  pull_request:
    types: [ labeled ]

jobs:
  test:
    if: ${{ github.event.label.name == 'service-sqs' }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: [ "3.11" ]
        botocore-version: ["1.29.126", "1.29.127", "1.29.128"]

    steps:
    - name: Checkout repository
      uses: actions/checkout@v4

    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}

    - name: Update pip
      run: |
        python -m pip install --upgrade pip

    - name: Install project dependencies
      run: |
        pip install -r requirements-dev.txt
        pip install botocore==${{ matrix.botocore-version }}

    - name: Run tests
      run: |
        pytest -sv tests/test_sqs


Unit test

name: Unit test
on: [workflow_call]

jobs:
  test:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: [3.8, 3.9, "3.10", "3.11", "3.12"]

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Get pip cache dir
      id: pip-cache
      run: |
        echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
    - name: pip cache
      uses: actions/cache@v4
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
    - name: Update pip
      run: |
        python -m pip install --upgrade pip
    - name: Install project dependencies
      run: |
        pip install -r requirements-dev.txt
        pip install pytest-cov
        pip install pytest-github-actions-annotate-failures
    - name: Download Docker dependencies
      # This happens automatically during test execution
      # However, our tests can run concurrently
      # Which means that our tests can pull this image concurrently
      # Pulling it once is more efficient
      run: |
        docker pull mlupin/docker-lambda:python3.11
    - name: Test with pytest
      run: |
        make test-only
    - name: "Upload coverage to Codecov"
      if: ${{ github.repository == 'getmoto/moto'}}
      uses: codecov/codecov-action@v4
      with:
        fail_ci_if_error: false
        flags: unittests
        token: ${{ secrets.CODECOV_TOKEN }}
        verbose: true


Unit tests in Proxy Mode

name: Unit tests in Proxy Mode
on: [workflow_call]

jobs:
  test:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.10", "3.11", "3.12"]

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Get pip cache dir
      id: pip-cache
      run: |
        echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
    - name: pip cache
      uses: actions/cache@v4
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
    - name: Update pip
      run: |
        python -m pip install --upgrade pip
    - name: Install project dependencies
      run: |
        pip install -r requirements-tests.txt
        pip install .[all,server]
    - name: Start MotoProxy
      run: |
        moto_proxy -h > moto_proxy.log
        moto_proxy -H 0.0.0.0 -v > moto_proxy.log &
    - name: Test ProxyMode
      env:
        TEST_PROXY_MODE: ${{ true }}
      run: |
        pytest -sv tests/test_acmpca tests/test_athena tests/test_awslambda tests/test_apigateway tests/test_core/test_proxy.py tests/test_s3
    - name: "Stop MotoProxy"
      if: always()
      run: |
        kill $(lsof -t -i:5005)
    - name: Archive Proxy logs
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: motoproxy-${{ matrix.python-version }}
        path: |
          moto_proxy.log


Execute tests against AWS

name: Execute tests against AWS
on:
  schedule:
    - cron: '00 6 * * 0'
  workflow_dispatch:

jobs:
  test:
    runs-on: ubuntu-latest
    permissions:
      id-token: write

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: "3.11"
    - name: Get pip cache dir
      id: pip-cache
      run: |
        echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
    - name: pip cache
      uses: actions/cache@v4
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: pip-3.11-${{ hashFiles('**/setup.cfg') }}
    - name: Update pip
      run: |
        python -m pip install --upgrade pip
    - name: Install project dependencies
      run: |
        pip install -r requirements-dev.txt
    - name: Configure AWS
      uses: aws-actions/configure-aws-credentials@v4
      with:
        aws-region: us-east-1
        role-to-assume: arn:aws:iam::682283128318:role/GithubActionsRole
    - name: Test with pytest
      env:
        MOTO_TEST_ALLOW_AWS_REQUEST: ${{ true }}
      run: |
        pytest -sv tests/test_cloudformation/ tests/test_dynamodb/ tests/test_ec2/ tests/test_iam/ tests/test_lakeformation/ tests/test_logs/ tests/test_sqs/ tests/test_ses/ tests/test_s3* tests/test_sns/ -m aws_verified


.NET SDK test

name: .NET SDK test
on: [workflow_call]

jobs:
  test:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python 3.8
      uses: actions/setup-python@v5
      with:
        python-version: "3.8"
    - name: Start MotoServer
      run: |
        pip install build
        python -m build
        docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:3.10-slim /moto/scripts/ci_moto_server.sh &
        python scripts/ci_wait_for_server.py
    - uses: actions/setup-dotnet@v4
    - uses: actions/cache@v4
      with:
        path: ~/.nuget/packages
        # Look to see if there is a cache hit for the corresponding requirements file
        key: ${{ runner.os }}-nuget
        restore-keys: |
          ${{ runner.os }}-nuget
    - name: Install dependencies
      run: cd other_langs/tests_dotnet && dotnet restore ExampleTestProject/ && dotnet restore ebs
    - name: Run tests
      run: |
        mkdir ~/.aws && touch ~/.aws/credentials && echo -e "[default]\naws_access_key_id = test\naws_secret_access_key = test" > ~/.aws/credentials
        cd other_langs/tests_dotnet && dotnet test ExampleTestProject/ && dotnet restore ebs


Java SDK test

name: Java SDK test
on: [workflow_call]

jobs:
  test:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python 3.8
      uses: actions/setup-python@v5
      with:
        python-version: "3.10"
    - name: Start MotoServer
      run: |
        pip install build
        python -m build
        docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:3.10-slim /moto/scripts/ci_moto_server.sh &
        python scripts/ci_wait_for_server.py
    - uses: actions/setup-java@v4
      with:
        distribution: 'temurin'
        java-version: '17'
        cache: 'maven'
    - name: Build with Maven
      run: |
        mkdir ~/.aws && touch ~/.aws/credentials && echo -e "[default]\naws_access_key_id = test\naws_secret_access_key = test" > ~/.aws/credentials
        cd other_langs/tests_java && mvn test


Ruby SDK test

name: Ruby SDK test
on: [workflow_call]

jobs:
  test:

    runs-on: ubuntu-latest

    strategy:
      matrix:
        ruby-version: ['3.3']

    steps:
      - uses: actions/checkout@v4
      - name: Set up Ruby ${{ matrix.ruby-version }}
        uses: ruby/setup-ruby@3783f195e29b74ae398d7caca108814bbafde90e
        with:
          ruby-version: ${{ matrix.ruby-version }}
      - name: Set up Python 3.8
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"
      - name: Start MotoServer
        run: |
          pip install build
          python -m build
          docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:3.10-slim /moto/scripts/ci_moto_server.sh &
          python scripts/ci_wait_for_server.py
      - name: Install dependencies
        run: cd other_langs/tests_ruby && bundle install
      - name: Run tests
        env:
          AWS_ENDPOINT_URL: "http://localhost:5000"
        run: |
          mkdir ~/.aws && touch ~/.aws/credentials && echo -e "[default]\naws_access_key_id = test\naws_secret_access_key = test" > ~/.aws/credentials
          cd other_langs/tests_ruby && ruby test/*


Unit tests in Server Mode

name: Unit tests in Server Mode
on: [workflow_call]

jobs:
  test:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: [3.8, 3.9, "3.10", "3.11", "3.12"]

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Start MotoServer
      run: |
        pip install build
        python -m build
        docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e MOTO_EC2_LOAD_DEFAULT_AMIS=false -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:${{ matrix.python-version }}-slim /moto/scripts/ci_moto_server.sh &
        python scripts/ci_wait_for_server.py
    - name: Get pip cache dir
      id: pip-cache
      run: |
        echo "dir=$(pip cache dir)" >> $GITHUB_OUTPUT
    - name: pip cache
      uses: actions/cache@v4
      with:
        path: ${{ steps.pip-cache.outputs.dir }}
        key: pip-${{ matrix.python-version }}-${{ hashFiles('**/setup.cfg') }}
    - name: Update pip
      run: |
        python -m pip install --upgrade pip
    - name: Install project dependencies
      run: |
        pip install -r requirements-dev.txt
    - name: Download Docker dependencies
      # This happens automatically during test execution
      # However, our tests can run concurrently
      # Which means that our tests can pull this image concurrently
      # Pulling it once is more efficient
      run: |
        docker pull mlupin/docker-lambda:python3.11
    - name: Test ServerMode/Coverage
      env:
        TEST_SERVER_MODE: ${{ true }}
      run: |
        make test-only
    - name: "Upload coverage to Codecov"
      if: ${{ github.repository == 'getmoto/moto'}}
      uses: codecov/codecov-action@v4
      with:
        fail_ci_if_error: false
        flags: servertests
        token: ${{ secrets.CODECOV_TOKEN }}
        verbose: true
    - name: "Stop MotoServer"
      if: always()
      run: |
        ls -la
        docker stop motoserver
        mkdir serverlogs
        cp server_output.log serverlogs/server_output.log
    - name: Archive Server logs
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: motoserver-${{ matrix.python-version }}
        path: |
          serverlogs/*


Terraform Examples

# Small, self contained Terraform examples
# Scripts should be placed in:
#    other_langs/terraform/service

name: Terraform Examples
on: [workflow_call]

jobs:
  test:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        service: ["acm", "awslambda", "cloudfront", "elb", "iam", "rds", "route53", "sqs"]

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python 3.8
      uses: actions/setup-python@v5
      with:
        python-version: "3.8"
    - name: Start MotoServer
      run: |
        pip install build
        python -m build
        docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:3.10-slim /moto/scripts/ci_moto_server.sh &
        python scripts/ci_wait_for_server.py
    - name: Run tests
      run: |
        mkdir ~/.aws && touch ~/.aws/credentials && echo -e "[default]\naws_access_key_id = test\naws_secret_access_key = test" > ~/.aws/credentials
        cd other_langs/terraform/${{ matrix.service }}
        terraform init
        terraform apply --auto-approve
        echo "Verify nothing changes when ACM certificates are validated"
        sleep 30
        terraform plan -detailed-exitcode
        sleep 30
        terraform plan -detailed-exitcode
        terraform apply -destroy --auto-approve

  test_create_only:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        service: ["ec2"]

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python 3.8
      uses: actions/setup-python@v5
      with:
        python-version: "3.8"
    - name: Start MotoServer
      run: |
        pip install build
        python -m build
        docker run --rm -t --name motoserver -e TEST_SERVER_MODE=true -e AWS_SECRET_ACCESS_KEY=server_secret -e AWS_ACCESS_KEY_ID=server_key -v `pwd`:/moto -p 5000:5000 -v /var/run/docker.sock:/var/run/docker.sock python:3.10-slim /moto/scripts/ci_moto_server.sh &
        python scripts/ci_wait_for_server.py
    - name: Run tests
      run: |
        mkdir ~/.aws && touch ~/.aws/credentials && echo -e "[default]\naws_access_key_id = test\naws_secret_access_key = test" > ~/.aws/credentials
        cd other_langs/terraform/${{ matrix.service }}
        terraform init
        terraform apply --auto-approve


Unit tests without Docker

# Verify that the `TESTS_SKIP_REQUIRES_DOCKER` environment variable works
# Or in other words:
# Verify that all tests that require Docker have the @requires_docker decorator

name: Unit tests without Docker
on:
  workflow_dispatch:
  schedule:
    - cron: '0 1 * * *'  # every day at 1AM

jobs:
  test:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11"]

    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python-version }}
    - name: Update pip
      run: |
        python -m pip install --upgrade pip
    - name: Install project dependencies
      run: |
        pip install -r requirements-dev.txt
        pip install pytest-github-actions-annotate-failures
    # We should be skipping all tests that require Docker
    # Uninstalling the module enforces that all tests are indeed skipped
    - name: Uninstall Docker
      run: |
        sudo systemctl stop docker
        sudo systemctl stop docker.socket
    - name: Test with pytest
      env:
        TESTS_SKIP_REQUIRES_DOCKER: true
      run: |
        pytest tests


name: Build & Publish h2oGPT Python wheel to PYPI

on:
  workflow_dispatch:
    inputs:
      pypi-index:
        type: choice
        description: PyPI index that needed to be published
        required: true
        default: Test-PyPI
        options:
          - PyPI
          - Test-PyPI
      version:
        description: |
          Override the current version for the python package for dev purposes when uploading to Test-PyPI
        type: string

jobs:
  build_and_upload:
    runs-on: ubuntu-latest
    steps:
        - uses: actions/checkout@v3.5.3

        - uses: actions/setup-python@v4
          with:
            python-version: '3.10'

        - name: Install Dependencies
          run: |
            python3.10 -m pip install --upgrade pip
            python3.10 -m pip install setuptools wheel twine --upgrade

        - name: Modify Version
          if: ${{ inputs.version != ''}}
          run: |
            echo ${{ inputs.version}} > version.txt
            echo "h2ogpt-wheel-version  = $(cat version.txt)"

        - name: Build Wheel
          run: make clean dist

        - name: Publish to Test-PyPI
          if: ${{ inputs.pypi-index == 'Test-PyPI' }}
          run: |
            twine upload -r testpypi dist/*
          env:
            TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
            TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}

        - name: Publish to PyPI
          if: ${{ inputs.pypi-index == 'PyPI' }}
          run: |
            twine upload dist/*
          env:
            TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
            TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}


Snyk Security Vulnerability Scan

name: Snyk Security Vulnerability Scan

on:
  workflow_dispatch:
  pull_request:
  push:
    tags:
      - 'v[0-9]+.[0-9]+.[0-9]+'
    branches:
      - main

jobs:
  snyk_scan_test:
    if: ${{ github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@master
      - uses: snyk/actions/setup@master

      - uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - name: Check changed Deps files
        uses: tj-actions/changed-files@v35
        id: changed-files
        with:
          files: | # This will match all the files with below patterns
            requirements.txt

      - name: Scan python dependencies
        if: contains(steps.changed-files.outputs.all_changed_and_modified_files, 'requirements.txt')
        env:
          SNYK_TOKEN: '${{ secrets.SNYK_TOKEN }}'
        run: |
          head -n 41 requirements.txt > temp-requirements.txt #remove test deps
          python3.10 -m pip install -r temp-requirements.txt
          snyk test \
            -d \
            --file=temp-requirements.txt \
            --package-manager=pip \
            --command=python3.10 \
            --skip-unresolved \
            --severity-threshold=high

  snyk_scan_monitor:
    if: ${{ github.event_name == 'push' || github.event_name == 'workflow_dispatch'}}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@master
      - uses: snyk/actions/setup@master

      - uses: actions/setup-python@v4
        with:
          python-version: '3.10'

      - name: Extract github branch/tag name
        shell: bash
        run: echo "ref=$(echo ${GITHUB_REF##*/})" >> $GITHUB_OUTPUT
        id: extract_ref

      - name: Monitor python dependencies
        env:
          SNYK_TOKEN: '${{ secrets.SNYK_TOKEN }}'
        run: |
          head -n 41 requirements.txt > temp-requirements.txt #remove test deps
          python3.10 -m pip install -r temp-requirements.txt
          snyk monitor \
            -d \
            --file=temp-requirements.txt \
            --command=python3.10 \
            --package-manager=pip \
            --skip-unresolved \
            --remote-repo-url=h2ogpt/${{ steps.extract_ref.outputs.ref }} \
            --org=h2o-gpt \
            --project-name=H2O-GPT/h2ogpt/${{ steps.extract_ref.outputs.ref }}/requirements.txt


Build and test

name: Build and test

on:
  push:
    paths:
      - "mage_ai/**"
      - "mage_integrations/**"
      - "pyproject.toml"
      - "requirements.txt"
      - "setup.py"
  pull_request:
    paths:
      - "mage_ai/**"
      - "mage_integrations/**"
      - "pyproject.toml"
      - "requirements.txt"
      - "setup.py"

jobs:
  check-code-quality:

    runs-on: ubuntu-latest
    name: Check Code Quality
    steps:
      - name: Check out source repository
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      # https://github.com/actions/setup-python/blob/main/docs/advanced-usage.md#caching-packages
      - name: Install Poetry
        run: pipx install poetry
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.10"
          cache: 'poetry'
      - name: Install Python dev dependencies
        run: |
          poetry install --no-interaction --only dev
          poetry run pre-commit install
      - id: changed-files
        uses: tj-actions/changed-files@v36
        with:
          files_ignore: |
            requirements.txt
      - name: Run pre-commit hooks
        run: poetry run pre-commit run --show-diff-on-failure --files ${{ steps.changed-files.outputs.all_changed_files }}

  test_backend:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10"]
    needs: check-code-quality
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          sudo apt-get install freetds-dev
          python -m pip install --upgrade pip
          pip install -r requirements.txt
          pip install mage_integrations/
          pip install "git+https://github.com/mage-ai/singer-python.git#egg=singer-python"
          pip install "git+https://github.com/mage-ai/sqlglot#egg=sqlglot"
      - name: Run unit tests
        run: |
          python3 -m unittest discover -s mage_ai --failfast
        env:
          OPENAI_API_KEY: abcdefg
      - name: Run mage integrations unit tests
        run: |
          cd mage_integrations && python3 -m unittest discover mage_integrations.tests

      # Run Playwright web UI tests only run once per workflow (Python 3.10).
      - name: Set up Node
        uses: actions/setup-node@v3
        with:
          node-version: "18.18.0"
        if: matrix.python-version == '3.10'
      - name: Create Mage test project
        run: |
          python mage_ai/cli/main.py init test_project
        env:
          PYTHONPATH: ${{ env.pythonLocation }}:.
        if: matrix.python-version == '3.10'
      - name: Build frontend, start server, and run Playwright tests
        run: |
          yarn install &&
          yarn export_prod &&
          yarn playwright install chromium &&
          yarn playwright test -c playwright.config.ci.ts
        working-directory: mage_ai/frontend/
        env:
          PYTHONPATH: ${{ env.pythonLocation }}:. # Playwright tests on CI depend on the Python web server
        if: matrix.python-version == '3.10'
      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: playwright-report
          path: mage_ai/frontend/playwright-report/
          retention-days: 3

  test_web_server_windows:
    runs-on: windows-latest
    strategy:
      matrix:
        python-version: ["3.10"]
    needs: check-code-quality
    steps:
      - uses: actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v3
        with:
          python-version: ${{ matrix.python-version }}
      # Run Playwright web UI tests only run once per workflow (Python 3.10).
      - name: Set up Node
        uses: actions/setup-node@v3
        with:
          node-version: "18.18.0"
      - name: Create Mage test project
        run: |
          Start-Process -Wait -FilePath "C:\Program Files\Git\unins000.exe" -ArgumentList "/SILENT"
          python -m venv venv3
          venv3/Scripts/Activate.ps1
          $env:PYTHONPATH = "${env:PYTHONPATH};${pwd}"
          python -m pip install -r requirements.txt
          python mage_ai/cli/main.py init test_project
      - name: Build frontend, start server, and run Playwright tests
        run: |
          yarn install &&
          yarn export_prod &&
          yarn playwright install chromium &&
          yarn playwright test -c playwright-windows.config.ci.ts
        working-directory: mage_ai/frontend/
      - uses: actions/upload-artifact@v3
        if: always()
        with:
          name: playwright-report
          path: mage_ai/frontend/playwright-report/
          retention-days: 3


CodeQL

# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"

on:
  push:
    branches: [ "master" ]
  pull_request:
    # The branches below must be a subset of the branches above
    branches: [ "master" ]
  schedule:
    - cron: '24 16 * * 2'

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write

    strategy:
      fail-fast: false
      matrix:
        language: [ 'javascript', 'python' ]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support

    steps:
    - name: Checkout repository
      uses: actions/checkout@v3

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v2
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
        # By default, queries listed here will override any specified in a config file.
        # Prefix the list here with "+" to use these queries and those in the config file.

        # Details on CodeQL's query packs refer to : https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
        # queries: security-extended,security-and-quality

    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
      uses: github/codeql-action/autobuild@v2

    # ℹ Command-line programs to run using the OS shell.
    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun

    #   If the Autobuild fails above, remove it and uncomment the following three lines.
    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.

    # - run: |
    #   echo "Run, Build Application using script"
    #   ./location_of_script_within_repo/buildscript.sh

    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v2
      with:
        category: "/language:${{matrix.language}}"


Lint Docker Images

name: Lint Docker Images

on:
  pull_request:
    paths:
      - '*Dockerfile'

jobs:
  changed-images:
    runs-on: ubuntu-latest
    name: List changed images
    steps:
      - name: Check out source repository
        uses: actions/checkout@v3
        with:
          fetch-depth: 0
      - name: Get changed files
        id: changed-files
        uses: tj-actions/changed-files@v38
        with:
          files: '*Dockerfile'
          json: true
      - name: Create Job Matrix
        id: image-matrix
        run: echo images=${{ steps.changed-files.outputs.all_changed_files }} >> $GITHUB_OUTPUT
    outputs:
      images: ${{ steps.image-matrix.outputs.images }}

  hadolint:
    name: Lint Docker Images
    runs-on: ubuntu-latest
    permissions:
      pull-requests: write
    # Skip job if there are no images to test
    # https://github.com/dorny/paths-filter/issues/66
    if: ${{ needs.changed-images.outputs.images != '' && toJson(fromJson(needs.changed-images.outputs.images)) != '[]' }}
    needs: [changed-images]
    strategy:
      matrix:
        image: ${{ fromJSON(needs.changed-images.outputs.images) }}
      fail-fast: false
    steps:
      - name: Checkout
        uses: actions/checkout@v3
      - uses: hadolint/hadolint-action@v3.1.0
        id: hadolint
        with:
          dockerfile: ${{ matrix.image }}
          ignore: DL3002,DL3008,DL3013,DL3016
      - name: Annotate Pull Request
        uses: actions/github-script@v6
        if: github.event_name == 'pull_request' && failure()
        with:
          script: |
            const output = `
            #### Hadolint: \`${{ steps.hadolint.outcome }}\`
            \`\`\`
            ${process.env.HADOLINT_RESULTS}
            \`\`\`
            `;

            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: output
            })


Build and publish Docker image

name: Build and publish Docker image

on:
  workflow_dispatch:
    inputs:
      dockerfile_branch:
        description: 'Dockerfile branch'
        required: true
        default: master
      feature_branch:
        description: 'Feature branch'
        required: false
        default: null
      name:
        description: 'Docker image name'
        required: true
        default: mageai
      tag:
        description: 'Docker image tag'
        required: true
        default: latest

jobs:
  push_to_registry:
    name: Push Docker image to Docker Hub
    runs-on: ubuntu-latest
    steps:
      - name: Check out the repo
        uses: actions/checkout@v3
        with:
          ref: ${{ github.event.inputs.dockerfile_branch }}

      - name: Log in to Docker Hub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_PASSWORD }}

      - name: Extract metadata (tags, labels) for Docker
        id: meta
        uses: docker/metadata-action@v4
        with:
          images: mageai/${{ github.event.inputs.name }}:${{ github.event.inputs.tag }}

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Build and push Docker image
        uses: docker/build-push-action@v3
        with:
          context: .
          platforms: linux/amd64,linux/arm64
          push: true
          tags: mageai/${{ github.event.inputs.name }}:${{ github.event.inputs.tag }}
          labels: ${{ steps.meta.outputs.labels }}
          build-args: |
            FEATURE_BRANCH=${{ github.event.inputs.feature_branch }}


Build and publish to PyPI

name: Build and publish to PyPI

on:
  workflow_dispatch:
    inputs:
      dry_run:
        description: 'Dry Run'
        required: true
        default: true
        type: boolean

jobs:
  build_and_publish:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@master
    - name: Set up Python 3.8
      uses: actions/setup-python@v1
      with:
        python-version: 3.8
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install --user --upgrade setuptools wheel requests
    - name: Build stable distribution
      run: |
        rm -rf dist/*
        python setup.py sdist bdist_wheel
    - name: Publish distribution to Test PyPI
      if: github.event.inputs.dry_run == 'true'
      uses: pypa/gh-action-pypi-publish@release/v1.8
      with:
        password: ${{ secrets.TEST_PYPI_API_TOKEN }}
        repository_url: https://test.pypi.org/legacy/
        verbose: true
        skip-existing: true
    - name: Publish distribution to PyPI
      if: github.event.inputs.dry_run == 'false'
      uses: pypa/gh-action-pypi-publish@release/v1.8
      with:
        password: ${{ secrets.PYPI_API_TOKEN }}
        verbose: true
        skip-existing: true


clang-format

name: clang-format

on:
  # Trigger the workflow on push or pull request,
  # but only for the main branch
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

jobs:
  clang-format:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.11"]
    steps:
    - uses: actions/checkout@v2
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install clang-format==18.1.5
    - name: Running clang-format
      run: |
        EXCLUDES=(
            'csrc/moe/topk_softmax_kernels.cu'
            'csrc/punica/bgmv/bgmv_bf16_bf16_bf16.cu'
            'csrc/punica/bgmv/bgmv_config.h'
            'csrc/punica/bgmv/bgmv_impl.cuh'
            'csrc/punica/bgmv/vec_dtypes.cuh'
            'csrc/punica/punica_ops.cu'
            'csrc/punica/type_convert.h'
        )
        find csrc/ \( -name '*.h' -o -name '*.cpp' -o -name '*.cu' -o -name '*.cuh' \) -print \
            | grep -vFf <(printf "%s\n" "${EXCLUDES[@]}") \
            | xargs clang-format --dry-run --Werror


mypy

name: mypy

on:
  # Trigger the workflow on push or pull request,
  # but only for the main branch
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

jobs:
  ruff:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11"]
    steps:
    - uses: actions/checkout@v2
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install mypy==1.9.0
        pip install types-setuptools
        pip install types-PyYAML
        pip install types-requests
        pip install types-setuptools
    - name: Mypy
      run: |
        mypy vllm/attention --config-file pyproject.toml
        mypy vllm/core --config-file pyproject.toml
        mypy vllm/distributed --config-file pyproject.toml
        mypy vllm/entrypoints --config-file pyproject.toml
        mypy vllm/executor --config-file pyproject.toml
        mypy vllm/multimodal --config-file pyproject.toml
        mypy vllm/usage --config-file pyproject.toml
        mypy vllm/*.py --config-file pyproject.toml
        mypy vllm/transformers_utils --config-file pyproject.toml
        mypy vllm/engine  --config-file pyproject.toml
        mypy vllm/worker --config-file pyproject.toml
        mypy vllm/spec_decode --config-file pyproject.toml
        mypy vllm/model_executor  --config-file pyproject.toml
        mypy vllm/lora --config-file pyproject.toml
        mypy vllm/logging --config-file pyproject.toml
        mypy tests --config-file pyproject.toml


Create Release

# This workflow will upload a Python Package to Release asset
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions

name: Create Release

on:
  push:
    tags:
      - v*

# Needed to create release and upload assets
permissions:
  contents: write

jobs:
  release:
    # Retrieve tag and create release
    name: Create Release
    runs-on: ubuntu-latest
    outputs:
      upload_url: ${{ steps.create_release.outputs.upload_url }}
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Extract branch info
        shell: bash
        run: |
          echo "release_tag=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV

      - name: Create Release
        id: create_release
        uses: "actions/github-script@v6"
        env:
          RELEASE_TAG: ${{ env.release_tag }}
        with:
          github-token: "${{ secrets.GITHUB_TOKEN }}"
          script: |
            const script = require('.github/workflows/scripts/create_release.js')
            await script(github, context, core)

  wheel:
    name: Build Wheel
    runs-on: ${{ matrix.os }}
    needs: release

    strategy:
      fail-fast: false
      matrix:
          os: ['ubuntu-20.04']
          python-version: ['3.8', '3.9', '3.10', '3.11']
          pytorch-version: ['2.3.0']  # Must be the most recent version that meets requirements-cuda.txt.
          cuda-version: ['11.8', '12.1']

    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Setup ccache
        uses: hendrikmuhs/ccache-action@v1.2
        with:
          create-symlink: true
          key: ${{ github.job }}-${{ matrix.python-version }}-${{ matrix.cuda-version }}

      - name: Set up Linux Env
        if: ${{ runner.os == 'Linux' }}
        run: |
          bash -x .github/workflows/scripts/env.sh

      - name: Set up Python
        uses: actions/setup-python@v4
        with:
            python-version: ${{ matrix.python-version }}

      - name: Install CUDA ${{ matrix.cuda-version }}
        run: |
          bash -x .github/workflows/scripts/cuda-install.sh ${{ matrix.cuda-version }} ${{ matrix.os }}

      - name: Install PyTorch ${{ matrix.pytorch-version }} with CUDA ${{ matrix.cuda-version }}
        run: |
          bash -x .github/workflows/scripts/pytorch-install.sh ${{ matrix.python-version }} ${{ matrix.pytorch-version }} ${{ matrix.cuda-version }}

      - name: Build wheel
        shell: bash
        env:
          CMAKE_BUILD_TYPE: Release # do not compile with debug symbol to reduce wheel size
        run: |
          bash -x .github/workflows/scripts/build.sh ${{ matrix.python-version }} ${{ matrix.cuda-version }}
          wheel_name=$(ls dist/*whl | xargs -n 1 basename)
          asset_name=${wheel_name//"linux"/"manylinux1"}
          echo "wheel_name=${wheel_name}" >> $GITHUB_ENV
          echo "asset_name=${asset_name}" >> $GITHUB_ENV

      - name: Upload Release Asset
        uses: actions/upload-release-asset@v1
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        with:
          upload_url: ${{ needs.release.outputs.upload_url }}
          asset_path: ./dist/${{ env.wheel_name }}
          asset_name: ${{ env.asset_name }}
          asset_content_type: application/*

      # (Danielkinz): This last step will publish the .whl to pypi. Warning: untested
      # - name: Publish package
      #   uses: pypa/gh-action-pypi-publish@release/v1.8
      #   with:
      #     repository-url: https://test.pypi.org/legacy/
      #     password: ${{ secrets.PYPI_API_TOKEN }}
      #     skip-existing: true


ruff

name: ruff

on:
  # Trigger the workflow on push or pull request,
  # but only for the main branch
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

jobs:
  ruff:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11"]
    steps:
    - uses: actions/checkout@v2
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install ruff==0.1.5 codespell==2.3.0 tomli==2.0.1 isort==5.13.2
    - name: Analysing the code with ruff
      run: |
        ruff .
    - name: Spelling check with codespell
      run: |
        codespell --toml pyproject.toml
    - name: Run isort
      run: |
        isort . --check-only


yapf

name: yapf

on:
  # Trigger the workflow on push or pull request,
  # but only for the main branch
  push:
    branches:
      - main
  pull_request:
    branches:
      - main
jobs:
  yapf:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10", "3.11"]
    steps:
    - uses: actions/checkout@v2
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v2
      with:
        python-version: ${{ matrix.python-version }}
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install yapf==0.32.0
        pip install toml==0.10.2
    - name: Running yapf
      run: |
        yapf --diff --recursive .


Build and Push Backend Image on Tag

name: Build and Push Backend Image on Tag

on:
  push:
    tags:
      - '*'

jobs:
  build-and-push:
    runs-on: ubuntu-latest

    steps:
    - name: Checkout code
      uses: actions/checkout@v2

    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v3

    - name: Login to Docker Hub
      uses: docker/login-action@v3
      with:
        username: ${{ secrets.DOCKER_USERNAME }}
        password: ${{ secrets.DOCKER_TOKEN }}

    - name: Backend Image Docker Build and Push
      uses: docker/build-push-action@v5
      with:
        context: ./backend
        file: ./backend/Dockerfile
        platforms: linux/amd64,linux/arm64
        push: true
        tags: |
          danswer/danswer-backend:${{ github.ref_name }}
          danswer/danswer-backend:latest
        build-args: |
          DANSWER_VERSION=${{ github.ref_name }}

    - name: Run Trivy vulnerability scanner
      uses: aquasecurity/trivy-action@master
      with:
        # To run locally: trivy image --severity HIGH,CRITICAL danswer/danswer-backend
        image-ref: docker.io/danswer/danswer-backend:${{ github.ref_name }}
        severity: 'CRITICAL,HIGH'
        trivyignores: ./backend/.trivyignore


name: Build and Push Model Server Image on Tag

on:
  push:
    tags:
      - '*'

jobs:
  build-and-push:
    runs-on: ubuntu-latest

    steps:
    - name: Checkout code
      uses: actions/checkout@v2

    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v3

    - name: Login to Docker Hub
      uses: docker/login-action@v3
      with:
        username: ${{ secrets.DOCKER_USERNAME }}
        password: ${{ secrets.DOCKER_TOKEN }}

    - name: Model Server Image Docker Build and Push
      uses: docker/build-push-action@v5
      with:
        context: ./backend
        file: ./backend/Dockerfile.model_server
        platforms: linux/amd64,linux/arm64
        push: true
        tags: |
          danswer/danswer-model-server:${{ github.ref_name }}
          danswer/danswer-model-server:latest
        build-args: |
          DANSWER_VERSION=${{ github.ref_name }}

    - name: Run Trivy vulnerability scanner
      uses: aquasecurity/trivy-action@master
      with:
        image-ref: docker.io/danswer/danswer-model-server:${{ github.ref_name }}
        severity: 'CRITICAL,HIGH'