CI Skill Manager

name: "CI Skill Manager"

on:
  push:
    branches: [ master ]
    paths:
      - "skill-manager/**"
      - ".github/**"
  pull_request:
    branches: [ master ]
    paths:
      - "skill-manager/**"
      - ".github/**"
  workflow_dispatch:

jobs:
  skill-manager:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@v2

      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v1
        with:
          install: true

      - name: Cache Docker layers
        uses: actions/cache@v2
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-skill-manager-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-buildx-skill-manager-
            ${{ runner.os }}-buildx-

      - name: Build test image
        uses: docker/build-push-action@v2
        with:
          builder: ${{ steps.buildx.outputs.name }}
          context: ./skill-manager
          target: test
          load: True
          push: False
          tags: skill-manager:${{ github.sha }}
          cache-from: type=local,src=/tmp/.buildx-cache
          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new

      - name: Run test image
        uses: addnab/docker-run-action@v3
        with:
          image: skill-manager:${{ github.sha }}
          options: -v /var/run/docker.sock:/var/run/docker.sock -v ${{ github.workspace }}/skill-manager/test-reports:/app/test-reports
          run: |
            python -m pytest --junitxml=test-reports/junit.xml --cov --cov-report=xml:test-reports/coverage.xml --cov-report=html:test-reports/coverage.html

      - uses: actions/upload-artifact@v2
        with:
          name: skill-manager-test-reports
          path: ${{ github.workspace }}/skill-manager/test-reports

      - name: Move cache
        run: |
          rm -rf /tmp/.buildx-cache
          mv /tmp/.buildx-cache-new /tmp/.buildx-cache


CI Model Manager

name: "CI Model Manager"

on:
  push:
    branches: [ master ]
    paths:
      - "model-manager/**"
      - ".github/**"
  pull_request:
    branches: [ master ]
    paths:
      - "model-manager/**"
      - ".github/**"
  workflow_dispatch:

jobs:
  model-management:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v3

      - name: Prepare
        id: prep
        run: |
          TAG=$(echo $GITHUB_SHA | head -c7)
          IMAGE="ukpsquare/model-manager"
          echo "image=${IMAGE}" >> $GITHUB_OUTPUT
          echo "tag=${TAG}" >> $GITHUB_OUTPUT

      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          install: true

      - name: Cache Docker layers
        uses: actions/cache@v2
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-model_api-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-buildx-model_api-
            ${{ runner.os }}-buildx-

      - name: Build test image
        uses: docker/build-push-action@v3
        with:
          builder: ${{ steps.buildx.outputs.name }}
          context: ./model-manager
          target: test
          load: true
          tags: ${{ steps.prep.outputs.image }}:${{ steps.prep.outputs.tag }}-test
          cache-from: type=local,src=/tmp/.buildx-cache
          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new

        # Temp fix
        # https://github.com/docker/build-push-action/issues/252
        # https://github.com/moby/buildkit/issues/1896
      - name: Move cache
        run: |
          rm -rf /tmp/.buildx-cache
          mv /tmp/.buildx-cache-new /tmp/.buildx-cache

      - name: Retrieve Test Reports
        id: extract
        uses: shrink/actions-docker-extract@v1
        with:
          image: ${{ steps.prep.outputs.image }}:${{ steps.prep.outputs.tag }}-test
          path: /app/test-reports

      - uses: actions/upload-artifact@v3
        with:
          name: model_api-test-reports
          path: ${{ steps.extract.outputs.destination }}/test-reports

      - name: Publish Test Report
        uses: mikepenz/action-junit-report@v3
        with:
          report_paths: ${{ steps.extract.outputs.destination }}/test-reports/junit.xml
          check_name: Model API Test Report
          fail_on_failure: true

      - name: Login to Docker Hub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}

      - name: Build deployable image
        uses: docker/build-push-action@v3
        with:
          builder: ${{ steps.buildx.outputs.name }}
          context: ./model-manager
          target: build
          push: ${{github.ref == 'refs/heads/master'}}
          tags: ${{ steps.prep.outputs.image }}:${{ steps.prep.outputs.tag }}, ${{ steps.prep.outputs.image }}:latest
          cache-from: type=local,src=/tmp/.buildx-cache
          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new

      - name: Move cache
        run: |
          rm -rf /tmp/.buildx-cache
          mv /tmp/.buildx-cache-new /tmp/.buildx-cache


CI Model

name: "CI Model"

on:
  push:
    branches: [ master, model-api-v3 ]
    paths:
      - "model-inference/**"
      - ".github/**"
  pull_request:
    branches: [ master ]
    paths:
      - "model-inference/**"
      - ".github/**"
  workflow_dispatch:

jobs:

  test:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        model_type: [
          "transformer",
          "graph_transformer",
          "sentence_transformer",
           "onnx",
        ]
        include:
          - model_type: transformer
            test_dir: transformer
            test_setup: transformer
            image_suffix: transformer
          - model_type: graph_transformer
            test_dir: transformer
            test_setup: transformer
            image_suffix: graph-transformer
          - model_type: sentence_transformer
            test_dir: sentence_transformer
            test_setup: sentence_transformer
            image_suffix: sentence-transformer
          - model_type: onnx
            test_dir: onnx
            test_setup: onnx
            image_suffix: onnx

    steps:
      - uses: actions/checkout@v3

      - name: Delete huge unnecessary tools folder
        run: rm -rf /opt/hostedtoolcache

      - name: Prepare
        id: transformer-prep
        run: |
          TAG=$(echo $GITHUB_SHA | head -c7)
          echo $TAG
          echo "tag=${TAG}" >> $GITHUB_OUTPUT

          IMAGE="ukpsquare/model-inference-${{ matrix.image_suffix }}"
          echo $IMAGE
          echo "image=${IMAGE}" >> $GITHUB_OUTPUT
      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          install: true

      - name: Build test image
        uses: docker/build-push-action@v3
        with:
          builder: ${{ steps.buildx.outputs.name }}
          context: ./model-inference
          file: ./model-inference/Dockerfile
          target: test
          build-args: |
            MODEL_TYPE=${{ matrix.model_type }}
            TEST_DIR=${{ matrix.test_dir }}
            TEST_SETUP=${{ matrix.test_setup }}
          load: true
          push: false
          tags: ${{ steps.transformer-prep.outputs.image }}:ci-${{ steps.transformer-prep.outputs.tag }}

      - name: Retrieve Test Reports
        id: transformer-extract
        uses: shrink/actions-docker-extract@v1
        with:
          image: ${{ steps.transformer-prep.outputs.image }}:ci-${{ steps.transformer-prep.outputs.tag }}
          path: /app/test-reports

      - name: Upload Test Reports
        uses: actions/upload-artifact@v3
        with:
          name: model_api-test-reports
          path: ${{ steps.transformer-extract.outputs.destination }}/test-reports

      - name: Publish Test Report
        uses: mikepenz/action-junit-report@v3
        with:
          report_paths: ${{ steps.transformer-extract.outputs.destination }}/test-reports/junit.xml
          check_name: Transformers Test Report
          fail_on_failure: true


CI Datastore

name: "CI Datastore"

on:
  push:
    branches: [ master ]
    paths:
      - "datastore-api/**"
      - ".github/**"
  pull_request:
    branches: [ master ]
    paths:
      - "datastore-api/**"
      - ".github/**"
  workflow_dispatch:

jobs:
  datastore-api:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@v2

      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v1
        with:
          install: true

      - name: Cache Docker layers
        uses: actions/cache@v2
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-datastore-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-buildx-datastore-
            ${{ runner.os }}-buildx-

      - name: Build test image
        uses: docker/build-push-action@v2
        with:
          builder: ${{ steps.buildx.outputs.name }}
          context: ./datastore-api
          target: test
          load: true
          tags: datastore:${{ github.sha }}
          cache-from: type=local,src=/tmp/.buildx-cache
          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new

      - name: Run test image
        uses: addnab/docker-run-action@v3
        with:
          image: datastore:${{ github.sha }}
          options: -v /var/run/docker.sock:/var/run/docker.sock -v ${{ github.workspace }}/datastore-api/test-reports:/app/test-reports
          run: |
            PYTHONPATH="." MOCK_DEPENDENCIES=1 pytest --junitxml=test-reports/junit.xml --cov --cov-report=xml:test-reports/coverage.xml --cov-report=html:test-reports/coverage.html ./tests

      - uses: actions/upload-artifact@v2
        with:
          name: datastore_api-test-reports
          path: ${{ steps.extract.outputs.destination }}/test-reports

      - name: Publish Test Report
        uses: mikepenz/action-junit-report@v2
        with:
          # report_paths: ${{ steps.extract.outputs.destination }}/test-reports/junit.xml
          report_paths: ${{ github.workspace }}/datastore-api/test-reports/junit.xml
          check_name: Datastore API Test Report
          fail_on_failure: true

      #  Temp fix
      # https://github.com/docker/build-push-action/issues/252
      # https://github.com/moby/buildkit/issues/1896
      - name: Move cache
        run: |
          rm -rf /tmp/.buildx-cache
          mv /tmp/.buildx-cache-new /tmp/.buildx-cache


Build and Deploy

name: Build and Deploy

on:
  push:
    branches: [ master, cicd ]
  workflow_dispatch:
    inputs:
      environment:
        description: 'Environment to deploy to'
        required: true
        default: 'test'
        type: choice
        options:
          - 'test'
          - 'production'
      branch:
        description: 'Branch to deploy'
        required: true
        default: 'master'
jobs:
  build:
    runs-on: ubuntu-latest
    outputs:
      tag: ${{ steps.image_tag.outputs.tag }}
    strategy:
      matrix:
        service: [
          "datastore",
          "docs",
          "frontend",
          "model-manager",
          "model-inference-transformer",
          "model-inference-onnx",
          "model-inference-sentence-transformer",
          "model-inference-graph-transformer",
          "skill-manager",
          "skill-boolq",
          "skill-commonsense-qa",
          "skill-extractive-qa",
          "skill-generative-qa",
          "skill-information-retrieval",
          "skill-multiple-choice-qa",
          "skill-open-extractive-qa",
          "skill-qa-gnn",
          "skill-tweac",
          "skill-metaqa",
          "skill-extractive-metaqa",
          "skill-multiple-choice-metaqa",
          "skill-kgqa-kqapro",
          "evaluator",
          "sensitivity",
          "replicate",
        ]
        include:
          - build-args: ""
          - service: datastore
            context: ./datastore-api
          - service: docs
            context: ./docs
          - service: frontend
            context: ./frontend
            build-args: "mode=${{ github.event.inputs.environment || 'test' }}"
          - service: model-manager
            context: ./model-manager
          - service: model-inference-transformer
            context: ./model-inference
            build-args: "MODEL_TYPE=transformer"
          - service: model-inference-onnx
            context: ./model-inference
            build-args: "MODEL_TYPE=onnx"
          - service: model-inference-sentence-transformer
            context: ./model-inference
            build-args: "MODEL_TYPE=sentence_transformer"
          - service: model-inference-graph-transformer
            context: ./model-inference
            build-args: "MODEL_TYPE=graph_transformer"
          - service: skill-manager
            context: ./skill-manager
          - service: skill-boolq
            context: ./skills
            build-args: "skill=boolq"
          - service: skill-commonsense-qa
            context: ./skills
            build-args: "skill=commonsense-qa"
          - service: skill-extractive-qa
            context: ./skills
            build-args: "skill=extractive-qa"
          - service: skill-generative-qa
            context: ./skills
            build-args: "skill=generative-qa"
          - service: skill-information-retrieval
            context: ./skills
            build-args: "skill=information-retrieval"
          - service: skill-multiple-choice-qa
            context: ./skills
            build-args: "skill=multiple-choice-qa"
          - service: skill-open-extractive-qa
            context: ./skills
            build-args: "skill=open-extractive-qa"
          - service: skill-qa-gnn
            context: ./skills
            build-args: "skill=qa-gnn"
          - service: skill-tweac
            context: ./skills
            build-args: "skill=tweac"
          - service: skill-extractive-metaqa
            context: ./skills
            build-args: "skill=extractive-metaqa"
          - service: skill-multiple-choice-metaqa
            context: ./skills
            build-args: "skill=multiple-choice-metaqa"
          - service: skill-metaqa
            context: ./skills
            build-args: "skill=metaqa"
          - service: "skill-kgqa-kqapro"
            context: ./skills
            build-args: "skill=kgqa-kqapro"
          - service: evaluator
            context: ./evaluator
          - service: sensitivity
            context: ./sensitivity
          - service: replicate
            context: ./replicate
    steps:
      - name: Checkout
        uses: actions/checkout@v3

      - name: Login to Docker Hub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKER_HUB_USERNAME }}
          password: ${{ secrets.DOCKER_HUB_ACCESS_TOKEN }}

      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v2
        with:
          install: true

      - name: Cache Docker layers
        uses: actions/cache@v3
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-${{ matrix.service }}-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-buildx-${{ matrix.service }}-
            ${{ runner.os }}-buildx-

      - name: Set Image Tag
        id: image_tag
        run: |
          SHORT_SHA=$(git rev-parse --short ${{ github.sha }})
          echo $SHORT_SHA
          TAG="${{ github.event.inputs.environment || 'build' }}-$SHORT_SHA"
          echo $TAG
          echo "tag=$TAG" >> $GITHUB_OUTPUT

      - name: Add latest tag if deploying to prod
        uses: haya14busa/action-cond@v1
        id: latest_tag_str
        with:
          cond: ${{ github.event.inputs.environment == 'production' }}
          if_true: "ukpsquare/${{matrix.service}}:latest"
          if_false: ""

      - name: Build deployable image
        uses: docker/build-push-action@v3
        with:
          builder: ${{ steps.buildx.outputs.name }}
          context: ${{ matrix.context }}
          file: ${{ matrix.file }}
          target: build
          build-args: ${{ matrix.build-args }}
          push: ${{ github.event.inputs.environment == 'production' || github.event.inputs.environment == 'test' }}
          tags: |
            ukpsquare/${{ matrix.service }}:${{ steps.image_tag.outputs.tag }}
            ${{ steps.latest_tag_str.outputs.value }}
          cache-from: type=local,src=/tmp/.buildx-cache
          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new

      #  Temp fix
      # https://github.com/docker/build-push-action/issues/252
      # https://github.com/moby/buildkit/issues/1896
      - name: Move cache
        run: |
          rm -rf /tmp/.buildx-cache
          mv /tmp/.buildx-cache-new /tmp/.buildx-cache
  deploy:
    needs: build
    if: github.event_name == 'workflow_dispatch'
    runs-on: ubuntu-latest
    environment: ${{ github.event.inputs.environment }}
    steps:
      - name: Trigger Deployment on ${{ github.event.inputs.environment }}
        uses: distributhor/workflow-webhook@v2
        env:
          webhook_url: ${{ secrets.WEBHOOK_URL }}
          webhook_secret: ${{ secrets.WEBHOOK_SECRET }}
          data: '{"environment": "${{ github.event.inputs.environment }}", "branch": "${{ github.event.inputs.branch }}", "tag": "${{ needs.build.outputs.tag }}"}'
          verify_ssl: false


CI Evaluator

name: "CI Evaluator"

on:
  push:
    branches: [ master ]
    paths:
      - "evaluator/**"
      - ".github/**"
  pull_request:
    branches: [ master ]
    paths:
      - "evaluator/**"
      - ".github/**"
  workflow_dispatch:

jobs:
  evaluator:
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@v2

      - name: Set up Docker Buildx
        id: buildx
        uses: docker/setup-buildx-action@v1
        with:
          install: true

      - name: Cache Docker layers
        uses: actions/cache@v2
        with:
          path: /tmp/.buildx-cache
          key: ${{ runner.os }}-buildx-evaluator-${{ github.sha }}
          restore-keys: |
            ${{ runner.os }}-buildx-evaluator-
            ${{ runner.os }}-buildx-

      - name: Build test image
        uses: docker/build-push-action@v2
        with:
          builder: ${{ steps.buildx.outputs.name }}
          context: ./evaluator
          target: test
          load: True
          push: False
          tags: evaluator:${{ github.sha }}
          cache-from: type=local,src=/tmp/.buildx-cache
          cache-to: type=local,mode=max,dest=/tmp/.buildx-cache-new

      - name: Run test image
        uses: addnab/docker-run-action@v3
        with:
          image: evaluator:${{ github.sha }}
          options: -v /var/run/docker.sock:/var/run/docker.sock -v ${{ github.workspace }}/evaluator/test-reports:/app/test-reports
          run: |
            python -m pytest --junitxml=test-reports/junit.xml --cov --cov-report=xml:test-reports/coverage.xml --cov-report=html:test-reports/coverage.html

      - uses: actions/upload-artifact@v2
        with:
          name: evaluator-test-reports
          path: ${{ github.workspace }}/evaluator/test-reports

      - name: Move cache
        run: |
          rm -rf /tmp/.buildx-cache
          mv /tmp/.buildx-cache-new /tmp/.buildx-cache


Element

name: Element

on: [deployment, workflow_dispatch]

jobs:
  send-message:
    runs-on: ubuntu-latest
    name: Send message via Matrix
    steps:
      - name: Echo event
        run: |
          echo ${{ github.event_name }}
          echo ${{ github.event.action }}

      - name: Set Deployment Message (workflow_dispatch)
        if: github.event_name == 'workflow_dispatch'
        run: |
          echo "MSG=This is a test message from github." >> $GITHUB_ENV

      - name: Set Deployment Message (deployment)
        if: github.event_name == 'deployment'
        run: |
          BRANCH=${{ github.event.deployment.ref }}
          SHA=${{ github.event.deployment.sha }}
          USER=${{ github.event.deployment.creator.login }}
          ENVIRONMENT=${{ github.event.deployment.environment }}
          TIMESTAMP=${{ github.event.deployment.created_at }}
          TIMESTAMP=$(date -d "$TIMESTAMP" "+%Y-%m-%d %H:%M:%S")
          MSG="Deployment of branch [$BRANCH](https://github.com/ukp-square/square-core/tree/$BRANCH) with commit [$SHA](https://github.com/UKP-SQuARE/square-core/commit/$SHA) to **$ENVIRONMENT** by [$USER](https://github.com/$USER) at $TIMESTAMP."
          echo "MSG=$MSG" >>$GITHUB_ENV

      - name: Send message to square-github channel
        id: matrix-chat-message
        uses: fadenb/matrix-chat-message@v0.0.6
        with:
          homeserver: ${{ secrets.ELEMENT_HOME_SERVER }}
          token: ${{ secrets.ELEMENT_ACCESS_TOKEN }}
          channel: ${{ secrets.ELEMENT_ROOM_ID }}
          message: ${{ env.MSG }}


Benchmark

name: Benchmark

on:
  push:
    branches: [main, "[0-9]+.[0-9]+.x"]
  pull_request:
    branches: [main]

jobs:
  benchmark:
    runs-on: ${{ matrix.os }}
    defaults:
      run:
        shell: bash -el {0} # -e to fail on error, -l for mamba

    strategy:
      fail-fast: false
      matrix:
        python: ["3.12"]
        os: [ubuntu-latest]

    env:
      OS: ${{ matrix.os }}
      PYTHON: ${{ matrix.python }}
      ASV_DIR: "./benchmarks"

    steps:
      - uses: actions/checkout@v2
        with:
          fetch-depth: 0

      - run: git fetch origin main:main
        if: ${{ github.ref_name != 'main' }}
        # Errors on main branch

      - uses: mamba-org/setup-micromamba@v1
        with:
          environment-name: asv
          cache-environment: true
          create-args: >-
            python=3.11
            asv
            mamba
            packaging

      - name: Cache datasets
        uses: actions/cache@v3
        with:
          path: |
            ~/.cache
          key: benchmark-state-${{ hashFiles('benchmarks/**') }}

      - name: Quick benchmark run
        working-directory: ${{ env.ASV_DIR }}
        run: |
          asv machine --yes
          asv run --quick --show-stderr --verbose


Pull Request Validation

name: Pull Request Validation

on:
  pull_request:
    branches:
      - main
      - master
    types:
      # milestone changes
      - milestoned
      - demilestoned
      # label changes for “no milestone”
      - labeled
      - unlabeled
      # initial check
      - opened
      - edited
      - reopened
      # code change (e.g. this workflow)
      - synchronize

env:
  LABELS: ${{ join(github.event.pull_request.labels.*.name, '|') }}

jobs:
  check-milestone:
    name: "Triage: Check Milestone"
    runs-on: ubuntu-latest
    steps:
      - name: Check if merging isn’t blocked
        if: contains(env.LABELS, 'DON’T MERGE')
        run: exit 1
      - name: Check if a milestone is necessary and exists
        if: github.event.pull_request.milestone == null && contains(env.LABELS, 'no milestone') == false
        run: exit 1


Close stale issues

name: "Close stale issues"
on:
  schedule:
    - cron: "0 2 * * *"
  workflow_dispatch:

jobs:
  stale:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/stale@v5
        with:
          days-before-issue-stale: -1 # We don't want to mark issues as stale in this action
          days-before-issue-close: 14
          days-before-pr-close: -1 # don't close PRs
          days-before-pr-stale: -1 # don't mark PRs as stale
          stale-issue-label: stale
          any-of-labels: "needs info"
          debug-only: true # enable dry-run, remove when we know from the logs it's working.


Codespell

---
name: Codespell

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

permissions:
  contents: read

jobs:
  codespell:
    name: Check for spelling errors
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Codespell
        uses: codespell-project/actions-codespell@v2


Label stale issues

name: "Label stale issues"
on:
  schedule:
    - cron: "30 1 * * 1,2,3,4,5"
  workflow_dispatch:

jobs:
  stale:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/stale@v5
        with:
          days-before-issue-stale: 60
          days-before-pr-stale: -1 # We don't want to mark PRs as stale
          days-before-close: -1 # We don't want to close issues in this action
          stale-issue-label: stale
          exempt-issue-labels: pinned,enhancement
          stale-issue-message: |
            This issue has been automatically marked as stale because it has not had recent activity.
            Please add a comment if you want to keep the issue open. Thank you for your contributions!
          debug-only: false # set to `true` to enable dry-run


Publish Python Package

name: Publish Python Package

on:
  release:
    types: [published]

jobs:
  publish:
    runs-on: ubuntu-latest
    environment: pypi
    permissions:
      id-token: write # to authenticate as Trusted Publisher to pypi.org
    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: "3.x"
          cache: "pip"
      - run: pip install build
      - run: python -m build
      - uses: pypa/gh-action-pypi-publish@release/v1


AWS GPU

name: AWS GPU

on:
  push:
    branches: [main, "[0-9]+.[0-9]+.x"]
  pull_request:
    types:
      - labeled
      - opened
      - synchronize

# Cancel the job if new commits are pushed
# https://stackoverflow.com/questions/66335225/how-to-cancel-previous-runs-in-the-pr-when-you-push-new-commitsupdate-the-curre
concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

# There are two jobs:
# 1. `check` determines if the second job (`test`) will be run (through a job dependency).
# 2. `test` runs on an AWS runner and executes the GPU tests.
jobs:
  # If the `skip-gpu-ci` label is set, this job is skipped, and consequently the `test` job too.
  # If the `run-gpu-ci` label is set or we reacted to a `push` event, this job succeeds (and `test` is run).
  # If neither is set, this job fails, `test` is skipped, and the whole workflow fails.
  check:
    name: "Triage: Check if GPU tests are allowed to run"
    if: (!contains(github.event.pull_request.labels.*.name, 'skip-gpu-ci'))
    runs-on: ubuntu-latest
    steps:
      - uses: flying-sheep/check@v1
        with:
          success: ${{ github.event_name == 'push' || contains(github.event.pull_request.labels.*.name, 'run-gpu-ci') }}
  # If `check` wasn’t skipped or failed, start an AWS runner and run the GPU tests on it.
  test:
    name: GPU Tests
    needs: check
    runs-on: "cirun-aws-gpu--${{ github.run_id }}"
    # Setting a timeout of 30 minutes, as the AWS costs money
    # At time of writing, a typical run takes about 5 minutes
    timeout-minutes: 30

    defaults:
      run:
        shell: bash -el {0}

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0

      - name: Nvidia SMI sanity check
        run: nvidia-smi

      - uses: mamba-org/setup-micromamba@v1
        with:
          micromamba-version: "1.5.6-0"
          environment-file: ci/gpu_ci.yml
          post-cleanup: "all"

      - name: Install AnnData
        run: uv pip install -e .[dev,test,gpu]

      - name: Env list
        run: |
          micromamba list
          pip list

      - name: Run test
        run: pytest -m gpu --cov --cov-report=xml --cov-context=test -n 4

      - uses: codecov/codecov-action@v3
        with:
          flags: gpu-tests


Benchmark

name: Benchmark

on:
  push:
    branches: [main]
  pull_request:
    branches: [main]

jobs:
  benchmark:
    runs-on: ${{ matrix.os }}
    defaults:
      run:
        shell: bash -e {0} # -e to fail on error

    strategy:
      fail-fast: false
      matrix:
        python: ["3.12"]
        os: [ubuntu-latest]

    env:
      OS: ${{ matrix.os }}
      PYTHON: ${{ matrix.python }}
      ASV_DIR: "./benchmarks"

    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          filter: blob:none

      - name: Fetch main branch for `asv run`’s hash
        run: git fetch origin main:main
        if: ${{ github.ref_name != 'main' }}

      - name: Set up Python ${{ matrix.python }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}
          cache: 'pip'

      - name: Cache datasets
        uses: actions/cache@v4
        with:
          path: |
            ~/.cache
          key: benchmark-state-${{ hashFiles('benchmarks/**') }}

      - name: Install dependencies
        # TODO: revert once this PR is merged: https://github.com/airspeed-velocity/asv/pull/1397
        run: pip install 'asv @ git+https://github.com/ivirshup/asv@fix-conda-usage'

      - name: Configure ASV
        working-directory: ${{ env.ASV_DIR }}
        run: asv machine --yes

      - name: Quick benchmark run
        working-directory: ${{ env.ASV_DIR }}
        run: asv run --dry-run --quick --show-stderr --verbose HEAD^!


Pull Request Validation

name: Pull Request Validation

on:
  pull_request:
    branches:
      - main
    types:
      # milestone changes
      - milestoned
      - demilestoned
      # label changes for “no milestone”
      - labeled
      - unlabeled
      # initial check
      - opened
      - edited
      - reopened
      # code change (e.g. this workflow)
      - synchronize

jobs:
  # This job verifies that the milestone is present or not necessary
  # and determines if “check-relnotes” needs to be run.
  check-milestone:
    name: Check Milestone
    runs-on: ubuntu-latest
    steps:
      - name: Check if milestone or “no milestone” label is present
        uses: flying-sheep/check@v1
        with:
          success: ${{ github.event.pull_request.milestone != null || contains(github.event.pull_request.labels.*.name, 'no milestone') }}
      - name: Check if the “Release notes” checkbox is checked and filled
        uses: kaisugi/action-regex-match@v1.0.0
        id: checked-relnotes
        with:
          text: ${{ github.event.pull_request.body }}
          regex: '^\s*- \[x\] Release notes not necessary because:(.*)$'
          flags: m
    outputs:
      no-relnotes-reason: ${{ steps.checked-relnotes.outputs.group1 }}
  # This job verifies that the relevant release notes file has been modified.
  check-relnotes:
    name: Check for release notes
    runs-on: ubuntu-latest
    needs: check-milestone
    if: ${{ needs.check-milestone.outputs.no-relnotes-reason == '' && !contains(github.event.pull_request.labels.*.name, 'Development Process 🚀') }}
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          filter: blob:none
      - name: Find out if relevant release notes are modified
        uses: dorny/paths-filter@v2
        id: changes
        with:
          filters: | # this is intentionally a string
            relnotes: 'docs/release-notes/${{ github.event.pull_request.milestone.title }}.md'
      - name: Check if relevant release notes are modified
        uses: flying-sheep/check@v1
        with:
          success: ${{ steps.changes.outputs.relnotes }}


Publish Python Package

name: Publish Python Package

on:
  release:
    types: [published]

jobs:
  publish:
    runs-on: ubuntu-latest
    environment: pypi
    permissions:
      id-token: write # to authenticate as Trusted Publisher to pypi.org
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          filter: blob:none
      - uses: actions/setup-python@v5
        with:
          python-version: "3.x"
          cache: "pip"
      - run: pip install build
      - run: python -m build
      - uses: pypa/gh-action-pypi-publish@release/v1


CodeQL

# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
#
# ******** NOTE ********
# We have attempted to detect the languages in your repository. Please check
# the `language` matrix defined below to confirm you have the correct set of
# supported CodeQL languages.
#
name: "CodeQL"

on:
  push:
    branches: [ main ]
  pull_request:
    # The branches below must be a subset of the branches above
    branches: [ main ]
  schedule:
    - cron: '22 7 * * 3'

jobs:
  analyze:
    name: Analyze
    runs-on: ubuntu-latest
    permissions:
      actions: read
      contents: read
      security-events: write

    strategy:
      fail-fast: false
      matrix:
        language: [ 'python' ]
        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python', 'ruby' ]
        # Learn more about CodeQL language support at https://git.io/codeql-language-support

    steps:
    - name: Checkout repository
      uses: actions/checkout@v3

    # Initializes the CodeQL tools for scanning.
    - name: Initialize CodeQL
      uses: github/codeql-action/init@v2
      with:
        languages: ${{ matrix.language }}
        # If you wish to specify custom queries, you can do so here or in a config file.
        # By default, queries listed here will override any specified in a config file.
        # Prefix the list here with "+" to use these queries and those in the config file.
        # queries: ./path/to/local/query, your-org/your-repo/queries@main

    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
    # If this step fails, then you should remove it and run the build manually (see below)
    - name: Autobuild
      uses: github/codeql-action/autobuild@v2

    # ℹ Command-line programs to run using the OS shell.
    # 📚 https://git.io/JvXDl

    # ✏ If the Autobuild fails above, remove it and uncomment the following three lines
    #    and modify them (or add more) to build your code if your project
    #    uses a compiled language

    #- run: |
    #   make bootstrap
    #   make release

    - name: Perform CodeQL Analysis
      uses: github/codeql-action/analyze@v2


Publish Docker image

# docker-image.yml
name: Publish Docker image   # workflow名称，可以在Github项目主页的【Actions】中看到所有的workflow

on:   # 配置触发workflow的事件
  push:
    branches:   # main分支有push时触发此workflow
      - 'main'
    tags:       # tag更新时触发此workflow
      - '*'
  workflow_dispatch:
          inputs:
              name:
                  description: 'Person to greet'
                  required: true
                  default: 'Mona the Octocat'
              home:
                  description: 'location'
                  required: false
                  default: 'The Octoverse'

# 定义环境变量, 后面会使用
# 定义 APP_NAME 用于 docker build-args
# 定义 DOCKERHUB_REPO 标记 docker hub repo 名称
env:
  APP_NAME: douyin_tiktok_download_api
  DOCKERHUB_REPO: evil0ctal/douyin_tiktok_download_api

jobs:
  main:
    # 在 Ubuntu 上运行
    runs-on: ubuntu-latest
    steps:
      # git checkout 代码
      - name: Checkout
        uses: actions/checkout@v2
      # 设置 QEMU, 后面 docker buildx 依赖此.
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v1
      # 设置 Docker buildx, 方便构建 Multi platform 镜像
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v1
      # 登录 docker hub
      - name: Login to DockerHub
        uses: docker/login-action@v1
        with:
          # GitHub Repo => Settings => Secrets 增加 docker hub 登录密钥信息
          # DOCKERHUB_USERNAME 是 docker hub 账号名.
          # DOCKERHUB_TOKEN: docker hub => Account Setting => Security 创建.
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      # 通过 git 命令获取当前 tag 信息, 存入环境变量 APP_VERSION
      - name: Generate App Version
        run: echo APP_VERSION=`git describe --tags --always` >> $GITHUB_ENV
      # 构建 Docker 并推送到 Docker hub
      - name: Build and push
        id: docker_build
        uses: docker/build-push-action@v2
        with:
          # 是否 docker push
          push: true
          # 生成多平台镜像, see https://github.com/docker-library/bashbrew/blob/v0.1.1/architecture/oci-platform.go
          platforms: |
            linux/amd64
            linux/arm64
          # docker build arg, 注入 APP_NAME/APP_VERSION
          build-args: |
            APP_NAME=${{ env.APP_NAME }}
            APP_VERSION=${{ env.APP_VERSION }}
          # 生成两个 docker tag: ${APP_VERSION} 和 latest
          tags: |
            ${{ env.DOCKERHUB_REPO }}:latest
            ${{ env.DOCKERHUB_REPO }}:${{ env.APP_VERSION }}


Translate README

name: Translate README

on:
  push:
    branches:
      - main
      - Dev
jobs:
  build:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v2
      - name: Setup Node.js
        uses: actions/setup-node@v1
        with:
          node-version: 12.x
      # ISO Langusge Codes: https://cloud.google.com/translate/docs/languages
      - name: Adding README - English
        uses: dephraiim/translate-readme@main
        with:
          LANG: en


Code style and test

name: Code style and test

on:
  push: {}
  workflow_dispatch: {}

jobs:
  unit-tests:
    name: Code style & unit tests
    runs-on: ubuntu-latest
    strategy:
      matrix:
        version:
          - "3.10"
          - "3.11"
          - "3.12"
        include:
          - version: "3.10"
            check-lint: "true"
            check-typing: "true"
          - version: "3.11"
            check-lint: "true"
            check-typing: "true"
          - version: "3.12"
            check-lint: "true"
            check-typing: "true"
    steps:
      - uses: actions/checkout@v4
      - name: Install poetry
        run: pipx install poetry
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "${{ matrix.version }}"
          cache: "poetry"
      - name: Install dependencies
        run: |
          poetry install -n
          poetry run pip install -U --no-dependencies aioboto3 aiobotocore types-aioboto3 types-aiobotocore
          poetry install -n
      - name: Lint
        if: ${{ matrix.check-lint }}
        run: |
          poetry run ruff check mypy_boto3_builder --output-format github
          poetry run ruff check scripts --output-format github
      - name: Typing
        if: ${{ matrix.check-typing }}
        run: |
          poetry run pyright mypy_boto3_builder
      - name: Test
        run: |
          poetry run pytest


Bump version and dockerize

name: Bump version and dockerize

on:
  release:
    types: [published]
  workflow_dispatch:
    inputs:
      version:
        description: Version for publishing
        required: true

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          ref: main
      - name: Install poetry
        run: pipx install poetry
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: pyproject.toml
      - name: Get version
        id: version
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            if (context.payload.inputs && context.payload.inputs.version) {
              return context.payload.inputs.version
            }
            return context.ref.split('/').pop()
      - name: Install builder
        run: |
          poetry install -n
      - name: Bump version
        env:
          VERSION: ${{ steps.version.outputs.result }}
        run: |
          echo "Bumping version to ${VERSION}"
          poetry version ${VERSION}
          rm -rf *.egg-info || true
          poetry install -n
      - name: Set up git
        run: |
          git config --global user.email "github-actions@github.com"
          git config --global user.name "github-actions"
      - name: Commit changes
        env:
          VERSION: ${{ steps.version.outputs.result }}
        run: |
          git add pyproject.toml
          git commit -m "Release ${VERSION}"
          git push
      - name: Publish to PyPI
        env:
          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        run: |
          poetry run pip install -U twine wheel setuptools
          poetry build
          poetry run twine upload  --non-interactive dist/*
      - name: Dockerize
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          poetry run bash ./scripts/dockerize.sh


Send coverage report

name: Send coverage report

on:
  push:
    branches:
      - main
    paths:
      - mypy_boto3_builder/**
      - tests/**
  workflow_dispatch: {}

jobs:
  coverage:
    name: Coverage
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Install poetry
        run: pipx install poetry
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: pyproject.toml
          cache: "poetry"
      - name: Install dependencies
        run: |
          poetry install -n
          poetry run pip install -U --no-dependencies aiobotocore
      - name: Build coverage report
        run: |
          poetry run pytest --cov mypy_boto3_builder --cov-report=xml
      - name: Send coverage report
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}


Publish boto3-stubs

name: Publish boto3-stubs
concurrency: update_boto3_stubs

on:
  schedule:
    - cron: "*/15 * * * *"
  workflow_dispatch:
    inputs:
      boto3_version:
        description: Target boto3 version
        required: false
        default: ""
        type: string
      stubs_version:
        description: Specify version explicitly instead of finding it automatically
        required: false
        default: ""
        type: string
      force:
        description: Continue if version is already published
        required: false
        default: false
        type: boolean
      build_all:
        description: Build all packages instead of updated only
        required: false
        default: false
        type: boolean
      skip_published:
        description: Skip already published packages
        required: false
        default: true
        type: boolean
      no_smart_version:
        description: Use exact version provided in stubs_version
        required: false
        default: false
        type: boolean

jobs:
  versions:
    runs-on: ubuntu-latest
    outputs:
      boto3: ${{ steps.boto3.outputs.result }}
      botocore: ${{ steps.botocore.outputs.result }}
      boto3-url: ${{ steps.boto3-url.outputs.result }}
      botocore-url: ${{ steps.botocore-url.outputs.result }}
      version: ${{ steps.version.outputs.result }}
      build-all: ${{ steps.build-all.outputs.result }}
      extra-flags: ${{ steps.extra-flags.outputs.result }}
    steps:
      - uses: actions/checkout@v4
      - name: Find boto3 version
        id: boto3
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getBoto3Version, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })

            const inputVersion = context.payload.inputs && context.payload.inputs.boto3_version
            const version = inputVersion ? inputVersion : await getBoto3Version()
            core.notice(`boto3 version ${version}`)
            return version
      - name: Find botocore version
        id: botocore
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getBotocoreVersion, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })

            const version = getBotocoreVersion("${{ steps.boto3.outputs.result }}")
            core.notice(`botocore version ${version}`)
            return version
      - name: Find build version
        id: version
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getPackageVersions, getNextPostVersion, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })

            if (context.payload.inputs && context.payload.inputs.stubs_version) {
              const version = context.payload.inputs.stubs_version
              core.notice(`stubs version ${version}`)
              return version
            }

            const force = context.payload.inputs ? context.payload.inputs.force !== 'false' : false
            const boto3Version = "${{ steps.boto3.outputs.result }}"

            const versions = await getPackageVersions('boto3-stubs', boto3Version)
            core.info(`Built versions ${versions}`)

            if (!versions.length) {
              core.notice(`No builds found, building initial ${boto3Version}`)
              return boto3Version
            }

            if (!force) {
              core.notice(`Builds ${versions} found, skipping`)
              return ''
            }

            const lastVersion = versions.pop()
            core.notice(`Last version ${lastVersion}`)

            const version = getNextPostVersion(lastVersion)
            core.notice(`Version ${version}`)
            return version
      - name: Build all
        id: build-all
        if: steps.version.outputs.result
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            if (context.payload.inputs && context.payload.inputs.build_all !== 'false') {
              core.notice('Build all = true')
              return 'true'
            }

            const boto3Version = "${{ steps.boto3.outputs.result }}"
            if (boto3Version.endsWith('.0')) {
              core.notice('Build all = true (due to minor version update)')
              return 'true'
            }

            core.notice('Build all = false')
            return 'false'
      - name: Extra flags
        id: extra-flags
        if: steps.version.outputs.result
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const extraFlags = []

            const skipPublished = context.payload.inputs ? context.payload.inputs.skip_published !== 'false' : true
            const noSmartVersion = context.payload.inputs ? context.payload.inputs.no_smart_version !== 'false' : true
            if (skipPublished) extraFlags.push('--skip-published')
            if (noSmartVersion) extraFlags.push('--no-smart-version')

            core.notice(`Extra flags = ${extraFlags}`)
            return extraFlags.join(' ')
      - name: Boto3 download URL
        id: boto3-url
        if: steps.version.outputs.result
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getDownloadURL, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })

            const url = await getDownloadURL('boto3', "${{ steps.boto3.outputs.result }}")
            core.notice(`boto3 download URL ${url}`)
            return url
      - name: Botocore download URL
        id: botocore-url
        if: steps.version.outputs.result
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getDownloadURL, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })

            const url = await getDownloadURL('botocore', "${{ steps.botocore.outputs.result }}")
            core.notice(`botocore download URL ${url}`)
            return url

  publish-boto3-stubs:
    name: Publish boto3-stubs
    runs-on: ubuntu-latest
    needs: versions
    if: needs.versions.outputs.version
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: pyproject.toml
      - name: Install boto3
        run: |
          mkdir installtemp
          cd installtemp
          wget ${{ needs.versions.outputs.boto3-url }}
          wget ${{ needs.versions.outputs.botocore-url }}
          pip uninstall -y s3transfer
          pip install botocore*.whl
          pip install boto3*.whl
          cd ..
          rm -rf installtemp
          python -m pip install newversion
      - name: Install builder
        run: |
          rm -rf mypy_boto3_builder
          python -m pip install mypy_boto3_builder
      - name: Build updated packages only
        if: needs.versions.outputs.build-all == 'false'
        env:
          VERSION: ${{ needs.versions.outputs.version }}
          EXTRA_FLAGS: ${{ needs.versions.outputs.extra-flags }}
        run: |
          rm -rf mypy_boto3_output/*

          echo "Building updated packages"
          python -m mypy_boto3_builder mypy_boto3_output -b ${VERSION} --product boto3 boto3-services ${EXTRA_FLAGS} -s updated -d
      - name: Build all packages
        if: needs.versions.outputs.build-all == 'true'
        env:
          VERSION: ${{ needs.versions.outputs.version }}
          EXTRA_FLAGS: ${{ needs.versions.outputs.extra-flags }}
        run: |
          rm -rf mypy_boto3_output/*

          echo "Building all packages"
          python -m mypy_boto3_builder mypy_boto3_output -b ${VERSION} --product boto3 boto3-services ${EXTRA_FLAGS} -s all -d
      - name: Install dependencies for publishing
        run: |
          python -m pip install setuptools wheel twine
      - name: Publish to PyPI
        env:
          PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
          PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        run: |
          python ./scripts/release.py


boto3-stubs sanity

name: boto3-stubs sanity

on:
  schedule:
    - cron: "0 0 * * *"
  workflow_dispatch:
    inputs:
      boto3_version:
        description: Target boto3 version
        required: false

jobs:
  versions:
    runs-on: ubuntu-latest
    outputs:
      boto3: ${{ steps.boto3.outputs.result }}
      botocore: ${{ steps.botocore.outputs.result }}
    steps:
      - uses: actions/checkout@v4
      - name: Extract boto3 version
        id: boto3
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getBoto3Version, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })
            const inputBoto3Version = context.payload.inputs && context.payload.inputs.boto3_version
            const version = inputBoto3Version ? inputBoto3Version : await getBoto3Version()
            core.notice(`boto3 version ${version}`)
            return version
      - name: Extract botocore version
        id: botocore
        uses: actions/github-script@v7
        env:
          BOTO3_VERSION: ${{ steps.boto3.outputs.result }}
        with:
          result-encoding: string
          script: |
            const { getBotocoreVersion } = require('./.github/workflows/helpers.js')
            const version = getBotocoreVersion(process.env.BOTO3_VERSION)
            core.notice(`botocore version ${version}`)
            return version
  sanity-check:
    runs-on: ubuntu-latest
    needs: versions
    steps:
      - uses: actions/checkout@v4
      - name: Install poetry
        run: pipx install poetry
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: pyproject.toml
          cache: "poetry"
      - name: Install dependencies
        run: |
          poetry install -n
      - name: Install boto3
        env:
          BOTO3_VERSION: ${{ needs.versions.outputs.boto3 }}
          BOTOCORE_VERSION: ${{ needs.versions.outputs.botocore }}
        run: |
          poetry run pip install -U boto3==${BOTO3_VERSION} botocore==${BOTOCORE_VERSION}
      - name: Build master packages
        run: |
          rm -rf mypy_boto3_output/*
          poetry run mypy_boto3_builder mypy_boto3_output --product boto3
      - name: Install master packages
        run: |
          bash ./scripts/install.sh master
      - name: Check s3 output
        run: |
          poetry run mypy_boto3_builder mypy_boto3_output --product boto3-services -s s3
          poetry run python ./scripts/check_output.py -p ./mypy_boto3_output s3
      - name: Build packages
        run: |
          poetry run mypy_boto3_builder mypy_boto3_output --product boto3-services
      - name: Check output
        run: |
          poetry run python ./scripts/check_output.py -p ./mypy_boto3_output


Publish types-aiobotocore

name: Publish types-aiobotocore
concurrency: update_types_aiobotocore

on:
  schedule:
    - cron: "0 0 * * *"
  workflow_dispatch:
    inputs:
      aiobotocore_version:
        description: Target aiobotocore version
        required: false
        type: string
      stubs_version:
        description: Specify version explicitly instead of finding it automatically
        required: false
        default: ""
        type: string
      force:
        description: Continue if version is already published
        required: false
        default: false
        type: boolean
      skip_published:
        description: Skip already published packages
        required: false
        default: true
        type: boolean
      no_smart_version:
        description: Use exact version provided in stubs_version
        required: false
        default: false
        type: boolean

jobs:
  versions:
    runs-on: ubuntu-latest
    outputs:
      aiobotocore: ${{ steps.aiobotocore.outputs.result }}
      aiobotocore-url: ${{ steps.aiobotocore-url.outputs.result }}
      version: ${{ steps.version.outputs.result }}
      extra-flags: ${{ steps.extra-flags.outputs.result }}
    steps:
      - uses: actions/checkout@v4
      - name: Find aiobotocore version
        id: aiobotocore
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getAioBotocoreVersion, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })

            const inputVersion = context.payload.inputs && context.payload.inputs.aiobotocore_version
            const version = inputVersion ? inputVersion : await getAioBotocoreVersion()
            core.notice(`aiobotocore version ${version}`)
            return version
      - name: Find build version
        id: version
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getPackageVersions, getNextPostVersion, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })

            if (context.payload.inputs && context.payload.inputs.stubs_version) {
              const version = context.payload.inputs.stubs_version
              core.notice(`stubs version ${version}`)
              return version
            }

            const force = context.payload.inputs ? context.payload.inputs.force !== 'false' : false
            const aiobotocoreVersion = "${{ steps.aiobotocore.outputs.result }}"

            const versions = await getPackageVersions('types-aiobotocore', aiobotocoreVersion)
            core.info(`Built versions ${versions}`)

            if (!versions.length) {
              core.notice(`No builds found, building initial ${aiobotocoreVersion}`)
              return aiobotocoreVersion
            }

            if (!force) {
              core.notice(`Builds ${versions} found, skipping`)
              return ''
            }

            const lastVersion = versions.pop()
            core.notice(`Last version ${lastVersion}`)

            const version = getNextPostVersion(lastVersion)
            core.notice(`Version ${version}`)
            return version
      - name: Extra flags
        id: extra-flags
        if: steps.version.outputs.result
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const extraFlags = []

            const skipPublished = context.payload.inputs ? context.payload.inputs.skip_published !== 'false' : true
            const noSmartVersion = context.payload.inputs ? context.payload.inputs.no_smart_version !== 'false' : true
            if (skipPublished) extraFlags.push('--skip-published')
            if (noSmartVersion) extraFlags.push('--no-smart-version')

            core.notice(`Extra flags = ${extraFlags}`)
            return extraFlags.join(' ')
      - name: Aiobotocore download URL
        id: aiobotocore-url
        if: steps.version.outputs.result
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getDownloadURL, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })

            const url = await getDownloadURL('aiobotocore', "${{ steps.aiobotocore.outputs.result }}")
            core.notice(`aiobotocore download URL ${url}`)
            return url
  publish-types-aiobotocore:
    name: Publish types-aiobotocore
    runs-on: ubuntu-latest
    needs: versions
    if: needs.versions.outputs.version
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: pyproject.toml
      - name: Install builder
        run: |
          rm -rf mypy_boto3_builder
          python -m pip install mypy_boto3_builder
      - name: Install aiobotocore
        run: |
          mkdir installtemp
          cd installtemp
          wget ${{ needs.versions.outputs.aiobotocore-url }}
          pip uninstall -y s3transfer
          pip install aiobotocore*.whl s3transfer
          cd ..
          rm -rf installtemp
      - name: Build all packages
        env:
          VERSION: ${{ needs.versions.outputs.version }}
          EXTRA_FLAGS: ${{ needs.versions.outputs.extra-flags }}
        run: |
          rm -rf mypy_boto3_output/*

          echo "Building all packages"
          python -m mypy_boto3_builder mypy_boto3_output -b ${VERSION} --product aiobotocore aiobotocore-services ${EXTRA_FLAGS} -s all -d
      - name: Install dependencies for publishing
        run: |
          python -m pip install setuptools wheel twine
      - name: Publish to PyPI
        env:
          PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
          PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        run: |
          python ./scripts/release.py


Publish types-aioboto3

name: Publish types-aioboto3
concurrency: update_types_aioboto3

on:
  schedule:
    - cron: "0 0 * * *"
  workflow_dispatch:
    inputs:
      aioboto3_version:
        description: Target aioboto3 version
        required: false
        type: string
        default: ""
      force:
        description: Continue if version is already published
        required: false
        default: false
        type: boolean
      stubs_version:
        description: Specify version explicitly instead of finding it automatically
        required: false
        type: string
        default: ""
      no_smart_version:
        description: Use exact version provided in stubs_version
        required: false
        default: false
        type: boolean

jobs:
  versions:
    runs-on: ubuntu-latest
    outputs:
      aioboto3: ${{ steps.aioboto3.outputs.result }}
      aioboto3-url: ${{ steps.aioboto3-url.outputs.result }}
      version: ${{ steps.version.outputs.result }}
      extra-flags: ${{ steps.extra-flags.outputs.result }}
    steps:
      - uses: actions/checkout@v4
      - name: Find aioboto3 version
        id: aioboto3
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getAioBoto3Version, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })

            const inputVersion = context.payload.inputs && context.payload.inputs.aioboto3_version
            const version = inputVersion ? inputVersion : await getAioBoto3Version()
            core.notice(`aioboto3 version ${version}`)
            return version
      - name: Find build version
        id: version
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getPackageVersions, getNextPostVersion, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })

            if (context.payload.inputs && context.payload.inputs.stubs_version) {
              const version = context.payload.inputs.stubs_version
              core.notice(`stubs version ${version}`)
              return version
            }

            const force = context.payload.inputs ? context.payload.inputs.force !== 'false' : false
            const aioboto3Version = "${{ steps.aioboto3.outputs.result }}"

            const versions = await getPackageVersions('types-aioboto3', aioboto3Version)
            core.info(`Built versions ${versions}`)

            if (!versions.length) {
              core.notice(`No builds found, building initial ${aioboto3Version}`)
              return aioboto3Version
            }

            if (!force) {
              core.notice(`Builds ${versions} found, skipping`)
              return ''
            }

            const lastVersion = versions.pop()
            core.notice(`Last version ${lastVersion}`)

            const version = getNextPostVersion(lastVersion)
            core.notice(`Version ${version}`)
            return version
      - name: Extra flags
        id: extra-flags
        if: steps.version.outputs.result
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const extraFlags = []

            const noSmartVersion = context.payload.inputs ? context.payload.inputs.no_smart_version !== 'false' : true
            if (noSmartVersion) extraFlags.push('--no-smart-version')

            core.notice(`Extra flags = ${extraFlags}`)
            return extraFlags.join(' ')
      - name: Aioboto3 download URL
        id: aioboto3-url
        if: steps.version.outputs.result
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getDownloadURL, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })

            const url = await getDownloadURL('aioboto3', "${{ steps.aioboto3.outputs.result }}")
            core.notice(`aioboto3 download URL ${url}`)
            return url
  publish-types-aioboto3:
    name: Publish types-aioboto3
    runs-on: ubuntu-latest
    needs: versions
    if: needs.versions.outputs.version
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: pyproject.toml
      - name: Install builder
        run: |
          rm -rf mypy_boto3_builder
          python -m pip install mypy_boto3_builder
      - name: Install aioboto3
        run: |
          mkdir installtemp
          cd installtemp
          wget ${{ needs.versions.outputs.aioboto3-url }}
          pip uninstall -y s3transfer
          pip install aioboto3* s3transfer
          cd ..
          rm -rf installtemp
      - name: Build all packages
        env:
          VERSION: ${{ needs.versions.outputs.version }}
          EXTRA_FLAGS: ${{ needs.versions.outputs.extra-flags }}
        run: |
          rm -rf mypy_boto3_output/*

          echo "Building all packages"
          python -m mypy_boto3_builder mypy_boto3_output -b ${VERSION} --product aioboto3 ${EXTRA_FLAGS} -s all -d
      - name: Install dependencies for publishing
        run: |
          python -m pip install setuptools wheel twine
      - name: Publish to PyPI
        env:
          PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
          PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        run: |
          python ./scripts/release.py


Generate docs

name: Generate docs
concurrency:
  group: generate_docs
  cancel-in-progress: true

on:
  release:
    types: [published]
  workflow_dispatch: {}

jobs:
  build-docs:
    name: Build docs
    runs-on: ubuntu-latest
    env:
      MAIN_BRANCH: main
    steps:
      - name: Checkout main branch
        uses: actions/checkout@v4
        with:
          ref: ${{ env.MAIN_BRANCH }}
      - name: Set up git
        run: |
          git config --global user.email "github-actions@github.com"
          git config --global user.name "github-actions"
      - name: Create gh-pages branch if not exists
        run: |
          GIT_URL=`git config --get remote.origin.url`
          GH_PAGES_BRANCH=`git ls-remote --heads ${GIT_URL} gh-pages`
          if [[ "${GH_PAGES_BRANCH}" == "" ]]; then
            echo "creating gh-pages branch"
            git symbolic-ref HEAD refs/heads/gh-pages
            rm .git/index
            git clean -fdx
            echo "Placeholder" > index.html
            git add .
            git commit -am "First pages commit"
            git push origin gh-pages
            echo "gh-pages branch created"
            git checkout ${MAIN_BRANCH}
          fi
      - name: Install poetry
        run: pipx install poetry
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: pyproject.toml
      - name: Install project
        run: |
          poetry install -n
      - name: Install handsdown
        run: |
          poetry run pip install -U handsdown mkdocs mkdocs-material
      - name: Build docs
        run: |
          poetry run handsdown --external `git config --get remote.origin.url` -o docsmd --cleanup --theme material --branch main $@
          rm -rf docs/*
          poetry run mkdocs build
      - name: Checkout gh-pages branch
        uses: actions/checkout@v4
        with:
          ref: gh-pages
          path: ./gh-pages
          fetch-depth: 0
      - name: Commit changes
        run: |
          cd ./gh-pages
          rm -rf docs docsmd || true
          mv ../docs .
          mv ../docsmd .
          git add docs
          git add docsmd
          git commit -am "Update docs"
          git push


types-aiobotocore sanity

name: types-aiobotocore sanity

on:
  schedule:
    - cron: "0 0 1 * *"
  workflow_dispatch:
    inputs:
      boto3_version:
        description: Target boto3 version
        required: false

jobs:
  versions:
    runs-on: ubuntu-latest
    outputs:
      boto3: ${{ steps.boto3.outputs.result }}
      botocore: ${{ steps.botocore.outputs.result }}
    steps:
      - uses: actions/checkout@v4
      - name: Extract boto3 version
        id: boto3
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getBoto3Version, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })
            const inputBoto3Version = context.payload.inputs && context.payload.inputs.boto3_version
            const version = inputBoto3Version ? inputBoto3Version : await getBoto3Version()
            core.notice(`boto3 version ${version}`)
            return version
      - name: Extract botocore version
        id: botocore
        uses: actions/github-script@v7
        env:
          BOTO3_VERSION: ${{ steps.boto3.outputs.result }}
        with:
          result-encoding: string
          script: |
            const { getBotocoreVersion } = require('./.github/workflows/helpers.js')
            const version = getBotocoreVersion(process.env.BOTO3_VERSION)
            core.notice(`botocore version ${version}`)
            return version
  sanity-check:
    runs-on: ubuntu-latest
    needs: versions
    steps:
      - uses: actions/checkout@v4
      - name: Install poetry
        run: pipx install poetry
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: pyproject.toml
          cache: "poetry"
      - name: Install dependencies
        run: |
          poetry install -n
          poetry run pip install -U aioboto3 boto3-stubs types-aiobotocore types-aioboto3
          poetry install -n
      - name: Install boto3
        env:
          BOTO3_VERSION: ${{ needs.versions.outputs.boto3 }}
          BOTOCORE_VERSION: ${{ needs.versions.outputs.botocore }}
        run: |
          poetry run pip install -U boto3==${BOTO3_VERSION} botocore==${BOTOCORE_VERSION}
      - name: Build master packages
        run: |
          rm -rf mypy_boto3_output/*
          poetry run mypy_boto3_builder mypy_boto3_output --product aioboto3 aiobotocore
      - name: Install master packages
        run: |
          bash ./scripts/install_aiobotocore.sh master
      - name: Check s3 output
        run: |
          poetry run mypy_boto3_builder mypy_boto3_output --product aiobotocore-services -s s3
          poetry run python ./scripts/check_output.py -p ./mypy_boto3_output s3
      - name: Build packages
        run: |
          poetry run mypy_boto3_builder mypy_boto3_output --product aiobotocore-services
      - name: Check output
        run: |
          poetry run python ./scripts/check_output.py -p ./mypy_boto3_output


types-aiobotocore consistency

name: types-aiobotocore consistency

on:
  schedule:
    - cron: "0 0 1 * *"
  workflow_dispatch:
    inputs:
      boto3_version:
        description: Target boto3 version
        required: false

jobs:
  versions:
    runs-on: ubuntu-latest
    outputs:
      boto3: ${{ steps.boto3.outputs.result }}
      botocore: ${{ steps.botocore.outputs.result }}
    steps:
      - uses: actions/checkout@v4
      - name: Extract boto3 version
        id: boto3
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getBoto3Version, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })
            const inputBoto3Version = context.payload.inputs && context.payload.inputs.boto3_version
            const version = inputBoto3Version ? inputBoto3Version : await getBoto3Version()
            core.notice(`boto3 version ${version}`)
            return version
      - name: Extract botocore version
        id: botocore
        uses: actions/github-script@v7
        env:
          BOTO3_VERSION: ${{ steps.boto3.outputs.result }}
        with:
          result-encoding: string
          script: |
            const { getBotocoreVersion } = require('./.github/workflows/helpers.js')
            const version = getBotocoreVersion(process.env.BOTO3_VERSION)
            core.notice(`botocore version ${version}`)
            return version
  check-aio-stubs:
    runs-on: ubuntu-latest
    needs: versions
    steps:
      - uses: actions/checkout@v4
      - name: Install poetry
        run: pipx install poetry
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - name: Install dependencies
        run: |
          poetry install -n
      - name: Install aioboto3
        run: |
          poetry run pip install -U aiobotocore aioboto3
      - name: Install boto3
        env:
          BOTO3_VERSION: ${{ needs.versions.outputs.boto3 }}
          BOTOCORE_VERSION: ${{ needs.versions.outputs.botocore }}
        run: |
          poetry run pip install -U boto3==${BOTO3_VERSION} botocore==${BOTOCORE_VERSION}
          poetry run pip install -U botocore-stubs
      - name: Uninstall stubs
        run: |
          poetry run pip uninstall -y boto3-stubs
      - name: Check aioboto3 stubs
        run: |
          poetry run istub -c ./istub_aio.yml -bid aioboto3
      - name: Check aiobotocore stubs
        run: |
          poetry run istub -c ./istub_aio.yml -bid aiobotocore


boto3-stubs consistency

name: boto3-stubs consistency

on:
  schedule:
    - cron: "0 0 * * *"
  workflow_dispatch:
    inputs:
      boto3_version:
        description: Target boto3 version
        required: false

jobs:
  versions:
    runs-on: ubuntu-latest
    outputs:
      boto3: ${{ steps.boto3.outputs.result }}
      botocore: ${{ steps.botocore.outputs.result }}
    steps:
      - uses: actions/checkout@v4
      - name: Extract boto3 version
        id: boto3
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getBoto3Version, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })
            const inputBoto3Version = context.payload.inputs && context.payload.inputs.boto3_version
            const version = inputBoto3Version ? inputBoto3Version : await getBoto3Version()
            core.notice(`boto3 version ${version}`)
            return version
      - name: Extract botocore version
        id: botocore
        uses: actions/github-script@v7
        env:
          BOTO3_VERSION: ${{ steps.boto3.outputs.result }}
        with:
          result-encoding: string
          script: |
            const { getBotocoreVersion } = require('./.github/workflows/helpers.js')
            const version = getBotocoreVersion(process.env.BOTO3_VERSION)
            core.notice(`botocore version ${version}`)
            return version
  check-stubs:
    runs-on: ubuntu-latest
    needs: versions
    steps:
      - uses: actions/checkout@v4
      - name: Install poetry
        run: pipx install poetry
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: pyproject.toml
      - name: Install dependencies
        run: |
          poetry install -n
      - name: Install boto3
        env:
          BOTO3_VERSION: ${{ needs.versions.outputs.boto3 }}
          BOTOCORE_VERSION: ${{ needs.versions.outputs.botocore }}
        run: |
          poetry run pip install -U boto3==${BOTO3_VERSION} botocore==${BOTOCORE_VERSION}
          poetry run pip install -U botocore-stubs
      - name: Uninstall stubs
        run: |
          poetry run pip uninstall -y boto3-stubs
      - name: Check boto3 stubs
        run: |
          poetry run istub -c ./istub.yml -bid boto3


types-aiobotocore integration

name: types-aiobotocore integration

on:
  schedule:
    - cron: "0 0 * * *"
  workflow_dispatch: {}

jobs:
  integration:
    name: AIO integration tests
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Install poetry
        run: pipx install poetry
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: pyproject.toml
          cache: "poetry"
      - name: Install dependencies
        run: |
          poetry install -n
          poetry run pip install -U aioboto3 boto3-stubs types-aiobotocore types-aioboto3
          poetry install -n
      - name: Integration tests
        run: |
          poetry run python scripts/integration.py --product aioboto3 -d
      - name: Store artifacts
        uses: actions/upload-artifact@v3
        with:
          name: aio_mypy_boto3_output
          path: mypy_boto3_output

  sanity-check:
    name: Sanity check
    needs: integration
    runs-on: ubuntu-latest
    strategy:
      matrix:
        version:
          - "3.8"
          - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v3
        with:
          name: aio_mypy_boto3_output
          path: mypy_boto3_output
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.version }}
          cache: "pip"
      - name: Install dependencies
        run: |
          python -m pip install -U requests cryptography types-requests
          python -m pip install -U mypy ruff pyright aioboto3 boto3-stubs types-aiobotocore types-aioboto3
      - name: Check output
        run: |
          python ./scripts/check_output.py -p ./mypy_boto3_output


boto3-stubs integration

name: boto3-stubs integration

on:
  schedule:
    - cron: "0 0 * * *"
  workflow_dispatch: {}

jobs:
  integration:
    name: Integration tests
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Install poetry
        run: pipx install poetry
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: pyproject.toml
          cache: "poetry"
      - name: Install dependencies
        run: |
          poetry install -n
          poetry run pip install -U --no-dependencies aiobotocore
      - name: Integration tests
        run: |
          poetry run python scripts/integration.py --product boto3 -d
      - name: Store artifacts
        uses: actions/upload-artifact@v3
        with:
          name: mypy_boto3_output
          path: mypy_boto3_output

  sanity-check:
    name: Sanity check
    needs: integration
    runs-on: ubuntu-latest
    strategy:
      matrix:
        version:
          - "3.8"
          - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"
    steps:
      - uses: actions/checkout@v4
      - uses: actions/download-artifact@v3
        with:
          name: mypy_boto3_output
          path: mypy_boto3_output
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.version }}
          cache: "pip"
      - name: Install dependencies
        run: |
          python -m pip install mypy ruff pyright boto3 botocore botocore-stubs boto3-stubs-lite types-requests
      - name: Check output
        run: |
          python scripts/check_output.py


Publish types-aiobotocore-full

name: Publish types-aiobotocore-full
concurrency: update_types_aiobotocore_full

on:
  workflow_dispatch:
    inputs:
      aiobotocore_version:
        description: Target aiobotocore version
        required: false
        type: string
      stubs_version:
        description: Specify version explicitly instead of finding it automatically
        required: false
        default: ""
        type: string
      force:
        description: Continue if version is already published
        required: false
        default: false
        type: boolean
      no_smart_version:
        description: Use exact version provided in stubs_version
        required: false
        default: false
        type: boolean

jobs:
  versions:
    runs-on: ubuntu-latest
    outputs:
      aiobotocore: ${{ steps.aiobotocore.outputs.result }}
      aiobotocore-url: ${{ steps.aiobotocore-url.outputs.result }}
      version: ${{ steps.version.outputs.result }}
      extra-flags: ${{ steps.extra-flags.outputs.result }}
    steps:
      - uses: actions/checkout@v4
      - name: Find aiobotocore version
        id: aiobotocore
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getAioBotocoreVersion, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })

            const inputVersion = context.payload.inputs && context.payload.inputs.aiobotocore_version
            const version = inputVersion ? inputVersion : await getAioBotocoreVersion()
            core.notice(`aiobotocore version ${version}`)
            return version
      - name: Find build version
        id: version
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getPackageVersions, getNextPostVersion, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })

            if (context.payload.inputs && context.payload.inputs.stubs_version) {
              const version = context.payload.inputs.stubs_version
              core.notice(`stubs version ${version}`)
              return version
            }

            const force = context.payload.inputs ? context.payload.inputs.force !== 'false' : false
            const aiobotocoreVersion = "${{ steps.aiobotocore.outputs.result }}"

            const versions = await getPackageVersions('types-aiobotocore-full', aiobotocoreVersion)
            core.info(`Built versions ${versions}`)

            if (!versions.length) {
              core.notice(`No builds found, building initial ${aiobotocoreVersion}`)
              return aiobotocoreVersion
            }

            if (!force) {
              core.notice(`Builds ${versions} found, skipping`)
              return ''
            }

            const lastVersion = versions.pop()
            core.notice(`Last version ${lastVersion}`)

            const version = getNextPostVersion(lastVersion)
            core.notice(`Version ${version}`)
            return version
      - name: Extra flags
        id: extra-flags
        if: steps.version.outputs.result
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const extraFlags = []

            const noSmartVersion = context.payload.inputs ? context.payload.inputs.no_smart_version !== 'false' : true
            if (noSmartVersion) extraFlags.push('--no-smart-version')

            core.notice(`Extra flags = ${extraFlags}`)
            return extraFlags.join(' ')
      - name: Aiobotocore download URL
        id: aiobotocore-url
        if: steps.version.outputs.result
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getDownloadURL, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })

            const url = await getDownloadURL('aiobotocore', "${{ steps.aiobotocore.outputs.result }}")
            core.notice(`aiobotocore download URL ${url}`)
            return url
  publish-types-aiobotocore:
    name: Publish types-aiobotocore
    runs-on: ubuntu-latest
    needs: versions
    if: needs.versions.outputs.version
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: pyproject.toml
      - name: Install builder
        run: |
          rm -rf mypy_boto3_builder
          python -m pip install mypy_boto3_builder
      - name: Install aiobotocore
        run: |
          mkdir installtemp
          cd installtemp
          wget ${{ needs.versions.outputs.aiobotocore-url }}
          pip uninstall -y s3transfer
          pip install aiobotocore*.whl s3transfer
          cd ..
          rm -rf installtemp
      - name: Build
        env:
          VERSION: ${{ needs.versions.outputs.version }}
          EXTRA_FLAGS: ${{ needs.versions.outputs.extra-flags }}
        run: |
          rm -rf mypy_boto3_output/*

          echo "Building"
          python -m mypy_boto3_builder mypy_boto3_output -b ${VERSION} --product aiobotocore aiobotocore-full ${EXTRA_FLAGS} -s all -d
      - name: Install dependencies for publishing
        run: |
          python -m pip install setuptools wheel twine
      - name: Publish to PyPI
        env:
          PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
          PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        run: |
          python ./scripts/release.py


Publish boto3-stubs-full

name: Publish boto3-stubs-full
concurrency: update_boto3_stubs_full

on:
  workflow_dispatch:
    inputs:
      boto3_version:
        description: Target boto3 version
        required: false
        default: ""
        type: string
      stubs_version:
        description: Specify version explicitly instead of finding it automatically
        required: false
        default: ""
        type: string
      force:
        description: Continue if version is already published
        required: false
        default: false
        type: boolean
      no_smart_version:
        description: Use exact version provided in stubs_version
        required: false
        default: false
        type: boolean

jobs:
  versions:
    runs-on: ubuntu-latest
    outputs:
      boto3: ${{ steps.boto3.outputs.result }}
      botocore: ${{ steps.botocore.outputs.result }}
      boto3-url: ${{ steps.boto3-url.outputs.result }}
      botocore-url: ${{ steps.botocore-url.outputs.result }}
      version: ${{ steps.version.outputs.result }}
      extra-flags: ${{ steps.extra-flags.outputs.result }}
    steps:
      - uses: actions/checkout@v4
      - name: Find boto3 version
        id: boto3
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getBoto3Version, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })

            const inputVersion = context.payload.inputs && context.payload.inputs.boto3_version
            const version = inputVersion ? inputVersion : await getBoto3Version()
            core.notice(`boto3 version ${version}`)
            return version
      - name: Find botocore version
        id: botocore
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getBotocoreVersion, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })

            const version = getBotocoreVersion("${{ steps.boto3.outputs.result }}")
            core.notice(`botocore version ${version}`)
            return version
      - name: Find build version
        id: version
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getPackageVersions, getNextPostVersion, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })

            if (context.payload.inputs && context.payload.inputs.stubs_version) {
              const version = context.payload.inputs.stubs_version
              core.notice(`stubs version ${version}`)
              return version
            }

            const force = context.payload.inputs ? context.payload.inputs.force !== 'false' : false
            const boto3Version = "${{ steps.boto3.outputs.result }}"

            const versions = await getPackageVersions('boto3-stubs-full', boto3Version)
            core.info(`Built versions ${versions}`)

            if (!versions.length) {
              core.notice(`No builds found, building initial ${boto3Version}`)
              return boto3Version
            }

            if (!force) {
              core.notice(`Builds ${versions} found, skipping`)
              return ''
            }

            const lastVersion = versions.pop()
            core.notice(`Last version ${lastVersion}`)

            const version = getNextPostVersion(lastVersion)
            core.notice(`Version ${version}`)
            return version
      - name: Extra flags
        id: extra-flags
        if: steps.version.outputs.result
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const extraFlags = []

            const noSmartVersion = context.payload.inputs ? context.payload.inputs.no_smart_version !== 'false' : true
            if (noSmartVersion) extraFlags.push('--no-smart-version')

            core.notice(`Extra flags = ${extraFlags}`)
            return extraFlags.join(' ')
      - name: Boto3 download URL
        id: boto3-url
        if: steps.version.outputs.result
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getDownloadURL, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })

            const url = await getDownloadURL('boto3', "${{ steps.boto3.outputs.result }}")
            core.notice(`boto3 download URL ${url}`)
            return url
      - name: Botocore download URL
        id: botocore-url
        if: steps.version.outputs.result
        uses: actions/github-script@v7
        with:
          result-encoding: string
          script: |
            const { getDownloadURL, setupGlobals } = require('./.github/workflows/helpers.js')
            setupGlobals({ core, context, fetch })

            const url = await getDownloadURL('botocore', "${{ steps.botocore.outputs.result }}")
            core.notice(`botocore download URL ${url}`)
            return url

  publish-boto3-stubs:
    name: Publish boto3-stubs
    runs-on: ubuntu-latest
    needs: versions
    if: needs.versions.outputs.version
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version-file: pyproject.toml
      - name: Install boto3
        run: |
          mkdir installtemp
          cd installtemp
          wget ${{ needs.versions.outputs.boto3-url }}
          wget ${{ needs.versions.outputs.botocore-url }}
          pip uninstall -y s3transfer
          pip install botocore*.whl
          pip install boto3*.whl
          cd ..
          rm -rf installtemp
          python -m pip install newversion
      - name: Install builder
        run: |
          rm -rf mypy_boto3_builder
          python -m pip install mypy_boto3_builder
      - name: Build
        env:
          VERSION: ${{ needs.versions.outputs.version }}
          EXTRA_FLAGS: ${{ needs.versions.outputs.extra-flags }}
        run: |
          rm -rf mypy_boto3_output/*

          echo "Building"
          python -m mypy_boto3_builder mypy_boto3_output -b ${VERSION} --product boto3 boto3-full ${EXTRA_FLAGS} -s all -d
      - name: Install dependencies for publishing
        run: |
          python -m pip install setuptools wheel twine
      - name: Publish to PyPI
        env:
          PYPI_USERNAME: ${{ secrets.PYPI_USERNAME }}
          PYPI_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        run: |
          python ./scripts/release.py


Check PR change log

name: Check PR change log

on:
  pull_request:
    types: [opened, synchronize, labeled, unlabeled]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  changelog_checker:
    name: Check if change log entry is correct
    runs-on: ubuntu-latest
    steps:
    - name: Check change log entry
      uses: scientific-python/action-check-changelogfile@0.3
      env:
        CHANGELOG_FILENAME: CHANGES.rst
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


Check PR milestone

name: Check PR milestone

on:
  # So it cannot be skipped.
  pull_request_target:
    types: [opened, synchronize, milestoned, demilestoned]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  # https://stackoverflow.com/questions/69434370/how-can-i-get-the-latest-pr-data-specifically-milestones-when-running-yaml-jobs
  milestone_checker:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/github-script@v7
      if: github.repository == 'spacetelescope/jdaviz'
      with:
        github-token: ${{ secrets.GITHUB_TOKEN }}
        script: |
          const { data } = await github.request("GET /repos/{owner}/{repo}/pulls/{pr}", {
              owner: context.repo.owner,
              repo: context.repo.repo,
              pr: context.payload.pull_request.number
            });
            if (data.milestone) {
              core.info(`This pull request has a milestone set: ${data.milestone.title}`);
            } else {
              core.setFailed(`A maintainer needs to set the milestone for this pull request.`);
            }


Cron Scheduled CI Tests

# GitHub Actions workflow that runs on a cron schedule.
name: Cron Scheduled CI Tests

on:
  workflow_dispatch:
  schedule:
    # run at 9am UTC on Mondays
    - cron: '0 9 * * 1'
  pull_request:
    # We also want this workflow triggered if the 'Extra CI' label is added
    # or present when PR is updated
    types:
      - synchronize
      - labeled

permissions:
  contents: read

jobs:
  # The linkcheck job tests that the links in the docs point to real places
  # The if statement is to prevent cron from running on forks.
  linkcheck:
    runs-on: ubuntu-latest
    if: (github.repository == 'spacetelescope/jdaviz' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'Extra CI')))
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python to build docs with sphinx
      uses: actions/setup-python@v5
      with:
        python-version: '3.11'
    - name: Install base dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install tox
    - name: Check links in docs using tox
      run: tox -e linkcheck

  ci_cron_tests_dev_roman:
    name: Python 3.11 with latest dev versions of key dependencies and Roman
    runs-on: ubuntu-latest
    if: (github.repository == 'spacetelescope/jdaviz' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'Extra CI')))
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up python
      uses: actions/setup-python@v5
      with:
        python-version: '3.11'
    - name: Install base dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install tox
    - name: Test with tox
      run: tox -e py311-test-devdeps-romandeps

  ci_cron_tests_stable_roman:
    name: Python 3.10 with stable versions of dependencies and Roman
    runs-on: ubuntu-latest
    if: (github.repository == 'spacetelescope/jdaviz' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || contains(github.event.pull_request.labels.*.name, 'Extra CI')))
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up python
      uses: actions/setup-python@v5
      with:
        python-version: '3.10'
    - name: Install base dependencies
      run: |
        python -m pip install --upgrade pip
        python -m pip install tox
    - name: Test with tox
      run: tox -e py310-test-romandeps


CI

name: CI

on:
  push:
    branches:
    - main
    - 'v*'
    tags:
    - 'v*'
  pull_request:

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  # Github Actions supports ubuntu, windows, and macos virtual environments:
  # https://help.github.com/en/actions/reference/virtual-environments-for-github-hosted-runners
  ci_tests:
    name: ${{ matrix.name }}
    runs-on: ${{ matrix.os }}
    continue-on-error: ${{ matrix.allow_failure }}
    strategy:
      matrix:
        include:
          - name: Code style checks
            os: ubuntu-latest
            python: 3.x
            toxenv: codestyle
            allow_failure: false

          - name: PEP 517
            os: ubuntu-latest
            python: 3.x
            toxenv: pep517
            allow_failure: false

          - name: Security audit
            os: ubuntu-latest
            python: 3.x
            toxenv: securityaudit
            allow_failure: false

          - name: Python 3.11 with coverage checking, all deps, and remote data
            os: ubuntu-latest
            python: '3.11'
            toxenv: py311-test-alldeps-cov
            toxposargs: --remote-data
            allow_failure: false

          - name: OS X - Python 3.12
            os: macos-latest
            python: '3.12'
            toxenv: py312-test
            allow_failure: false

          - name: Windows - Python 3.10
            os: windows-latest
            python: '3.10'
            toxenv: py310-test
            allow_failure: false

          # This also runs on cron but we want to make sure new changes
          # won't break this job at the PR stage.
          - name: Python 3.12 with latest dev versions of key dependencies, and remote data
            os: ubuntu-latest
            python: '3.12'
            toxenv: py312-test-devdeps
            toxposargs: --remote-data --run-slow
            allow_failure: true

    steps:
    - name: Checkout code
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - name: Set up python ${{ matrix.python }} on ${{ matrix.os }}
      uses: actions/setup-python@v5
      with:
        python-version: ${{ matrix.python }}
    - name: Install base dependencies
      run: python -m pip install --upgrade pip tox
    - name: Test/run with tox
      run: tox -e ${{ matrix.toxenv }} -- ${{ matrix.toxposargs }}
    - name: Upload coverage to artifacts
      if: "contains(matrix.toxenv, '-cov')"
      uses: actions/upload-artifact@v4
      with:
        name: coverage_${{ matrix.toxenv }}.xml
        path: coverage.xml
        if-no-files-found: error

  upload-codecov:
    needs: [ ci_tests ]
    permissions:
      contents: none
    runs-on: ubuntu-latest
    name: Upload Coverage
    steps:
    - name: Download coverage artifacts
      uses: actions/download-artifact@v4
      with:
        path: coverage
        pattern: coverage_*
        merge-multiple: true
    - name: Upload report to Codecov
      if: ${{ hashFiles('coverage/') != '' }}
      uses: codecov/codecov-action@v4
      with:
        token: ${{ secrets.CODECOV_TOKEN }}
        directory: coverage
        fail_ci_if_error: true
        verbose: true