debug-matrix

---
# $schema https://json.schemastore.org/github-workflow.json
# ./.github/workflows/debug-matrix.yaml
# https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners#preinstalled-software
# yamllint disable rule:line-length

name: debug-matrix

on: # yamllint disable-line rule:truthy
  # pull_request:
  #   branches:
  #     - main
  push:
    branches:
      - main
      - master # @TODO remove
    paths:
      - .github/scripts/debug-*
      - .github/workflows/debug-matrix.yaml
  workflow_call: # Required so this workflow can be called from another workflow
  workflow_dispatch: # Allows you to run this workflow manually from the Actions tab

concurrency:
  group: ${{ github.workflow }}
  cancel-in-progress: true

jobs:
  debug-matrix:
    runs-on: ${{ matrix.os }}
    continue-on-error: true
    defaults:
      run:
        # Docs: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell
        shell: bash --noprofile --norc -e -u -o pipefail -v -x {0}
    strategy:
      fail-fast: false
      max-parallel: 1
      # Docs: https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#choosing-github-hosted-runners
      matrix:
        os:
          # https://github.com/actions/runner-images/tree/main#available-images
          # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners/about-github-hosted-runners#supported-runners-and-hardware-resources
          - ubuntu-latest  # aka ubuntu-22.04
          - windows-latest # aka windows-2022
          - macos-latest   # aka macos-12
          - macos-13       # beta
          # is this coming?:
          # - macos-13-arm64 # https://github.com/actions/runner-images/blob/main/images/macos/macos-13-arm64-Readme.md
          # - ubuntu-20.04
          # - windows-2019
          # - macos-11
    steps:
      - name: USES crazy-max/ghaction-dump-context@v2.3.0
        uses: crazy-max/ghaction-dump-context@v2.3.0

      - if: runner.os == 'Linux'
        name: List installed apt/snap packages (Linux)
        continue-on-error: true
        run: |
          # bash: List installed apt/snap packages (Linux)
          echo "apt --manual-installed:"
          apt list --manual-installed | sort
          echo "apt list --installed:"
          apt list --installed | sort
          # echo "apt-mark:"
          # apt-mark showmanual
          echo "snap list:"
          snap list | sort

      - if: runner.os == 'macOS'
        name: List installed pkgutil/brew packages (macOS)
        continue-on-error: true
        run: |
          # bash: List installed pkgutil/brew packages (macOS)
          echo "pkgutil:"
          pkgutil --pkgs | sort
          echo "brew list:"
          brew list --versions | sort || true

      - if: runner.os == 'Windows'
        name: USES MinoruSekine/setup-scoop@main
        uses: MinoruSekine/setup-scoop@main
        with:
          buckets: extras versions
        continue-on-error: true

      - if: runner.os == 'Windows'
        name: List installed choco/scoop/winget packages (Windows)
        shell: cmd
        continue-on-error: true
        run: |
          @rem cmd: List installed choco/scoop/winget packages (Windows)

          echo scoop --version:
          call scoop --version

          echo choco --version:
          choco --version

          echo winget --version (if installed):
          where /q winget && winget --version

          echo scoop list:
          call scoop list

          echo choco list:
          choco list --include-programs --yes

          @rem main/git main/innounp main/dark main/lessmsi
          call scoop install --skip --no-update-scoop main/7zip
          call scoop install --skip --no-update-scoop versions/winget-preview

          echo winget list:
          winget list --disable-interactivity --accept-source-agreements | sort

          echo winget --info:
          winget --info --disable-interactivity

          @rem https://github.com/actions/runner-images/issues/6472#issue-1422886379
          @rem yamllint disable rule:line-length
          @rem yamllint disable
          @rem yamllint disable-line rule:line-length
          @rem Invoke-WebRequest -Uri 'https://aka.ms/Microsoft.VCLibs.x64.14.00.Desktop.appx' -OutFile 'VCLibs.appx' | Out-Null # automatically downloads latest version
          @rem yamllint disable-line rule:line-length
          @rem Invoke-WebRequest -Uri 'https://github.com/microsoft/winget-cli/releases/download/latest/Microsoft.DesktopAppInstaller_8wekyb3d8bbwe.msixbundle' -OutFile 'winget.msixbundle' | Out-Null
          @rem yamllint disable-line rule:line-length
          @rem $ReleaseInfo = Invoke-RestMethod 'https://api.github.com/repos/microsoft/winget-cli/releases/latest'
          @rem yamllint disable-line rule:line-length
          @rem Invoke-WebRequest -Uri $ReleaseInfo.assets.Where({ $_.name.EndsWith('License1.xml') }).browser_download_url -OutFile 'license.xml' | Out-Null # because the prefix of the file changes with every release
          @rem yamllint disable-line rule:line-length
          @rem Add-AppxProvisionedPackage -Online -PackagePath winget.msixbundle -LicensePath license.xml -DependencyPackagePath VCLibs.appx | Out-Null
          @rem yamllint enable

      # - if: false # runner.os == 'Windows'
      #   shell: powershell
      #   run: |
      #     $hasPackageManager = Get-AppPackage -name 'Microsoft.DesktopAppInstaller'
      #     if (!$hasPackageManager -or [version]$hasPackageManager.Version -lt [version]"1.10.0.0") {
      #       "Installing winget Dependencies"
      #       Add-AppxPackage -Path 'https://aka.ms/Microsoft.VCLibs.x64.14.00.Desktop.appx'
      #       $releases_url = 'https://api.github.com/repos/microsoft/winget-cli/releases/latest'

      #       [Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
      #       # yamllint disable-line rule:line-length
      #       $releases = Invoke-RestMethod -uri $releases_url
      #       $latestRelease = $releases.assets | Where { $_.browser_download_url.EndsWith('msixbundle') } | Select -First 1

      #       "Installing winget from $($latestRelease.browser_download_url)"
      #       Add-AppxPackage -Path $latestRelease.browser_download_url
      #     } else {
      #       "winget already installed"
      #     }
      #     #### Creating settings.json #####
      #     if ([System.Security.Principal.WindowsIdentity]::GetCurrent().IsSystem) {
      #       # yamllint disable-line rule:line-length
      #       $SettingsPath = "$Env:windir\system32\config\systemprofile\AppData\Local\Microsoft\WinGet\Settings\settings.json"
      #     } else {
      #      # yamllint disable-line rule:line-length
      #       $SettingsPath = "$env:LOCALAPPDATA\Packages\Microsoft.DesktopAppInstaller_8wekyb3d8bbwe\LocalState\settings.json"
      #     }
      #     if (Test-Path $SettingsPath) {
      #       $ConfigFile = Get-Content -Path $SettingsPath | Where-Object {$_ -notmatch '//'} | ConvertFrom-Json
      #     }
      #     if (!$ConfigFile){
      #       $ConfigFile = @{}
      #     }
      #     if ($ConfigFile.installBehavior.preferences.scope) {
      #       $ConfigFile.installBehavior.preferences.scope = "Machine"
      #     } else {
      #       Add-Member -InputObject $ConfigFile -MemberType NoteProperty -Name 'installBehavior' -Value $(
      #         New-Object PSObject -Property $(@{preferences = $(
      #           New-Object PSObject -Property $(@{scope = "Machine"}))
      #         })
      #       ) -Force
      #     }
      #     $ConfigFile | ConvertTo-Json | Out-File $SettingsPath -Encoding utf8 -Force

      - name: USES actions/checkout
        uses: actions/checkout@v4.1.1
        with:
          show-progress: false

      # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsshell
      # - if: contains(fromJSON('["Linux", "macOS"]'), runner.os)
      - name: RUN sh ./.github/scripts/debug*.sh
        # default: sh -e {0}
        shell: sh -e -v -x {0}
        continue-on-error: true
        run: |
          mapfile -t < <(find ./.github/scripts -name 'debug*.sh') || true
          for script in "${MAPFILE[@]}"; do
            "${script}" "$0" "$@" || true
          done

      - name: RUN bash ./.github/scripts/debug*.bash
        # default: bash --noprofile --norc -eo pipefail {0}
        # shell: bash
        continue-on-error: true
        run: |
          mapfile -t < <(find ./.github/scripts -name 'debug*.bash') || true
          for script in "${MAPFILE[@]}"; do
            "${script}" "$0" "$@" || true
          done

      - name: RUN bash ./.github/scripts/debug*.bash
        # default: bash --noprofile --norc -eo pipefail {0}
        shell: bash
        continue-on-error: true
        run: |
          mapfile -t < <(find ./.github/scripts -name 'debug*.bash') || true
          for script in "${MAPFILE[@]}"; do
            "${script}" "$0" "$@" || true
          done

      - name: RUN pwsh ./.github/scripts/debug*.ps1
        # default: pwsh -command ". '{0}'"
        shell: pwsh -Noninteractive -command ". '{0}'"
        continue-on-error: true
        run: |
          Get-ChildItem ".github\scripts\debug*.ps1" | Foreach-Object { & $_.FullName "pwsh" }
          Exit 0

      - if: runner.os == 'Windows'
        name: RUN powershell ./.github/scripts/debug*.ps1 (Windows)
        # default: powershell -command ". '{0}'"
        shell: powershell -Noninteractive -command ". '{0}'"
        continue-on-error: true
        run: |
          Get-ChildItem ".github\scripts\debug*.ps1" | Foreach-Object { & $_.FullName "powershell" }
          Exit 0

      - if: runner.os == 'Windows'
        name: RUN cmd .github/scripts/debug*.cmd (Windows)
        # default: %ComSpec% /D /E:ON /V:OFF /S /C "CALL "{0}""
        # /D     Ignore registry AutoRun commands
        # /E:ON  Enable CMD Command Extensions
        # /V:OFF Disable delayed environment variable expansion
        # /S     Strip " quote characters from command.
        # /C     Run Command and then terminate
        shell: 'cmd /D /E:ON /V:ON /S /C "CALL "{0}""'
        continue-on-error: true
        run: |
          for %%A in (.github\scripts\debug*.ps1) do "%%~A" "%~fs0" %*
          exit 0

      - if: runner.os == 'Linux'
        name: RUN list executables (Linux)
        continue-on-error: true
        run: |
          # list executables (Linux)
          IFS=: read -r -a paths <<<"${PATH}"
          find "${paths[@]}" -maxdepth 1 -type f,l -executable -printf '%P\t%h\n' | \
            LC_COLLATE=C sort --ignore-case -u | \
            column -ts $'\t' || true

      - if: runner.os == 'macOS'
        name: RUN list executables (macOS)
        continue-on-error: true
        run: |
          # list executables (macOS)
          which bash
          which sh
          IFS=: read -r -a paths <<<"${PATH}"
          find "${paths[@]}" -maxdepth 1 -perm +111 -type f -or -type l -print | \
            awk '{b=$0;sub(/\/.*\//,"", b);sub(/\/[a-z]*$/,"",$0);print b "\t" $0}' | \
            LC_COLLATE=C sort --ignore-case -u | \
            column -ts $'\t' || true

      - if: runner.os == 'Windows'
        name: RUN list executables (Windows)
        shell: bash
        continue-on-error: true
        run: |
          # list executables (Windows)
          set -vx
          IFS=: read -r -a paths <<<"${PATH}"
          find "${paths[@]}" -maxdepth 1 -type f,l -regextype posix-extended -iregex '.*\.(bat|cmd|com|exe)$' -printf '%P\t%h\n' | \
            LC_COLLATE=C sort --ignore-case -u | \
            column -ts $'\t' || true

      - name: RUN python (inline)
        # default: python {0}
        # note: this is python2 on macos-12
        shell: python
        continue-on-error: true
        run: |
          # python (inline)
          import os
          import pprint
          import sys
          print("os.name=%s" % os.name)
          print("sys.executable=%s" % sys.executable)
          print("sys.argv=")
          pprint.pprint(sys.argv)
          print("sys.version=%s" % sys.version)
          print("sys.platform=%s" % sys.platform)
          print("os.environ=")
          pprint.pprint(dict(os.environ), width=4096)
          sys.exit(0)

      - # if: runner.os == 'Linux'
        name: RUN install pip
        continue-on-error: true
        run: |
          # install pip
          set -x
          type -P pip >/dev/null && exit 0
          PYTHON=$(type -P python3 python py python2 | head -n 1 || true)
          # shellcheck disable=SC2139 # This expands when defined, not when used. Consider escaping
          alias pip="${PYTHON} -m pip"
          sudo=$(type -P sudo || true)

          pyversion=$("${PYTHON}" -c 'import sys; print(sys.version_info[0]);')

          if command -v apt-get >/dev/null; then
            sudo apt-get -qq -y update

            if ((pyversion>2)); then
              sudo apt-get -qq -y install python3-pip
              exit 0
            fi

            sudo apt-get -qq -y install python-pip
            exit 0
          fi

          GET_PIP_URL==https://bootstrap.pypa.io/pip/2.7/get-pip.py
          curl -s -o get-pip.py "${GET_PIP_URL}"

          ${sudo} "${PYTHON}" get-pip.py

      - name: RUN upgrade pip
        shell: bash
        continue-on-error: true
        run: |
          # upgrade pip
          set -x
          PYTHON=$(type -P python3 python py python2 | head -n 1 || true)
          if ! type -P pip >/dev/null; then
            # shellcheck disable=SC2139 # This expands when defined, not when used. Consider escaping
            alias pip="${PYTHON} -m pip"
          fi
          sudo=$(type -P sudo || true)

          ${sudo} "${PYTHON}" -m pip install --upgrade pip

      - name: RUN pip list
        shell: bash
        continue-on-error: true
        run: |
          # pip list
          set -x
          PYTHON=$(type -P python3 python py python2 | head -n 1 || true)
          if ! type -P pip >/dev/null; then
            # shellcheck disable=SC2139 # This expands when defined, not when used. Consider escaping
            alias pip="${PYTHON} -m pip"
          fi

          echo "pip list:"
          pip list
          echo "pip list --outdated:"
          pip list --outdated

      - name: RUN pip install future
        shell: bash
        continue-on-error: true
        run: |
          # pip install future
          set -vx
          PYTHON=$(type -P python3 python py python2 | head -n 1 || true)
          if ! type -P pip >/dev/null; then
            # shellcheck disable=SC2139 # This expands when defined, not when used. Consider escaping
            alias pip="${PYTHON} -m pip"
          fi
          sudo=$(type -P sudo || true)

          if pip show future >/dev/null; then
            exit 0
          fi

          py_version=$("${PYTHON}" -c 'import sys; print(sys.version_info[0]);')

          if command -v apt-get >/dev/null; then
            if ((py_version==3)); then
              app=python3-future
            else
              app=python-future
            fi
            ${sudo} apt-get -y install "${app}"
            if pip show future >/dev/null; then
              exit 0
            fi
          fi

          ${sudo} pip --no-python-version-warning --no-input install future
          if pip show future >/dev/null; then
            exit 0
          fi

          # --root-user-action=ignore
          ${sudo} pip install --upgrade setuptools

          # https://github.com/PythonCharmers/python-future/issues/604
          PYTHON_FUTURE_URL="https://github.com/PythonCharmers/python-future/archive/master.tar.gz"
          curl -s "${PYTHON_FUTURE_URL}" | tar xzf -

          cd python-future-master/src
          ${sudo} "${PYTHON}" setup.py install
          cd ..
          rm -fr python-future-master

          pip show future
          exit $?

      - name: RUN python .github/scripts/debug*.py (via os.system)
        # default: python {0}
        # note: this is python2 on macos-12
        shell: python
        continue-on-error: true
        run: |
          # python .github/scripts/debug*.py (via os.system)
          import glob
          import os
          import sys
          scripts = glob.iglob("./.github/scripts/debug*.py")
          for script in scripts:
            args = [sys.executable, script]
            args += sys.argv[:1]
            print("Executing %s" % " ".join(args))
            os.system(" ".join(args))
          sys.exit(0)

      - name: RUN python3 (inline)
        shell: python3 {0}
        continue-on-error: true
        run: |
          # python3 (inline)
          import os
          import pprint
          import sys
          print("os.name=%s" % os.name)
          print("sys.executable=%s" % sys.executable)
          print("sys.argv=")
          pprint.pprint(sys.argv)
          print("sys.version=%s" % sys.version)
          print("sys.platform=%s" % sys.platform)
          print("os.environ=")
          pprint.pprint(dict(os.environ), width=4096)
          sys.exit(0)

      - name: RUN python3 .github/scripts/debug*.py (via subprocess.run)
        shell: python3 {0}
        continue-on-error: true
        run: |
          # python3 .github/scripts/debug*.py (via subprocess.run)
          import glob
          import pprint
          import subprocess
          import sys
          scripts = glob.iglob("./.github/scripts/debug*.py")
          for script in scripts:
            args = [sys.executable, script]
            args += sys.argv[:1]
            print("Executing %s" % " ".join(args))
            rv = subprocess.run(args)
            print("rv=")
            pprint.pprint(rv)
          sys.exit(0)

      - name: RUN python3 .github/scripts/debug*.py (via subprocess.run/shell=true)
        shell: python3 {0}
        continue-on-error: true
        run: |
          # python3 .github/scripts/debug*.py (via subprocess.run/shell=true)
          import glob
          import pprint
          import subprocess
          import sys
          scripts = glob.iglob("./.github/scripts/debug*.py")
          for script in scripts:
            args = [sys.executable, script]
            args += sys.argv[:1]
            print("Executing %s" % " ".join(args))
            rv = subprocess.run(args, shell=True)
            print("rv:")
            pprint.pprint(rv)
            print('stdout:')
            pprint.pprint(rv.stdout)
            print('stderr:')
            pprint.pprint(rv.stderr)
          sys.exit(0)

      - name: RUN python .github/scripts/debug-vars.py
        shell: python .github/scripts/debug-vars.py {0}
        continue-on-error: true
        run: |
          # shell: python .github/scripts/debug-vars.py {0}

      - name: RUN python3 .github/scripts/debug-vars.py
        shell: python3 .github/scripts/debug-vars.py {0}
        continue-on-error: true
        run: |
          # shell: python3 .github/scripts/debug-vars.py {0}

# cspell:ignore

# cSpell:ignore bbwe, choco, innounp, iregex, lessmsi, Minoru, msixbundle
# cSpell:ignore Noninteractive, noprofile, norc, pkgs, pyversion, regextype, Sekine
# cSpell:ignore showmanual, wekyb, winget


debug-rasa

---
# $schema https://json.schemastore.org/github-workflow.json
# ./.github/workflows/debug-rasa.yaml
# checkov:skip=CKV_GHA_7:The build output cannot be affected by user parameters
#   other than the build entry point and the top-level source location.
#   GitHub Actions workflow_dispatch inputs MUST be empty.

name: debug-rasa

env:
  WORKFLOW_USER: rasa
  WORKFLOW_REPO: workflows

on: # yamllint disable-line rule:truthy
  # push:
  #   branches:
  #     - main
  # pull_request:
  #   branches:
  #     - main
  workflow_call: # Required so this workflow can be called from another workflow
    inputs:
      workflow_user:
        description: "Github user name (default is 'rasa')"
        required: false
        type: string
      workflow_repo:
        description: "Github repository name (default is 'workflows')"
        required: false
        type: string
  workflow_dispatch: # Allows you to run this workflow manually from the Actions tab
    inputs:
      # checkov:skip=CKV_GHA_7
      workflow_user:
        description: "Github user name (default is 'rasa')"
        required: false
        type: string
      # checkov:skip=CKV_GHA_7
      workflow_repo:
        description: "Github repository name (default is 'workflows')"
        required: false
        type: string

concurrency:
  group: ${{ github.workflow }}
  cancel-in-progress: true

jobs:
  debug-rasa:
    runs-on: ubuntu-22.04
    steps:
      - name: set vars
        run: |
          printf 'WORKFLOW_USER=%s\n' "${{ inputs.workflow_user }}" >> "${GITHUB_ENV}"
          printf 'WORKFLOW_REPO=%s\n' "${{ inputs.workflow_repo }}" >> "${GITHUB_ENV}"
      - name: USES crazy-max/ghaction-dump-context@v2.3.0
        uses: crazy-max/ghaction-dump-context@v2.3.0
        with:
          workflow_user: ${{ inputs.workflow_user }}
          workflow_repo: ${{ inputs.workflow_repo }}
        env:
          WORKFLOW_USER: ${{ env.WORKFLOW_USER }}
          WORKFLOW_REPO: ${{ env.WORKFLOW_REPO }}


dependabot

---
# Copied from: rasa/dotfiles/.github/workflows/dependabot.yml
# EDIT THE ABOVE FILE, NOT THIS COPY, OR YOUR CHANGES WILL BE LOST!
# $schema https://json.schemastore.org/github-workflow.json

name: dependabot

on: # yamllint disable-line rule:truthy
  # See https://stackoverflow.com/a/74959635
  pull_request_target:
    branches:
      - main
      - master # @TODO remove
  push:
    branches:
      - main
      - master # @TODO remove
  workflow_call: # Required so this workflow can be called from another workflow
  workflow_dispatch: # Allows you to run this workflow manually from the Actions tab

concurrency:
  group: ${{ github.workflow }}
  cancel-in-progress: true

permissions:
  pull-requests: write
  contents: write

jobs:
  job:
    runs-on: ubuntu-latest
    # See https://github.com/dependabot/fetch-metadata?tab=readme-ov-file#enabling-auto-merge
    if: startsWith(github.event.pull_request.user.login, 'dependabot')
    steps:
      - name: Dependabot metadata
        id: dependabot-metadata
        uses: dependabot/fetch-metadata@v2.1.0

      - name: Enable auto-merge for Dependabot PRs
        # if: steps.dependabot-metadata.outputs.update-type == 'version-update:semver-patch'
        run: gh pr merge --auto --merge "$PR_URL"
        env:
          PR_URL: ${{ github.event.pull_request.html_url }}
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}


Excavator

on:
  workflow_dispatch:
  schedule:
    # run every 4 hours
    - cron: '20 */4 * * *'
name: Excavator
jobs:
  excavate:
    name: Excavate
    runs-on: windows-2022 # was windows-latest
    steps:
    - uses: actions/checkout@main
    - name: Excavate
      uses: ScoopInstaller/GithubActions@main
      env:
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        SKIP_UPDATED: '1'


Issues

on:
  issues:
    types: [ opened, labeled ]
name: Issues
jobs:
  issueHandler:
    runs-on: ubuntu-latest
    name: Issue Verification
    steps:
      - uses: actions/checkout@main
      - name: Verify Issue
        uses: Ash258/Scoop-GithubActions@stable
        if: github.event.action == 'opened' || (github.event.action == 'labeled' && contains(github.event.issue.labels.*.name, 'verify'))
        env:
          GITH_EMAIL: ross@smithii.com
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


Commented Pull Request

on:
  issue_comment:
    types: [ created ]
name: Commented Pull Request
jobs:
  pullRequestHandler:
    name: Pull Request Validator
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@main
    - name: Pull Request Validation
      uses: Ash258/Scoop-GithubActions@stable
      if: startsWith(github.event.comment.body, '/verify')
      env:
        GITH_EMAIL: ross@smithii.com
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


mega-linter

---
# Copied from: rasa/dotfiles/.github/workflows/mega-linter.yml
# EDIT THE ABOVE FILE, NOT THIS COPY, OR YOUR CHANGES WILL BE LOST!
# yamllint disable rule:line-length
# $schema https://json.schemastore.org/github-workflow.json
# checkov:skip=CKV_GHA_7:The build output cannot be affected by user parameters
#   other than the build entry point and the top-level source location.
#   GitHub Actions workflow_dispatch inputs MUST be empty.

name: mega-linter

# Trigger mega-linter at every push. Action will also be visible from Pull
# Requests to main
on: # yamllint disable-line rule:truthy
  # Comment this line to trigger action only on pull-requests
  # (not recommended if you don't pay for GH Actions)
  push:
    branches:
      - main
      - master # @TODO remove
  pull_request:
    branches:
      - main
      - master # @TODO remove

# <@rasa>
  workflow_call: # Required so this workflow can be called from another workflow
    inputs:
      # checkov:skip=CKV_GHA_7
      fetch_depth:
        # description: "Number of commits to fetch. 0 indicates all history for all branches and tags."
        type: string
        default: "1"
      # checkov:skip=CKV_GHA_7
      validate_all_codebase:
        # description: "Validate the entire code base (enter any value)"
        type: string
        default: "true"
  workflow_dispatch: # Allows you to run this workflow manually from the Actions tab
    inputs:
      # checkov:skip=CKV_GHA_7
      fetch_depth:
        description: "Number of commits to fetch. 0 indicates all history for all branches and tags."
        type: string
        default: "1"
      # checkov:skip=CKV_GHA_7
      validate_all_codebase:
        description: "Validate the entire code base."
        type: choice
        options:
          - "false"
          - "true"
        default: "true"

concurrency:
  group: ${{ github.workflow }}
  cancel-in-progress: true
# </@rasa>

# Comment env block if you do not want to apply fixes
env:
  # Apply linter fixes configuration
  #
  # When active, APPLY_FIXES must also be defined as environment variable
  # (in github/workflows/mega-linter.yml or other CI tool)
  APPLY_FIXES: all

  # Decide which event triggers application of fixes in a commit or a PR
  # (pull_request, push, all)
  APPLY_FIXES_EVENT: all # was `pull_request` changed by @rasa

  # If APPLY_FIXES is used, defines if the fixes are directly committed (commit)
  # or posted in a PR (pull_request)
  APPLY_FIXES_MODE: commit

jobs:
  megalinter:
    name: MegaLinter
    runs-on: ubuntu-latest

    # <@rasa>
    # defaults:
    #   run:
    #     shell: bash --noprofile --norc -v -x {0}
    # </@rasa>

    # Give the default GITHUB_TOKEN write permission to commit and push, comment
    # issues & post new PR; remove the ones you do not need
    permissions:
      contents: write
      issues: write
      pull-requests: write

    steps:
      # Git Checkout
      - name: Checkout Code
        uses: actions/checkout@v4.1.1 # was v4, changed by @rasa
        with:
          token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }}

          # If you use VALIDATE_ALL_CODEBASE = true, you can remove this line to
          # improve performance
          fetch-depth: ${{ inputs.fetch_depth }} # added by @rasa
          show-progress: false # added by @rasa

      # <@rasa>
      - name: output debug vars
        run: |
          cat <<-EOF
          inputs=${{ toJSON(inputs) }}
          EOF
      # </@rasa>

      # MegaLinter
      - name: MegaLinter

        # You can override MegaLinter flavor used to have faster performances
        # More info at https://megalinter.io/flavors/
        uses: oxsecurity/megalinter@v7.12.0 # changed from v7 by @rasa

        id: ml

        # All available variables are described in documentation
        # https://megalinter.io/configuration/
        env:

          # Validates all source when push on main, else just the git diff with
          # main. Override with true if you always want to lint all sources
          #
          # To validate the entire codebase, set to:
          # VALIDATE_ALL_CODEBASE: true
          #
          # To validate only diff with main, set to:
          # VALIDATE_ALL_CODEBASE: >-
          #   ${{
          #     github.event_name == 'push' &&
          #     contains(fromJSON('["refs/heads/main", "refs/heads/master"]'), github.ref)
          #   }}
          VALIDATE_ALL_CODEBASE: >-
            ${{
              inputs.validate_all_codebase == 'true' ||
              github.event_name == 'push' &&
              contains(fromJSON('["refs/heads/main", "refs/heads/master"]'), github.ref)
            }}

          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

          # ADD YOUR CUSTOM ENV VARIABLES HERE OR DEFINE THEM IN A FILE
          # .mega-linter.yml AT THE ROOT OF YOUR REPOSITORY

          # Uncomment to disable copy-paste and spell checks
          # DISABLE: COPYPASTE,SPELL

      # Upload MegaLinter artifacts
      - name: Archive production artifacts
        uses: actions/upload-artifact@v4.3.3 # changed from v3 by @rasa
        if: success() || failure() # changed by @rasa: was ${{ success() }} || ${{ failure() }}
        with:
          name: MegaLinter reports
          path: |
            megalinter-reports
            mega-linter.log

      # <@rasa>
      - name: Remove production artifacts
        continue-on-error: true
        run: |
          sudo chown -Rc "${UID}" megalinter-reports mega-linter.log || true
          sudo chmod a+w megalinter-reports mega-linter.log || true
          sudo rm -fr megalinter-reports mega-linter.log || true
      # - name: USES crazy-max/ghaction-dump-context@v2.1.0
      #   uses: crazy-max/ghaction-dump-context@v2.1.0

      - name: output debug vars
        run: |
          cat <<-EOF
          steps.ml.outputs=${{ toJSON(steps.ml.outputs) }}
          EOF
      # </@rasa>

      # Set APPLY_FIXES_IF var for use in future steps
      - name: Set APPLY_FIXES_IF var
        run: |
          printf 'APPLY_FIXES_IF=%s\n' "${{
            steps.ml.outputs.has_updated_sources == 1 &&
            (
              env.APPLY_FIXES_EVENT == 'all' ||
              env.APPLY_FIXES_EVENT == github.event_name
            ) &&
            (
              github.event_name == 'push' ||
              github.event.pull_request.head.repo.full_name == github.repository
            )
          }}" >> "${GITHUB_ENV}"

      # Set APPLY_FIXES_IF_* vars for use in future steps
      - name: Set APPLY_FIXES_IF_* vars
        run: |
          printf 'APPLY_FIXES_IF_PR=%s\n' "${{
            env.APPLY_FIXES_IF == 'true' &&
            env.APPLY_FIXES_MODE == 'pull_request'
          }}" >> "${GITHUB_ENV}"
          printf 'APPLY_FIXES_IF_COMMIT=%s\n' "${{
            env.APPLY_FIXES_IF == 'true' &&
            env.APPLY_FIXES_MODE == 'commit' &&
            (!contains(fromJSON('["refs/heads/main", "refs/heads/master"]'), github.ref))
          }}" >> "${GITHUB_ENV}"

      # Create pull request if applicable
      # (for now works only on PR from same repository, not from forks)
      - name: Create Pull Request with applied fixes
        uses: peter-evans/create-pull-request@v6.1.0 # changed from v5 by @rasa
        id: cpr
        if: env.APPLY_FIXES_IF_PR == 'true'
        with:
          token: ${{ secrets.PAT || secrets.GITHUB_TOKEN }}
          commit-message: "[MegaLinter] Apply linters automatic fixes"
          title: "[MegaLinter] Apply linters automatic fixes"
          labels: bot

      - name: Create PR output
        if: env.APPLY_FIXES_IF_PR == 'true'
        run: |
          echo "PR Number - ${{ steps.cpr.outputs.pull-request-number }}"
          echo "PR URL - ${{ steps.cpr.outputs.pull-request-url }}"

      # Push new commit if applicable
      # (for now works only on PR from same repository, not from forks)
      - name: Prepare commit
        if: env.APPLY_FIXES_IF_COMMIT == 'true'
        run: sudo chown -Rc $UID .git/

      - name: Commit and push applied linter fixes
        uses: stefanzweifel/git-auto-commit-action@v5.0.1 # changed from v4 by @rasa
        if: env.APPLY_FIXES_IF_COMMIT == 'true'
        with:
          branch: >-
            ${{
              github.event.pull_request.head.ref ||
              github.head_ref ||
              github.ref
            }}
          commit_message: "[MegaLinter] Apply linters fixes"
          commit_user_name: megalinter-bot
          commit_user_email: nicolas.vuillamy@ox.security

# cspell:ignore stefanzweifel

# cSpell:ignore noprofile, norc


Pull Requests

on:
  pull_request:
    types: [ opened ]
name: Pull Requests
jobs:
  pullRequestHandler:
    name: Pull Request Validator
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@main
    - name: Pull Request Validation
      uses: Ash258/Scoop-GithubActions@stable
      env:
        GITH_EMAIL: ross@smithii.com
        GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


pydeps

---
# Copied from: rasa/dotfiles/.github/workflows/pydeps.yml
# EDIT THE ABOVE FILE, NOT THIS COPY, OR YOUR CHANGES WILL BE LOST!
# checkov:skip=CKV_GHA_7:The build output cannot be affected by user parameters
#   other than the build entry point and the top-level source location.
#   GitHub Actions workflow_dispatch inputs MUST be empty.
# $schema https://json.schemastore.org/github-workflow.json

name: pydeps

env:
  DEFAULT_MASK: "**/*requirements*.txt"

on: # yamllint disable-line rule:truthy
  schedule:
    # run every day at 15:17 UTC (07:17 PST/08:17 PDT)
    - cron: "17 15 * * *"
    #        ┬  ┬  ┬  ┬  ┬                               # *    : run every 5 min/every 1 hour/every 1 day/every 1 mon
    #        │  │  │  │  └─ day of week (0-6 or SUN-SAT) # */X  : run every X min/every X hour/every X day/every X mon
    #        │  │  │  └──── month (1-12 or JAN-DEC)      # A,B  : run when min/hour/day/mon = A or B
    #        │  │  └─────── day of month (1-31)          # A-B  : run every min/hour/day/mon between A and B
    #        │  └────────── hour (0-23) (UTC/GMT+0)      # A-B/X: run every X min/hour/day/mon between A and B
    #        └───────────── minute (0-59)                # Ex: '0 9-17/2 * * 1-5' : 9/11/1/3/5 on Mon/Tue/Wed/Thu/Fri
  push:
    branches:
      - main
      - master # @TODO remove
    paths:
      - "**/*requirements*.txt"
  workflow_call:
    inputs:
      # checkov:skip=CKV_GHA_7
      mask:
        type: string
        default: "**/*requirements*.txt"
        description: "File mask for requirements*.txt file(s)"
  workflow_dispatch: # Allows you to run this workflow manually from the Actions tab
    inputs:
      # checkov:skip=CKV_GHA_7
      mask:
        type: string
        default: "**/*requirements*.txt"
        description: "File mask for requirements*.txt file(s)"

concurrency:
  group: ${{ github.workflow }}
  cancel-in-progress: true

jobs:
  job:
    runs-on: ${{ matrix.os }}
    defaults:
      run:
        # force Windows to use bash:
        shell: bash
        # shell: bash -v -x {0}
        # shell: bash --noprofile --norc -e -o pipefail {0}
    strategy:
      fail-fast: true
      max-parallel: 1
      # yamllint disable-line rule:line-length
      # https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#choosing-github-hosted-runners
      matrix:
        os:
          - ubuntu-latest # aka ubuntu-22.04
          # - windows-latest # aka windows-2022
          # - macos-latest # aka macos-12 (macos-13 in beta)
        python-version:
          # - "3.10"
          - "3.11"
          # - "3.12"
    steps:
      - name: USES actions/checkout@v4.1.1
        uses: actions/checkout@v4.1.1
        with:
          show-progress: false

      - name: RUN gather file list
        continue-on-error: true
        run: |
          mask="${{ inputs.mask || env.DEFAULT_MASK }}"
          printf "mask=%s\n" "${mask}"
          dir=$(dirname "${mask}")
          base=$(basename "${mask}")
          if [[ "${dir}" == "**" ]]; then
            dir=.
            depth=99
          else
            depth=1
          fi
          mapfile -t < <(find "${dir}" -maxdepth "${depth}" -type f -iname "${base}")
          printf 'Found %d files matching %s:\n' "${#MAPFILE[@]}" "${mask}"
          typeset -p MAPFILE
          serialized=$(typeset -p MAPFILE | base64 -w 0)
          printf 'len(serialized)=%d\n' "${#serialized}"
          printf 'serialized=%s\n' "${serialized}" >>"${GITHUB_ENV}"
          # fail if MAPFILE is empty
          ((${#MAPFILE[@]}>0))

      - name: RUN find duplicate packages, if any, across all requirements files
        if: success()
        run: |
          set -v -x
          serialized="${serialized:-}"
          printf 'len(serialized)=%d\n' "${#serialized}"
          deserialized=$(base64 -d <<<"${serialized}")
          printf 'len(deserialized)=%d\n' "${#deserialized}"
          eval "${deserialized}"
          mask="${{ inputs.mask || env.DEFAULT_MASK }}"
          printf 'Found %d files matching %s:\n' "${#MAPFILE[@]}" "${mask}"
          tmp1=$(mktemp)
          cat "${MAPFILE[@]}" \
            | grep -E '^\s*[^#]' \
            | sed -E 's/\s+//g' \
            | sort -u \
            | sed -E 's/[<>~!]+/=/g' \
            | cut -d'=' -f 1 \
            | sort \
            | uniq -d > "${tmp1}"
          printf "Duplicates found: "
          wc -l < "${tmp1}"
          if [[ -s "${tmp1}" ]]; then
            printf "\n"
            cat "${tmp1}"
            printf "\n"
            sed -e 's/^/\^/g' "${tmp1}" \
              | grep -H -f - "${MAPFILE[@]}" \
              | sed "s|^|::notice::|g"
          fi
          exit 0

      - name: USES actions/setup-python@v5.0.0
        if: success()
        uses: actions/setup-python@v5.0.0
        with:
          python-version: ${{ matrix.python-version }}
          # yamllint disable-line rule:line-length
          # see https://stackoverflow.com/questions/73789054/reusable-github-action-workflow-with-actions-setup-python-fails-because-it-cant
          # cache: 'pip'
          # cache-dependency-path: '${{ inputs.mask }}'

      - name: RUN pip install --upgrade pip
        if: success()
        run: |
          python -m pip install --upgrade pip

      - name: RUN pip install pipdeptree pipreqs pur
        if: success()
        run: |
          pip install pipdeptree pipreqs pur

      - name: RUN sed -E -i.bak -e '/^\s*#/d; /^\s*$/d;' requirements.txt
        if: success()
        run: |
          eval "$(base64 -d <<<"${serialized}")"
          sed -E -i.bak -e '/^\s*#/d; /^\s*$/d;' "${MAPFILE[@]}"
          pr -f -n "${MAPFILE[@]}" \
            | tr -s '\n'

      - name: RUN pipreqs --print --diff requirements.txt
        if: success()
        run: |
          eval "$(base64 -d <<<"${serialized}")"
          for file in "${MAPFILE[@]}"; do
            printf '\n%s:\n' "${file}"
            pipreqs --print --diff "${file}" 2>/dev/null || true
          done

      - name: RUN pur --dry-run --dry-run-changed --no-recursive --requirement requirements.txt
        if: success()
        run: |
          eval "$(base64 -d <<<"${serialized}")"
          for file in "${MAPFILE[@]}"; do
            printf '\n%s: ' "${file}"
            out=$(
              pur --dry-run --dry-run-changed --no-recursive --requirement "${file}" \
                | grep -v '==>' || true
            )
            if [[ -z "${out}" ]]; then
              printf 'no changes found\n'
              continue
            fi
            printf 'changes found:\n%s\n' "${out}"
            # shellcheck disable=SC2001
            sed "s|^|::notice::${file}:|g" <<<"${out}"
          done

      - name: RUN pur --no-recursive --requirement requirements.txt
        if: success()
        run: |
          eval "$(base64 -d <<<"${serialized}")"
          for file in "${MAPFILE[@]}"; do
              printf '\n%s: ' "${file}"
              out=$(
                pur --no-recursive --requirement "${file}" \
                  | grep -v '==>' || true
              )
              if grep -q 'All requirements up-to-date' <<<"${out}"; then
                printf 'no changes found\n'
                continue
              fi
              printf 'changes found:\n%s\n' "${out}"
              # shellcheck disable=SC2001
              sed "s|^|::notice::${file}:|g" <<<"${out}"
          done

      - name: RUN pip install -r requirements.txt
        if: success()
        run: |
          eval "$(base64 -d <<<"${serialized}")"
          for file in "${MAPFILE[@]}"; do
            printf '\n%s: ' "${file}"
            pip install -r "${file}" || true
          done

      - name: RUN pipdeptree --warn silence
        if: success()
        run: |
          pipdeptree --warn silence || true

      - name: RUN pipdeptree --freeze --warn silence | grep -E '^[a-zA-Z0-9\-]+'
        if: success()
        run: |
          pipdeptree --freeze --warn silence \
            | grep -E '^[a-zA-Z0-9\-]+' || true

# cspell:ignore pipreqs

# cSpell:ignore noprofile, norc, pipdeptree, pydeps


toc

---
# Copied from: rasa/dotfiles/.github/workflows/toc.yml
# EDIT THE ABOVE FILE, NOT THIS COPY, OR YOUR CHANGES WILL BE LOST!
# $schema https://json.schemastore.org/github-workflow.json

name: toc

on: # yamllint disable-line rule:truthy
  pull_request:
    branches:
      - main
      - master # @TODO remove
    paths:
      - "**/*.md"
  push:
    paths:
      - "**/*.md"
  workflow_dispatch: # Allows you to run this workflow manually from the Actions tab

concurrency:
  group: ${{ github.workflow }}
  cancel-in-progress: true

jobs:
  job:
    runs-on: ubuntu-latest
    timeout-minutes: 5
    env:
      GH_TOC_TOKEN: ${{ secrets.GITHUB_TOKEN }}
    steps:
      - name: USES actions/checkout@v4.1.1
        uses: actions/checkout@v4.1.1
        with:
          show-progress: false

      - name: RUN gh-md-toc
        run: |
          curl -fsSL --output gh-md-toc https://raw.githubusercontent.com/ekalinin/github-markdown-toc/HEAD/gh-md-toc
          chmod a+x gh-md-toc
          find . -type f -iname '*.md' -exec ./gh-md-toc --indent 0 --no-backup --hide-footer --skip-header {} \;
          rm -f gh-md-toc
          true

      - name: USES stefanzweifel/git-auto-commit-action@v5.0.1
        uses: stefanzweifel/git-auto-commit-action@v5.0.1
        with:
          commit_message: "docs: auto update table-of-contents in markdown"

# cSpell:ignore stefanzweifel


update-readme

---
# From https://github.com/hoilc/scoop-lemon/blob/master/.github/workflows/make_index.yml
on:
  schedule:
    # Run daily at 00:59 UTC
    - cron: '59 0 * * *'
  workflow_dispatch:

name: update-readme

jobs:
  update-readme:
    name: update-readme
    # if: github.event_name == 'workflow_dispatch' || github.event.pull_request.merged == true
    runs-on: ubuntu-latest
    steps:
    - uses: actions/setup-python@v5
      with:
        python-version: '3.12'
        # unused:
        # cache: 'pip' # caching pip dependencies

    - uses: actions/checkout@v4
    - run: |
        python makeindex.py

    - uses: EndBug/add-and-commit@v9
      with:
        add: 'README.md'
        # branch: master
        # author_name: Ross Smith II
        # author_email: ross@smithii.com
        message: 'chore: update app index [ci skip]'

# eof


on:
  push:
    branches:
    - master
    - develop
    - feature/*

jobs:
  gen_docs:
    name: generate DOAJ documentation and fuctional tests and push to the docs repo
    runs-on: ubuntu-latest
    permissions:
      contents: write
      pages: write
      statuses: write

    steps:
      - name: checkout the branch of interest
        uses: actions/checkout@v3
        with:
          path: doaj

      - name: checkout the documentation repo
        uses: actions/checkout@v3
        with:
          path: doaj-docs
          repository: DOAJ/doaj-docs
          token: ${{ secrets.STEVE_PAT_DOAJ_ACTIONS_DOCS }}
          fetch-depth: 0

      - name: set up a python 3.8 environment
        uses: actions/setup-python@v4.4.0
        with:
          python-version: 3.8

      - name: install elasticsearch
        uses: getong/elasticsearch-action@v1.2
        with:
          # for some reason 7.10.2 isn't on docker hub
          elasticsearch version: 7.10.1

      - name: install the DOAJ dependencies
        run: |
          cd $GITHUB_WORKSPACE/doaj
          pip install -e .[test]
          pip install -e .[docs]

      - name: run the docs generation script and push to the docs repo
        run: |
          export DOAJENV=test
          cd $GITHUB_WORKSPACE/doaj/docs
          sh gendocs.sh --nocov
          cd $GITHUB_WORKSPACE/doaj-docs
          git config user.name github-actions
          git config user.email github-actions@github.com
          git add .
          git commit -m "GH Actions docs run for ${{ github.ref_name }}"
          git push


autofix.ci

name: autofix.ci

on:
  pull_request:
  push:
    branches:
      - main

permissions:
  contents: read

jobs:
  autofix:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version-file: .github/python-version.txt
      - run: pip install -e .[dev]
      - run: ruff --fix-only .
      - run: ruff format .

      - run: web/gen/all

      - uses: actions/setup-node@v4
        with:
          node-version-file: .github/node-version.txt
      - name: Run prettier
        run: |
          npm ci
          npm run prettier
        working-directory: web

      - uses: mhils/add-pr-ref-in-changelog@main

      - uses: autofix-ci/action@dd55f44df8f7cdb7a6bf74c78677eb8acd40cd0a


CI

name: CI

on:
  push:
    branches:
      - '**'
      - '!dependabot/**'
  pull_request:
  merge_group:
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  lint:
    uses: mhils/workflows/.github/workflows/python-tox.yml@main
    with:
      cmd: tox -e lint

  filename-matching:
    uses: mhils/workflows/.github/workflows/python-tox.yml@main
    with:
      cmd: tox -e filename_matching

  mypy:
    uses: mhils/workflows/.github/workflows/python-tox.yml@main
    with:
      cmd: tox -e mypy

  individual-coverage:
    uses: mhils/workflows/.github/workflows/python-tox.yml@main
    with:
      cmd: tox -e individual_coverage

  test:
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: ubuntu-latest
            py: "3.12"
          - os: windows-latest
            py: "3.12"
          - os: macos-latest
            py: "3.12"
          - os: ubuntu-latest
            py: "3.11"
          - os: ubuntu-latest
            py: "3.10"
    runs-on: ${{ matrix.os }}
    steps:
      - run: printenv
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
          fetch-depth: 0
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.py }}
      - run: pip install tox
      - run: tox -e py
        if: matrix.os != 'ubuntu-latest'
      - name: Run tox -e py (without internet)
        run: |
          # install dependencies (requires internet connectivity)
          tox -e py --notest
          # run tests with loopback only. We need to sudo for unshare, which means we need an absolute path for tox.
          sudo unshare --net -- sh -c "ip link set lo up; $(which tox) -e py"
        if: matrix.os == 'ubuntu-latest'
      - uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./coverage.xml

  test-old-dependencies:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
          fetch-depth: 0
      - uses: actions/setup-python@v5
        with:
          python-version-file: .github/python-version.txt
      - run: pip install tox-uv
      - run: tox -e old-dependencies

  build:
    strategy:
      fail-fast: false
      matrix:
        include:
          - image: macos-14
            platform: macos-arm64
          - image: macos-12
            platform: macos-x86_64
          - image: windows-2019
            platform: windows
          - image: ubuntu-20.04  # Oldest available version so we get oldest glibc possible.
            platform: linux
    runs-on: ${{ matrix.image }}
    env:
      CI_BUILD_KEY: ${{ secrets.CI_BUILD_KEY }}
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
          fetch-depth: 0
      - uses: actions/setup-python@v5
        with:
          python-version-file: .github/python-version.txt
      - if: matrix.platform == 'windows'
        uses: actions/cache@v4
        with:
          path: release/installbuilder/setup
          key: installbuilder
      - run: pip install .[dev]  # pyinstaller 5.9 does not like pyproject.toml + editable installs.

      - if: startsWith(matrix.platform, 'macos') && github.repository == 'mitmproxy/mitmproxy'
            && (startsWith(github.ref, 'refs/heads/') || startsWith(github.ref, 'refs/tags/'))
        id: keychain
        uses: apple-actions/import-codesign-certs@63fff01cd422d4b7b855d40ca1e9d34d2de9427d
        with:
          keychain: ${{ runner.temp }}/temp
          p12-file-base64: ${{ secrets.APPLE_CERTIFICATE }}
          p12-password: ${{ secrets.APPLE_CERTIFICATE_PASSWORD }}
      - if: startsWith(matrix.platform, 'macos') && github.repository == 'mitmproxy/mitmproxy'
            && (startsWith(github.ref, 'refs/heads/') || startsWith(github.ref, 'refs/tags/'))
        run: |
          python -u release/build.py macos-app \
            --keychain "${{ runner.temp }}/temp.keychain" \
            --team-id "S8XHQB96PW" \
            --apple-id "${{ secrets.APPLE_ID }}" \
            --password "${{ secrets.APPLE_APP_PASSWORD }}"

      # Linux
      - if: matrix.platform == 'linux'
        run: python -u release/build.py standalone-binaries wheel

      # Windows
      - if: matrix.platform == 'windows'
        run: python -u release/build.py standalone-binaries
      - if: matrix.platform == 'windows' && github.repository == 'mitmproxy/mitmproxy' &&
            (github.ref == 'refs/heads/citest' || startsWith(github.ref, 'refs/tags/'))
        run: python -u release/build.py --dirty installbuilder-installer msix-installer

      - uses: actions/upload-artifact@v4
        with:
          # artifacts must have different names, see https://github.com/actions/upload-artifact/issues/24
          name: binaries.${{ matrix.platform }}
          path: |
            release/dist

  test-web-ui:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
      - uses: actions/setup-node@v4
        with:
          node-version-file: .github/node-version.txt
      - name: Cache Node.js modules
        uses: actions/cache@v4
        with:
          # npm cache files are stored in `~/.npm` on Linux/macOS
          path: ~/.npm
          key: ${{ runner.OS }}-node-${{ hashFiles('**/package-lock.json') }}
          restore-keys: |
            ${{ runner.OS }}-node-
            ${{ runner.OS }}-
      - working-directory: ./web
        run: npm ci
      - working-directory: ./web
        run: npm test
      - uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          files: ./web/coverage/coverage-final.json

  docs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
      - uses: actions/setup-python@v5
        with:
          python-version-file: .github/python-version.txt
      - run: |
          wget -q https://github.com/gohugoio/hugo/releases/download/v0.92.1/hugo_extended_0.92.1_Linux-64bit.deb
          echo "a9440adfd3ecce40089def287dee4e42ffae252ba08c77d1ac575b880a079ce6 hugo_extended_0.92.1_Linux-64bit.deb" | sha256sum -c
          sudo dpkg -i hugo*.deb
      - run: pip install -e .[dev]
      - run: ./docs/build.py
      - uses: actions/upload-artifact@v4
        with:
          name: docs
          path: docs/public
      # For releases, also build the archive version of the docs.
      - if: startsWith(github.ref, 'refs/tags/')
        run: ./docs/build.py
        env:
          DOCS_ARCHIVE: true
      - if: startsWith(github.ref, 'refs/tags/')
        uses: actions/upload-artifact@v4
        with:
          name: docs-archive
          path: docs/public

  check:
    if: always()
    needs:
    - lint
    - filename-matching
    - mypy
    - individual-coverage
    - test
    - test-old-dependencies
    - test-web-ui
    - build
    - docs
    uses: mhils/workflows/.github/workflows/alls-green.yml@main
    with:
      jobs: ${{ toJSON(needs) }}

  # Separate from everything else because slow.
  build-and-deploy-docker:
    if: github.repository == 'mitmproxy/mitmproxy' && (
      github.ref == 'refs/heads/main'
      || github.ref == 'refs/heads/citest'
      || startsWith(github.ref, 'refs/tags/')
      )
    environment: deploy-docker
    needs: check
    runs-on: ubuntu-latest
    env:
      DOCKER_USERNAME: mitmbot
      DOCKER_PASSWORD: ${{ secrets.DOCKER_PASSWORD }}
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
      - uses: actions/setup-python@v5
        with:
          python-version-file: .github/python-version.txt
      - uses: actions/download-artifact@v4
        with:
          name: binaries.linux
          path: release/dist
      - uses: docker/setup-qemu-action@68827325e0b33c7199eb31dd4e31fbe9023e06e3 # v3.0.0
      - uses: docker/setup-buildx-action@d70bba72b1f3fd22344832f00baa16ece964efeb # v1.6.0
      - run: python release/build-and-deploy-docker.py

  deploy:
    # This action has access to our AWS keys, so we are extra careful here.
    # In particular, we don't blindly `pip install` anything to minimize the risk of supply chain attacks.
    if: github.repository == 'mitmproxy/mitmproxy' && (startsWith(github.ref, 'refs/heads/') || startsWith(github.ref, 'refs/tags/'))
    environment: ${{ (github.ref == 'refs/heads/citest' || startsWith(github.ref, 'refs/tags/')) && 'deploy-release' || 'deploy-snapshot' }}
    needs: check
    runs-on: ubuntu-latest
    permissions:
      id-token: write
      attestations: write
    env:
      # PyPI and MSFT keys are only available for the deploy-release environment
      # The AWS access key for snapshots is scoped to branches/* as well.
      TWINE_USERNAME: __token__
      TWINE_PASSWORD: ${{ secrets.TWINE_PASSWORD }}
      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
      AWS_DEFAULT_REGION: us-west-2
      MSFT_APP_ID: 9NWNDLQMNZD7
      MSFT_TENANT_ID: ${{ secrets.MSFT_TENANT_ID }}
      MSFT_CLIENT_ID: ${{ secrets.MSFT_CLIENT_ID }}
      MSFT_CLIENT_SECRET: ${{ secrets.MSFT_CLIENT_SECRET }}
      R2_ACCOUNT_ID: ${{ secrets.R2_ACCOUNT_ID }}
      R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
      R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
    steps:
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
      - uses: actions/setup-python@v5
        with:
          python-version-file: .github/python-version.txt
      - run: sudo apt-get update
      - run: sudo apt-get install -y awscli
      - if: startsWith(github.ref, 'refs/tags/')
        run: sudo apt-get install -y twine

      - uses: actions/download-artifact@v4
        with:
          name: docs
          path: docs/public
      - if: startsWith(github.ref, 'refs/tags/')
        uses: actions/download-artifact@v4
        with:
          name: docs-archive
          path: docs/archive
      - uses: actions/download-artifact@v4
        with:
          pattern: binaries.*
          merge-multiple: true
          path: release/dist
      - id: provenance
        uses: actions/attest-build-provenance@v1
        with:
          subject-path: 'release/dist/*'
      - run: |
          REF=${{ github.ref_name }}
          mv ${{ steps.provenance.outputs.bundle-path }} release/dist/mitmproxy-${REF#v}.sigstore
      - run: ls docs/public
      - run: ls release/dist

      - run: ./release/deploy.py

      - name: Deploy to Microsoft Store (test flight)
        if: github.ref == 'refs/heads/citest'
        run: ./release/deploy-microsoft-store.py release/dist/*.msix
        env:
          MSFT_APP_FLIGHT: 174ca570-8cae-4444-9858-c07293f1f13a
      - name: Deploy to Microsoft Store
        if: startsWith(github.ref, 'refs/tags/')
        run: ./release/deploy-microsoft-store.py release/dist/*.msix


Release

name: Release

on:
  workflow_dispatch:
    inputs:
      version:
        description: 'Version (major.minor.patch)'
        required: true
        type: string
      skip-branch-status-check:
        description: 'Skip CI status check.'
        default: false
        required: false
        type: boolean

permissions:
  actions: write
  contents: write

jobs:
  release:
    environment: deploy-release
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          token: ${{ secrets.GH_PUSH_TOKEN }}  # this token works to push to the protected main branch.
      - uses: actions/setup-node@v4
        with:
          node-version-file: .github/node-version.txt
      - uses: actions/setup-python@v5
        with:
          python-version-file: .github/python-version.txt
      - run: ./release/release.py ${{ inputs.version }} ${{ inputs.skip-branch-status-check }}
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # this token works with the GraphQL API


Issue Manager

name: Issue Manager

on:
  schedule:
    - cron: "0 0 * * *"
  issue_comment:
    types:
      - created
      - edited
  issues:
    types:
      - labeled

jobs:
  issue-manager:
    if: github.repository_owner == 'explosion'
    runs-on: ubuntu-latest
    steps:
      - uses: tiangolo/issue-manager@0.4.0
        with:
          token: ${{ secrets.GITHUB_TOKEN }}
          config: >
            {
              "resolved": {
                "delay": "P7D",
                "message": "This issue has been automatically closed because it was answered and there was no follow-up discussion.",
                "remove_label_on_comment": true,
                "remove_label_on_close": true
              },
              "more-info-needed": {
                "delay": "P7D",
                "message": "This issue has been automatically closed because there has been no response to a request for more information from the original author. With only the information that is currently in the issue, there's not enough information to take action. If you're the original author, feel free to reopen the issue if you have or find the answers needed to investigate further.",
                "remove_label_on_comment": true,
                "remove_label_on_close": true
              }
            }


Explosion Bot

name: Explosion Bot

on:
  issue_comment:
    types:
      - created
      - edited

jobs:
  explosion-bot:
    if: github.repository_owner == 'explosion'
    runs-on: ubuntu-latest
    steps:
      - name: Dump GitHub context
        env:
          GITHUB_CONTEXT: ${{ toJson(github) }}
        run: echo "$GITHUB_CONTEXT"
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v4
      - name: Install and run explosion-bot
        run: |
          pip install git+https://${{ secrets.EXPLOSIONBOT_TOKEN }}@github.com/explosion/explosion-bot
          python -m explosionbot
        env:
          INPUT_TOKEN: ${{ secrets.EXPLOSIONBOT_TOKEN }}
          INPUT_BK_TOKEN: ${{ secrets.BUILDKITE_SECRET }}
          ENABLED_COMMANDS: "test_gpu,test_slow,test_slow_gpu"
          ALLOWED_TEAMS: "spaCy"


Lock Threads

name: 'Lock Threads'

on:
  schedule:
    - cron: '0 0 * * *'  # check every day
  workflow_dispatch:

permissions:
  issues: write

concurrency:
  group: lock

jobs:
  action:
    if: github.repository_owner == 'explosion'
    runs-on: ubuntu-latest
    steps:
      - uses: dessant/lock-threads@v5
        with:
          process-only: 'issues'
          issue-inactive-days: '30'
          issue-comment: >
            This thread has been automatically locked since there
            has not been any recent activity after it was closed.
            Please open a new issue for related bugs.


tests

name: tests

on:
  push:
    branches-ignore:
      - "spacy.io"
      - "nightly.spacy.io"
      - "v2.spacy.io"
    paths-ignore:
      - "*.md"
      - "*.mdx"
      - "website/**"
      - ".github/workflows/**"
  pull_request:
    types: [opened, synchronize, reopened, edited]
    paths-ignore:
      - "*.md"
      - "*.mdx"
      - "website/**"

jobs:
  validate:
    name: Validate
    if: github.repository_owner == 'explosion'
    runs-on: ubuntu-latest
    steps:
      - name: Check out repo
        uses: actions/checkout@v4

      - name: Configure Python version
        uses: actions/setup-python@v4
        with:
          python-version: "3.7"

      - name: black
        run: |
          python -m pip install black -c requirements.txt
          python -m black spacy --check
      - name: isort
        run: |
          python -m pip install isort -c requirements.txt
          python -m isort spacy --check
      - name: flake8
        run: |
          python -m pip install flake8==5.0.4
          python -m flake8 spacy --count --select=E901,E999,F821,F822,F823,W605 --show-source --statistics
      - name: cython-lint
        run: |
          python -m pip install cython-lint -c requirements.txt
          # E501: line too log, W291: trailing whitespace, E266: too many leading '#' for block comment
          cython-lint spacy --ignore E501,W291,E266

  tests:
    name: Test
    needs: Validate
    strategy:
      fail-fast: true
      matrix:
        os: [ubuntu-latest, windows-latest, macos-latest]
        python_version: ["3.12"]
        include:
          - os: windows-latest
            python_version: "3.7"
          - os: macos-latest
            python_version: "3.8"
          - os: ubuntu-latest
            python_version: "3.9"
          - os: windows-latest
            python_version: "3.10"
          - os: macos-latest
            python_version: "3.11"

    runs-on: ${{ matrix.os }}

    steps:
      - name: Check out repo
        uses: actions/checkout@v4

      - name: Configure Python version
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python_version }}

      - name: Install dependencies
        run: |
          python -m pip install -U build pip setuptools
          python -m pip install -U -r requirements.txt

      - name: Build sdist
        run: |
          python -m build --sdist

      - name: Run mypy
        run: |
          python -m mypy spacy
        if: matrix.python_version != '3.7'

      - name: Delete source directory and .egg-info
        run: |
          rm -rf spacy *.egg-info
        shell: bash

      - name: Uninstall all packages
        run: |
          python -m pip freeze
          python -m pip freeze --exclude pywin32 > installed.txt
          python -m pip uninstall -y -r installed.txt

      - name: Install from sdist
        run: |
          SDIST=$(python -c "import os;print(os.listdir('./dist')[-1])" 2>&1)
          SPACY_NUM_BUILD_JOBS=2 python -m pip install dist/$SDIST
        shell: bash

      - name: Test import
        run: python -W error -c "import spacy"

      - name: "Test download CLI"
        run: |
          python -m spacy download ca_core_news_sm
          python -m spacy download ca_core_news_md
          python -c "import spacy; nlp=spacy.load('ca_core_news_sm'); doc=nlp('test')"
        if: matrix.python_version == '3.9'

      - name: "Test download_url in info CLI"
        run: |
          python -W error -m spacy info ca_core_news_sm | grep -q download_url
        if: matrix.python_version == '3.9'

      - name: "Test no warnings on load (#11713)"
        run: |
          python -W error -c "import ca_core_news_sm; nlp = ca_core_news_sm.load(); doc=nlp('test')"
        if: matrix.python_version == '3.9'

      - name: "Test convert CLI"
        run: |
          python -m spacy convert extra/example_data/ner_example_data/ner-token-per-line-conll2003.json .
        if: matrix.python_version == '3.9'

      - name: "Test debug config CLI"
        run: |
          python -m spacy init config -p ner -l ca ner.cfg
          python -m spacy debug config ner.cfg --paths.train ner-token-per-line-conll2003.spacy --paths.dev ner-token-per-line-conll2003.spacy
        if: matrix.python_version == '3.9'

      - name: "Test debug data CLI"
        run: |
          # will have errors due to sparse data, check for summary in output
          python -m spacy debug data ner.cfg --paths.train ner-token-per-line-conll2003.spacy --paths.dev ner-token-per-line-conll2003.spacy | grep -q Summary
        if: matrix.python_version == '3.9'

      - name: "Test train CLI"
        run: |
          python -m spacy train ner.cfg --paths.train ner-token-per-line-conll2003.spacy --paths.dev ner-token-per-line-conll2003.spacy --training.max_steps 10 --gpu-id -1
        if: matrix.python_version == '3.9'

      - name: "Test assemble CLI"
        run: |
          python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_sm'}; config.to_disk('ner_source_sm.cfg')"
          PYTHONWARNINGS="error,ignore::DeprecationWarning" python -m spacy assemble ner_source_sm.cfg output_dir
        if: matrix.python_version == '3.9'

      - name: "Test assemble CLI vectors warning"
        run: |
          python -c "import spacy; config = spacy.util.load_config('ner.cfg'); config['components']['ner'] = {'source': 'ca_core_news_md'}; config.to_disk('ner_source_md.cfg')"
          python -m spacy assemble ner_source_md.cfg output_dir 2>&1 | grep -q W113
        if: matrix.python_version == '3.9'

      - name: "Install test requirements"
        run: |
          python -m pip install -U -r requirements.txt

      - name: "Run CPU tests"
        run: |
          python -m pytest --pyargs spacy -W error
        if: "!(startsWith(matrix.os, 'macos') && matrix.python_version == '3.11')"

      - name: "Run CPU tests with thinc-apple-ops"
        run: |
          python -m pip install 'spacy[apple]'
          python -m pytest --pyargs spacy
        if: startsWith(matrix.os, 'macos') && matrix.python_version == '3.11'


universe validation

name: universe validation

on:
  push:
    branches-ignore:
      - "spacy.io"
      - "nightly.spacy.io"
      - "v2.spacy.io"
    paths:
      - "website/meta/universe.json"
  pull_request:
    types: [opened, synchronize, reopened, edited]
    paths:
      - "website/meta/universe.json"

jobs:
  validate:
    name: Validate
    if: github.repository_owner == 'explosion'
    runs-on: ubuntu-latest
    steps:
      - name: Check out repo
        uses: actions/checkout@v4

      - name: Configure Python version
        uses: actions/setup-python@v4
        with:
          python-version: "3.7"

      - name: Validate website/meta/universe.json
        run: |
          python .github/validate_universe_json.py website/meta/universe.json


spaCy universe project alert

name: spaCy universe project alert

on:
  pull_request_target:
    paths:
      - "website/meta/universe.json"

jobs:
  build:
    if: github.repository_owner == 'explosion'
    runs-on: ubuntu-latest

    steps:
      - name: Dump GitHub context
        env:
          GITHUB_CONTEXT: ${{ toJson(github) }}
          PR_NUMBER: ${{github.event.number}}
        run: |
          echo "$GITHUB_CONTEXT"

      - uses: actions/checkout@v4
      - uses: actions/setup-python@v4
        with:
          python-version: '3.10'
      - name: Install Bernadette app dependency and send an alert
        env:
          SLACK_BOT_TOKEN: ${{ secrets.SLACK_BOT_TOKEN }}
          GITHUB_CONTEXT: ${{ toJson(github) }}
          CHANNEL: "#alerts-universe"
        run: |
          pip install slack-sdk==3.17.2 aiohttp==3.8.1
          echo "$CHANNEL"
          python .github/spacy_universe_alert.py "$GITHUB_CONTEXT"


Backport

name: Backport

on:
  pull_request_target:
    types:
      - closed
      - labeled

jobs:
  backport:
    name: Create backport
    runs-on: ubuntu-latest
    # This workflow only applies to merged PRs; and triggers on a PR being closed, or the backport label being applied.
    # See https://docs.github.com/en/actions/using-workflows/events-that-trigger-workflows#pull_request_target.
    if: >
      github.event.pull_request.merged
      && (
        github.event.action == 'closed'
        ||
          (github.event.action == 'labeled' && contains(github.event.label.name, 'backport/')
        )
      )
    steps:
      - uses: actions/create-github-app-token@7bfa3a4717ef143a604ee0a99d859b8886a96d00 # v1.9.3
        id: app-token
        with:
          app-id: ${{ secrets.POETRY_TOKEN_APP_ID }}
          private-key: ${{ secrets.POETRY_TOKEN_APP_KEY }}
      - uses: tibdex/backport@9565281eda0731b1d20c4025c43339fb0a23812e # v2.0.4
        with:
          github_token: ${{ steps.app-token.outputs.token }}
          title_template: "[<%= base %>] <%= title %>"
          label_pattern: "^backport/(?<base>([^ ]+))$"


Tests

name: Tests

on:
  push:
  pull_request:
  merge_group:

concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.ref }}
  cancel-in-progress: ${{ github.event_name == 'pull_request' }}

defaults:
  run:
    shell: bash

jobs:
  changes:
    name: Detect changed files
    runs-on: ubuntu-latest
    outputs:
      project: ${{ steps.changes.outputs.project }}
      fixtures-pypi: ${{ steps.changes.outputs.fixtures-pypi }}
      src: ${{ steps.changes.outputs.src }}
      tests: ${{ steps.changes.outputs.tests }}
    steps:
      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4

      - uses: dorny/paths-filter@de90cc6fb38fc0963ad72b210f1f284cd68cea36 # v3.0.2
        id: changes
        with:
          filters: |
            workflow: &workflow
              - '.github/actions/**'
              - '.github/workflows/tests.yaml'
              - '.github/workflows/.tests-matrix.yaml'
            project: &project
              - *workflow
              - 'poetry.lock'
              - 'pyproject.toml'
            fixtures-pypi:
              - *workflow
              - 'tests/repositories/fixtures/pypi.org/**'
            src:
              - *project
              - 'src/**/*.py'
            tests:
              - *project
              - 'src/**/*.py'
              - 'tests/**'

  lockfile:
    name: Check poetry.lock
    runs-on: ubuntu-latest
    if: needs.changes.outputs.project == 'true'
    needs: changes
    steps:
      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4

      - uses: ./.github/actions/bootstrap-poetry

      - run: poetry check --lock

  smoke:
    name: Smoke-test build and install
    runs-on: ubuntu-latest
    if: needs.changes.outputs.project == 'true'
    needs: lockfile
    steps:
      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4

      - run: pipx run build

      - run: pipx run twine check --strict dist/*

      - run: pipx install --suffix=@build dist/*.whl

      - uses: ./.github/actions/bootstrap-poetry

      # Smoke test: confirm the version of the installed wheel matches the project.
      - run: poetry@build --version | grep $(poetry version --short)

  fixtures-pypi:
    name: Check fixtures (PyPI)
    runs-on: ubuntu-latest
    if: needs.changes.outputs.fixtures-pypi == 'true'
    needs: changes
    steps:
      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4

      - uses: ./.github/actions/bootstrap-poetry

      - uses: ./.github/actions/poetry-install
        with:
          args: --only main,test

      - run: poetry run env PYTHONPATH="$GITHUB_WORKSPACE" python tests/repositories/fixtures/pypi.org/generate.py

      - run: git diff --exit-code --stat HEAD tests/repositories/fixtures/pypi.org

  tests-matrix:
    # Use this matrix with multiple jobs defined in a reusable workflow:
    uses: ./.github/workflows/.tests-matrix.yaml
    name: "${{ matrix.os.name }} (Python ${{ matrix.python-version }})"
    if: '!failure()'
    needs:
      - lockfile
      - changes
    with:
      runner: ${{ matrix.os.image }}
      python-version: ${{ matrix.python-version }}
      run-mypy: ${{ needs.changes.outputs.tests == 'true' }}
      run-pytest: ${{ needs.changes.outputs.tests == 'true' }}
      run-pytest-export: ${{ needs.changes.outputs.src == 'true' }}
    secrets: inherit
    strategy:
      matrix:
        os:
          - name: Ubuntu
            image: ubuntu-22.04
          - name: macOS x86_64
            image: macos-13
          - name: Windows
            image: windows-2022
          - name: macOS aarch64
            image: macos-14
        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
      fail-fast: false

  status:
    name: Status
    runs-on: ubuntu-latest
    if: always()
    needs:
      - lockfile
      - smoke
      - fixtures-pypi
      - tests-matrix
    steps:
      - run: ${{ (contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')) && 'false' || 'true' }}


Documentation Preview

name: Documentation Preview

on:
  pull_request:
    # allow repository maintainers to modify and test workflow
    paths-ignore:
      - "**"
      - "!.github/workflows/docs.yaml"
  pull_request_target:
    # enable runs for this workflow when labeled as documentation only
    # prevent execution when the workflow itself is modified from a fork
    types:
      - labeled
      - synchronize
    paths-ignore:
      - "**"
      - "!docs/**"

jobs:
  deploy:
    name: Build & Deploy
    runs-on: ubuntu-latest
    if: >
      (github.event_name == 'pull_request_target' && contains(github.event.pull_request.labels.*.name, 'impact/docs'))
      || (github.event_name != 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository)
    permissions:
      contents: read
      pull-requests: write
    steps:
      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
        with:
          repository: python-poetry/website

      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
        with:
          path: poetry
          ref: ${{ github.event.pull_request.head.sha }}

      - uses: actions/setup-node@60edb5dd545a775178f52524783378180af0d1f8 # v4.0.2
        with:
          node-version: "18"

      - uses: peaceiris/actions-hugo@16361eb4acea8698b220b76c0d4e84e1fd22c61d # v2.6.0
        with:
          hugo-version: '0.83.1'

      - uses: ./poetry/.github/actions/bootstrap-poetry

      - uses: ./poetry/.github/actions/poetry-install
        with:
          args: --no-root --only main

      - name: website-build
        run: |
          # Rebuild the docs files from the PR checkout.
          poetry run python bin/website build --local ./poetry
          # Build website assets (CSS/JS).
          npm ci && npm run prod
          # Build the static website.
          hugo -v --minify

      - uses: amondnet/vercel-action@16e87c0a08142b0d0d33b76aeaf20823c381b9b9 # v25.2.0
        with:
          vercel-token: ${{ secrets.VERCEL_TOKEN }}
          github-token: ${{ secrets.GITHUB_TOKEN }}
          vercel-org-id: ${{ secrets.VERCEL_ORG_ID }}
          vercel-project-id: ${{ secrets.VERCEL_PROJECT_ID }}
          scope: python-poetry
          github-comment: true
          working-directory: public


Lock Threads

name: Lock Threads

on:
  schedule:
    - cron: '0 0 * * *' # every day at midnight
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}

jobs:
  lock-issues:
    runs-on: ubuntu-latest
    permissions:
      issues: write
    steps:
      - uses: dessant/lock-threads@1bf7ec25051fe7c00bdd17e6a7cf3d7bfb7dc771 # v5.0.1
        with:
          process-only: issues
          issue-inactive-days: 30
          issue-comment: >
            This issue has been automatically locked since there
            has not been any recent activity after it was closed.
            Please open a new issue for related bugs.

  lock-prs:
    runs-on: ubuntu-latest
    permissions:
      issues: write
      pull-requests: write
    steps:
      - uses: dessant/lock-threads@1bf7ec25051fe7c00bdd17e6a7cf3d7bfb7dc771 # v5.0.1
        with:
          process-only: prs
          pr-inactive-days: 30
          pr-comment: >
            This pull request has been automatically locked since there
            has not been any recent activity after it was closed.
            Please open a new issue for related bugs.


Release

name: Release

on:
  release:
    types: [published]

jobs:
  build:
    name: Build
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4

      - run: pipx run build

      - uses: actions/upload-artifact@65462800fd760344b1a7b4382951275a0abb4808 # v4.3.3
        with:
          name: distfiles
          path: dist/
          if-no-files-found: error

  upload-github:
    name: Upload (GitHub)
    runs-on: ubuntu-latest
    permissions:
      contents: write
    needs: build
    steps:
      # We need to be in a git repo for gh to work.
      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4

      - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
        with:
          name: distfiles
          path: dist/

      - run: gh release upload ${{ github.event.release.tag_name }} dist/*.{tar.gz,whl}
        env:
          GH_TOKEN: ${{ github.token }}

  upload-pypi:
    name: Upload (PyPI)
    runs-on: ubuntu-latest
    environment:
      name: pypi
      url: https://pypi.org/project/poetry/
    permissions:
      id-token: write
    needs: build
    steps:
      - uses: actions/download-artifact@65a9edc5881444af0b9093a5e628f2fe47ea3b2e # v4.1.7
        with:
          name: distfiles
          path: dist/

      - uses: pypa/gh-action-pypi-publish@81e9d935c883d0b210363ab89cf05f3894778450 # v1.8.14
        with:
          print-hash: true


# Reusable workflow consumed by tests.yaml; used to share a single matrix across jobs.
on:
  workflow_call:
    inputs:
      runner:
        required: true
        type: string
      python-version:
        required: true
        type: string
      run-mypy:
        required: true
        type: boolean
      run-pytest:
        required: true
        type: boolean
      run-pytest-export:
        required: true
        type: boolean

defaults:
  run:
    shell: bash

jobs:
  mypy:
    name: mypy
    runs-on: ${{ inputs.runner }}
    if: inputs.run-mypy
    steps:
      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4

      - uses: ./.github/actions/bootstrap-poetry
        id: bootstrap-poetry
        with:
          python-version: ${{ inputs.python-version }}

      - uses: ./.github/actions/poetry-install

      - uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
        with:
          path: .mypy_cache
          key: mypy-${{ runner.os }}-py${{ steps.bootstrap-poetry.outputs.python-version }}-${{ hashFiles('pyproject.toml', 'poetry.lock') }}
          restore-keys: |
            mypy-${{ runner.os }}-py${{ steps.bootstrap-poetry.outputs.python-version }}-
            mypy-${{ runner.os }}-

      - run: poetry run mypy

  pytest:
    name: pytest
    runs-on: ${{ inputs.runner }}
    if: inputs.run-pytest
    steps:
      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4

      - uses: ./.github/actions/bootstrap-poetry
        with:
          python-version: ${{ inputs.python-version }}

      - uses: ./.github/actions/poetry-install
        with:
          args: --with github-actions

      - run: poetry run pytest --integration -v
        env:
          POETRY_TEST_INTEGRATION_GIT_USERNAME: ${{ github.actor }}
          POETRY_TEST_INTEGRATION_GIT_PASSWORD: ${{ github.token }}

      - run: git diff --exit-code --stat HEAD

  pytest-export:
    name: pytest (poetry-plugin-export)
    runs-on: ${{ inputs.runner }}
    if: inputs.run-pytest-export
    steps:
      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4

      - uses: ./.github/actions/bootstrap-poetry
        with:
          python-version: ${{ inputs.python-version }}

      - uses: ./.github/actions/poetry-install
        with:
          args: --with github-actions

      - run: poetry run pip list --format json | jq -r '.[] | "\(.name)=\(.version)"' >> $GITHUB_OUTPUT
        id: package-versions

      - uses: actions/checkout@0ad4b8fadaa221de15dcec353f45205ec38ea70b # v4.1.4
        with:
          path: poetry-plugin-export
          repository: python-poetry/poetry-plugin-export
          ref: refs/tags/${{ steps.package-versions.outputs.poetry-plugin-export }}

      - run: poetry run -C .. pytest -v
        working-directory: ./poetry-plugin-export

      - run: git -C poetry-plugin-export diff --exit-code --stat HEAD


PR Preview

name: PR Preview

on:
  push:
    branches:
      - "develop"
  pull_request:
    types: [opened, synchronize, reopened]

# Avoid duplicate workflows on same branch
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  upload-whl:
    runs-on: ubuntu-latest

    permissions:
      contents: read # This is required for actions/checkout
      statuses: write # This is required for "Set S3 URL as Github status" step

    defaults:
      run:
        shell: bash --login -eo pipefail {0}

    outputs:
      enable-setup: ${{ steps.exports.outputs.enable-setup }}
      preview-branch: ${{ steps.exports.outputs.preview-branch }}
      s3-url: ${{ steps.exports.outputs.s3-url }}

    steps:
      - name: Checkout Streamlit code
        uses: actions/checkout@v4
        with:
          persist-credentials: false
          submodules: "recursive"
          fetch-depth: 2
      - name: Set Python version vars
        uses: ./.github/actions/build_info
      - name: Set up Python ${{ env.PYTHON_MAX_VERSION }}
        uses: actions/setup-python@v5
        with:
          python-version: "${{ env.PYTHON_MAX_VERSION }}"
      - name: Setup virtual env
        uses: ./.github/actions/make_init
      - name: Create Wheel File
        timeout-minutes: 120
        run: |
          sudo apt update
          sudo apt install rsync
          BUILD_AS_FAST_AS_POSSIBLE=1 make package
      - name: Store Whl File
        uses: actions/upload-artifact@v4
        with:
          name: whl_file
          path: lib/dist/*.whl
      # Uses action to safely process user input (branch name) to prevent script injection attacks
      - name: Set Environment Variables
        uses: ./.github/actions/preview_branch
        with:
          pull_request_number: ${{ github.event.pull_request.number }}
          ref_type: ${{ github.ref_type }}
          branch: ${{ github.ref_name }}
      - if: ${{ env.AWS_ACCESS_KEY_ID != '' }}
        name: Upload wheel to S3
        id: exports
        env:
          BRANCH: ${{ env.BRANCH }}
          PREVIEW_BRANCH: ${{ env.PREVIEW_BRANCH }}
          AWS_DEFAULT_REGION: us-west-2
          AWS_ACCESS_KEY_ID: ${{ secrets.CORE_PREVIEWS_S3_KEY_ID }}
          AWS_SECRET_ACCESS_KEY: ${{ secrets.CORE_PREVIEWS_S3_SECRET_KEY }}
        # To create a consistent location for the release/demo whl file, we need a
        # stagnent version number (streamlit-11.11.11)
        run: |
          sudo apt install -y awscli

          cd lib/dist
          export WHEELFILE="$(ls -t *.whl | head -n 1)"

          if [ "${BRANCH}" = "release/demo" ]
          then
            aws s3 cp "${WHEELFILE}" s3://core-previews/${PREVIEW_BRANCH}/streamlit-11.11.11-py2.py3-none-any.whl --acl public-read
            S3_URL="https://core-previews.s3-us-west-2.amazonaws.com/${PREVIEW_BRANCH}/streamlit-11.11.11-py2.py3-none-any.whl"
          else
            aws s3 cp "${WHEELFILE}" s3://core-previews/${PREVIEW_BRANCH}/ --acl public-read
            S3_URL="https://core-previews.s3-us-west-2.amazonaws.com/${PREVIEW_BRANCH}/${WHEELFILE}"
          fi
          echo -e "Wheel file download link: ${S3_URL}"

          cd ../..
          # env variables don't carry over between gh action jobs
          echo "enable-setup=${{ env.AWS_ACCESS_KEY_ID != '' }}" >> $GITHUB_OUTPUT
          echo "preview-branch=$PREVIEW_BRANCH" >> $GITHUB_OUTPUT
          echo "s3-url=${S3_URL}" >> $GITHUB_OUTPUT
      - if: steps.exports.outputs.enable-setup == 'true' && success() && github.repository == 'streamlit/streamlit' && github.event_name == 'pull_request' && github.event.sender.type == 'User'
        name: Set S3 URL as Github Status
        uses: actions/github-script@v7
        env:
          S3_URL: ${{ steps.exports.outputs.s3-url }}
        with:
          script: |
            const { sha } = context.payload.pull_request.head
            // For API documentation, see:
            // https://docs.github.com/en/rest/commits/statuses?apiVersion=2022-11-28#create-a-commit-status
            await github.request(`POST /repos/streamlit/streamlit/statuses/{sha}`, {
              owner: 'streamlit',
              repo: 'streamlit',
              sha,
              state: 'success',
              target_url: process.env.S3_URL,
              context: 'Wheel ready!'
            })

  setup-preview:
    runs-on: ubuntu-latest

    needs: upload-whl
    if: needs.upload-whl.outputs.enable-setup == 'true'

    defaults:
      run:
        shell: bash --login -eo pipefail {0}

    steps:
      - name: Checkout Core Previews Repo
        uses: actions/checkout@v4
        with:
          repository: streamlit/core-previews
          # The default GITHUB_TOKEN is scoped only to the triggering streamlit/streamlit repo.
          # Accessing streamlit/core-previews repo requires a separate auth token.
          token: ${{ secrets.CORE_PREVIEWS_REPO_TOKEN }}
          # Save the access token to the local git config, so
          # later git commands can work.
          persist-credentials: true
      - name: Setup preview repo
        env:
          PREVIEW_BRANCH: ${{ needs.upload-whl.outputs.preview-branch }}
          S3_URL: ${{ needs.upload-whl.outputs.s3-url }}
        run: |
          git config --global user.email "core+streamlitbot-github@streamlit.io"
          git config --global user.name "Streamlit Bot"
          git branch -D "$PREVIEW_BRANCH" &>/dev/null || true
          git checkout -b "$PREVIEW_BRANCH"

          echo "$S3_URL" >> requirements.txt

          git add .
          git commit -m "Prepare core preview: ${PREVIEW_BRANCH}"
          git push -f origin ${PREVIEW_BRANCH}
      - name: Ready to deploy!
        env:
          PREVIEW_BRANCH: ${{ needs.upload-whl.outputs.preview-branch }}
        run: |
          echo -e "https://share.streamlit.io/deploy?repository=streamlit/core-previews&branch=${PREVIEW_BRANCH}&mainModule=E2E_Tester_🧪.py"


Cypress

name: Cypress

on:
  push:
    branches:
      - "develop"
  pull_request:
    types: [opened, synchronize, reopened]
  # Allows workflow to be called from other workflows
  workflow_call:
    inputs:
      ref:
        required: true
        type: string

# Avoid duplicate workflows on same branch
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-cypress
  cancel-in-progress: true

jobs:
  cypress-e2e-tests:
    runs-on: ubuntu-latest-8-cores

    defaults:
      run:
        shell: bash --login -eo pipefail {0}

    strategy:
      fail-fast: false
      matrix:
        # In case of number of workers would change, update TOTAL_WORKERS env variable in "Cypress Tests" step.
        # Be aware that specs should be sequential integers starting from 1 without gaps
        # based on logic in "Cypress Tests" step.
        # prettier-ignore
        specs: [ 1, 2, ]

    steps:
      - name: Checkout Streamlit code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref }}
          persist-credentials: false
          submodules: "recursive"
          fetch-depth: 2
      - name: Set Python version vars
        uses: ./.github/actions/build_info
      - name: Set up Python ${{ env.PYTHON_MAX_VERSION }}
        uses: actions/setup-python@v5
        with:
          python-version: "${{ env.PYTHON_MAX_VERSION }}"
      - name: Setup virtual env
        uses: ./.github/actions/make_init
      - name: Install Cypress dependencies
        run: |
          sudo apt install -y xvfb libgtk2.0-0 libnotify-dev libgconf-2-4 libnss3 libxss1 libasound2 curl
      - name: Run make develop
        run: make develop
      - name: Run make protobuf
        run: make protobuf
      - name: Run make frontend-lib
        run: make frontend-lib
      - name: Register Streamlit User & Mapbox Token
        run: |
          if [ ! -d $HOME/.streamlit ] ; then
            mkdir $HOME/.streamlit
          fi
          echo '[mapbox]' >  ~/.streamlit/config.toml
          MAPBOX_TOKEN=$(curl -sS https://data.streamlit.io/tokens.json | jq -r '.["mapbox-localhost"]') \
            && echo 'token = "'$MAPBOX_TOKEN'"' >> ~/.streamlit/config.toml

          echo '[general]' >  ~/.streamlit/credentials.toml
          echo 'email = "test@streamlit.io"' >> ~/.streamlit/credentials.toml
      - name: Cypress Tests
        env:
          CURRENT_RUN: ${{matrix.specs}}
          CYPRESS_VERIFY_TIMEOUT: 90000
          TOTAL_WORKERS: 2
        run: |
          js_specs=(e2e/specs/*)

          # get how many specs to be split up into $TOTAL_WORKERS runs
          totalTests=$(( ${#js_specs[@]} ))
          testsPerWorker=$(( totalTests / TOTAL_WORKERS ))
          # get the remainder of tests to be split up into the first $remainder workers
          remainder=$(( totalTests % TOTAL_WORKERS ))

          # We handle $CURRENT_RUN = 1 separately because bash interprets 0 as empty string in this case
          if [ "$CURRENT_RUN" -eq 1 ]; then
            startIndex=0
            currentIntervalLength=$(( testsPerWorker + (remainder > 0 ? 1 : 0) ))
          elif [ $CURRENT_RUN -le $remainder ]; then
            startIndex=$(( (CURRENT_RUN - 1) * (testsPerWorker + 1) ))
            currentIntervalLength=$(( testsPerWorker + 1 ))
          else
            startIndex=$(( remainder * (testsPerWorker + 1) + (CURRENT_RUN - remainder - 1) * testsPerWorker ))
            currentIntervalLength=$(( testsPerWorker ))
          fi

          # ensure no blank specs list (which causes every spec to run)
          if [[ "${js_specs[@]:${startIndex}:${currentIntervalLength}}" != "" ]]; then
            echo "Specs tested in this job:"
            echo "${js_specs[@]:${startIndex}:${currentIntervalLength}}"
            scripts/run_e2e_tests.py -a "${js_specs[@]:${startIndex}:${currentIntervalLength}}"
          fi
      - name: Check that all screenshot have been committed
        run: |
          set -x;

          UNTRACKED_FILE_COUNT=$(git ls-files --others --exclude-standard | grep cypress | wc -l | bc -l || echo '0')
          echo "Untracked files count: ${UNTRACKED_FILE_COUNT}"
          if [[ "${UNTRACKED_FILE_COUNT}" -gt 0 ]]; then
            echo "Untracked files:";
            git ls-files --others --exclude-standard | grep cypress;
            exit 1;
          fi
      - name: Store Videos
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: cypress_videos
          path: frontend/cypress/videos
      - name: Store Snapshots
        uses: actions/upload-artifact@v3
        if: always()
        with:
          name: cypress_snapshots
          path: frontend/cypress/snapshots

  cypress-summary:
    runs-on: ubuntu-latest

    # Consolidates cypress test matrix results for a single pass/fail status check
    if: ${{ always() }}
    needs: cypress-e2e-tests

    defaults:
      run:
        shell: bash --login -eo pipefail {0}

    steps:
      - name: Check Cypress test matrix status
        if: ${{ needs.test.result == 'failure' }}
        run: exit 1


Javascript Unit Tests

name: Javascript Unit Tests

on:
  push:
    branches:
      - "develop"
  pull_request:
    types: [opened, synchronize, reopened]
  # Allows workflow to be called from other workflows
  workflow_call:
    inputs:
      ref:
        required: true
        type: string

# Avoid duplicate workflows on same branch
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}-javascript
  cancel-in-progress: true

jobs:
  js-unit-tests:
    runs-on: ubuntu-latest

    defaults:
      run:
        shell: bash --login -eo pipefail {0}

    steps:
      - name: Checkout Streamlit code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref }}
          persist-credentials: false
          submodules: "recursive"
          fetch-depth: 2
      - name: Set Python version vars
        uses: ./.github/actions/build_info
      - name: Set up Python ${{ env.PYTHON_MAX_VERSION }}
        uses: actions/setup-python@v5
        with:
          python-version: "${{ env.PYTHON_MAX_VERSION }}"
      - name: Setup virtual env
        uses: ./.github/actions/make_init
      - name: Run make develop
        run: make develop
      - name: Run make protobuf
        run: make protobuf
      - name: Run make frontend-lib
        run: make frontend-lib
      - name: Audit frontend licenses
        run: ./scripts/audit_frontend_licenses.py
      - name: Run linters
        run: |
          # Run eslint as a standalone command to generate the test report.
          # We need to --hook-stage manual to trigger Typecheck too
          PRE_COMMIT_NO_CONCURRENCY=true SKIP=eslint,ruff,ruff-format pre-commit run --hook-stage manual --show-diff-on-failure --color=always --all-files
          # Run eslint using Makefile omitting the pre-commit
          make jslint
      - name: Validate NOTICES
        run: |
          # Run `make notices`. If it results in changes, warn the user and fail.
          make notices

          git_status=$(git status --porcelain -- NOTICES)
          if [[ -n $git_status ]]; then
            echo "::error::The NOTICES file is out of date! Please run \`make notices\` and commit the result."
            echo "::group::git diff NOTICES"
            git diff NOTICES
            echo "::endgroup::"
            exit 1
          else
            echo "NOTICES is up to date."
          fi
      - name: Run frontend tests
        run: make jstest

  components-lib-tests:
    runs-on: ubuntu-latest

    defaults:
      run:
        shell: bash --login -eo pipefail {0}

    steps:
      - name: Checkout Streamlit code
        uses: actions/checkout@v4
        with:
          ref: ${{ inputs.ref }}
          persist-credentials: false
          submodules: "recursive"
          fetch-depth: 2
      - name: Setup Node
        uses: actions/setup-node@v4
        with:
          node-version: "20.x"
          cache: "yarn"
          cache-dependency-path: "component-lib/yarn.lock"
      - name: Install node dependencies
        working-directory: component-lib
        run: yarn install
      - name: Run frontend tests
        working-directory: component-lib
        run: yarn test
      - name: Build package
        working-directory: component-lib
        run: yarn build


CLI Regression

name: CLI Regression

on:
  push:
    branches:
      - "develop"
  pull_request:
    types: [opened, synchronize, reopened]

# Avoid duplicate workflows on same branch
concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  cli-regression:
    runs-on: ubuntu-latest

    defaults:
      run:
        shell: bash --login -eo pipefail {0}

    steps:
      - name: Checkout Streamlit code
        uses: actions/checkout@v4
        with:
          persist-credentials: false
          submodules: "recursive"
          fetch-depth: 2
      - name: Set Python version vars
        uses: ./.github/actions/build_info
      - name: Set up Python ${{ env.PYTHON_MAX_VERSION }}
        uses: actions/setup-python@v5
        with:
          python-version: "${{ env.PYTHON_MAX_VERSION }}"
      - name: Setup virtual env
        uses: ./.github/actions/make_init
      - name: Build Package - Fast
        timeout-minutes: 120
        run: |
          sudo apt install rsync
          BUILD_AS_FAST_AS_POSSIBLE=1 make package
      - name: Run CLI regression tests
        run: |
          export SKIP_VERSION_CHECK=true; make cli-regression-tests