Build and Push Web Image on Tag

name: Build and Push Web Image on Tag

on:
  push:
    tags:
      - '*'

env:
  REGISTRY_IMAGE: danswer/danswer-web-server

jobs:
  build:
    runs-on:
      group: ${{ matrix.platform == 'linux/amd64' && 'amd64-image-builders' || 'arm64-image-builders' }}
    strategy:
      fail-fast: false
      matrix:
        platform:
          - linux/amd64
          - linux/arm64

    steps:
      - name: Prepare
        run: |
          platform=${{ matrix.platform }}
          echo "PLATFORM_PAIR=${platform//\//-}" >> $GITHUB_ENV

      - name: Checkout
        uses: actions/checkout@v4

      - name: Docker meta
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.REGISTRY_IMAGE }}
          tags: |
            type=raw,value=danswer/danswer-web-server:${{ github.ref_name }}
            type=raw,value=danswer/danswer-web-server:latest

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_TOKEN }}

      - name: Build and push by digest
        id: build
        uses: docker/build-push-action@v5
        with:
          context: ./web
          file: ./web/Dockerfile
          platforms: ${{ matrix.platform }}
          push: true
          build-args: |
            DANSWER_VERSION=${{ github.ref_name }}
          # needed due to weird interactions with the builds for different platforms
          no-cache: true
          labels: ${{ steps.meta.outputs.labels }}
          outputs: type=image,name=${{ env.REGISTRY_IMAGE }},push-by-digest=true,name-canonical=true,push=true

      - name: Export digest
        run: |
          mkdir -p /tmp/digests
          digest="${{ steps.build.outputs.digest }}"
          touch "/tmp/digests/${digest#sha256:}"

      - name: Upload digest
        uses: actions/upload-artifact@v4
        with:
          name: digests-${{ env.PLATFORM_PAIR }}
          path: /tmp/digests/*
          if-no-files-found: error
          retention-days: 1

  merge:
    runs-on: ubuntu-latest
    needs:
      - build
    steps:
      - name: Download digests
        uses: actions/download-artifact@v4
        with:
          path: /tmp/digests
          pattern: digests-*
          merge-multiple: true

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Docker meta
        id: meta
        uses: docker/metadata-action@v5
        with:
          images: ${{ env.REGISTRY_IMAGE }}

      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKER_USERNAME }}
          password: ${{ secrets.DOCKER_TOKEN }}

      - name: Create manifest list and push
        working-directory: /tmp/digests
        run: |
          docker buildx imagetools create $(jq -cr '.tags | map("-t " + .) | join(" ")' <<< "$DOCKER_METADATA_OUTPUT_JSON") \
            $(printf '${{ env.REGISTRY_IMAGE }}@sha256:%s ' *)

      - name: Inspect image
        run: |
          docker buildx imagetools inspect ${{ env.REGISTRY_IMAGE }}:${{ steps.meta.outputs.version }}

      - name: Run Trivy vulnerability scanner
        uses: aquasecurity/trivy-action@master
        with:
          image-ref: docker.io/${{ env.REGISTRY_IMAGE }}:${{ github.ref_name }}
          severity: 'CRITICAL,HIGH'


Tag Latest Version

name: Tag Latest Version

on:
  workflow_dispatch:
    inputs:
      version:
        description: 'The version (ie v0.0.1) to tag as latest'
        required: true

jobs:
  tag:
    runs-on: ubuntu-latest
    steps:
    - name: Set up Docker Buildx
      uses: docker/setup-buildx-action@v1

    - name: Login to Docker Hub
      uses: docker/login-action@v1
      with:
        username: ${{ secrets.DOCKER_USERNAME }}
        password: ${{ secrets.DOCKER_TOKEN }}

    - name: Enable Docker CLI experimental features
      run: echo "DOCKER_CLI_EXPERIMENTAL=enabled" >> $GITHUB_ENV

    - name: Pull, Tag and Push Web Server Image
      run: |
        docker buildx imagetools create -t danswer/danswer-web-server:latest danswer/danswer-web-server:${{ github.event.inputs.version }}

    - name: Pull, Tag and Push API Server Image
      run: |
        docker buildx imagetools create -t danswer/danswer-backend:latest danswer/danswer-backend:${{ github.event.inputs.version }}


Python Checks

name: Python Checks

on:
  pull_request:
    branches: [ main ]

jobs:
  mypy-check:
    runs-on: ubuntu-latest

    steps:
    - name: Checkout code
      uses: actions/checkout@v3

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.11'
        cache: 'pip'
        cache-dependency-path: |
          backend/requirements/default.txt
          backend/requirements/dev.txt
          backend/requirements/model_server.txt
    - run: |
        python -m pip install --upgrade pip
        pip install -r backend/requirements/default.txt
        pip install -r backend/requirements/dev.txt
        pip install -r backend/requirements/model_server.txt

    - name: Run MyPy
      run: |
        cd backend
        mypy .

    - name: Run ruff
      run: |
        cd backend
        ruff .

    - name: Check import order with reorder-python-imports
      run: |
        cd backend
        find ./danswer -name "*.py" | xargs reorder-python-imports --py311-plus

    - name: Check code formatting with Black
      run: |
        cd backend
        black --check .


Python Unit Tests

name: Python Unit Tests

on:
  pull_request:
    branches: [ main ]

jobs:
  backend-check:
    runs-on: ubuntu-latest

    env:
      PYTHONPATH: ./backend

    steps:
    - name: Checkout code
      uses: actions/checkout@v4

    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.11'
        cache: 'pip'
        cache-dependency-path: |
          backend/requirements/default.txt
          backend/requirements/dev.txt

    - name: Install Dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r backend/requirements/default.txt
        pip install -r backend/requirements/dev.txt

    - name: Run Tests
      shell: script -q -e -c "bash --noprofile --norc -eo pipefail {0}"
      run: py.test -o junit_family=xunit2 -xv --ff backend/tests/unit


Quality Checks PR

name: Quality Checks PR
concurrency:
  group: Quality-Checks-PR-${{ github.head_ref }}
  cancel-in-progress: true

on:
  pull_request: null

jobs:
  quality-checks:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
    - uses: actions/setup-python@v5
      with:
        python-version: '3.11'
    - uses: pre-commit/action@v3.0.0
      with:
        extra_args: --from-ref ${{ github.event.pull_request.base.sha }} --to-ref ${{ github.event.pull_request.head.sha }}


Typos

---
# yamllint disable rule:line-length
name: Typos

on:  # yamllint disable-line rule:truthy
  push:
  pull_request:
    types:
      - opened
      - synchronize
      - reopened

jobs:
  build:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: typos-action
        uses: crate-ci/typos@v1.19.0


Build Release

name: Build Release

on: workflow_dispatch

permissions:
  contents: write

jobs:
  build_wheels:
    name: Build wheels on ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [ubuntu-20.04, windows-2019, macos-11]

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: "recursive"

      # Used to host cibuildwheel
      - uses: actions/setup-python@v5
        with:
          python-version: "3.8"

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install -e .[all]

      - name: Build wheels
        uses: pypa/cibuildwheel@v2.19.1
        env:
          # disable repair
          CIBW_REPAIR_WHEEL_COMMAND: ""
        with:
          package-dir: .
          output-dir: wheelhouse

      - uses: actions/upload-artifact@v4
        with:
          name: wheels-${{ matrix.os }}
          path: ./wheelhouse/*.whl

  build_wheels_arm64:
    name: Build arm64 wheels
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: "recursive"

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
        with:
          platforms: linux/arm64

      - name: Build wheels
        uses: pypa/cibuildwheel@v2.19.1
        env:
          CIBW_SKIP: "*musllinux* pp*"
          CIBW_REPAIR_WHEEL_COMMAND: ""
          CIBW_ARCHS: "aarch64"
          CIBW_BUILD: "cp38-* cp39-* cp310-* cp311-* cp312-*"
        with:
          output-dir: wheelhouse

      - name: Upload wheels as artifacts
        uses: actions/upload-artifact@v4
        with:
          name: wheels_arm64
          path: ./wheelhouse/*.whl

  build_sdist:
    name: Build source distribution
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: "recursive"
      - uses: actions/setup-python@v5
        with:
          python-version: "3.8"
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip build
          python -m pip install -e .[all]
      - name: Build source distribution
        run: |
          python -m build --sdist
      - uses: actions/upload-artifact@v4
        with:
          name: sdist
          path: ./dist/*.tar.gz

  release:
    name: Release
    needs: [build_wheels, build_wheels_arm64, build_sdist]
    runs-on: ubuntu-latest

    steps:
      - uses: actions/download-artifact@v4
        with:
          merge-multiple: true
          path: dist

      - uses: softprops/action-gh-release@v2
        with:
          files: dist/*
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


Build Docker

name: Build Docker

on: workflow_dispatch

permissions:
  contents: write
  packages: write

jobs:
  docker:
    name: Build and push Docker image
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          submodules: "recursive"

      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3

      - name: Login to GitHub Container Registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.repository_owner }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and push
        id: docker_build
        uses: docker/build-push-action@v6
        with:
          context: .
          file: "docker/simple/Dockerfile"
          push: ${{ startsWith(github.ref, 'refs/tags/') }}
          pull: true
          platforms: linux/amd64,linux/arm64
          tags: |
            ghcr.io/abetlen/llama-cpp-python:latest
            ghcr.io/abetlen/llama-cpp-python:${{ github.ref_name }}
          build-args: |
            BUILDKIT_INLINE_CACHE=1

      - name: Publish to GitHub Tag
        if: steps.docker_build.outputs.digest && startsWith(github.ref, 'refs/tags/')
        run: |
          echo "Docker image published for tag: ${{ github.ref_name }}"


Build Wheels (CUDA)

name: Build Wheels (CUDA)

on: workflow_dispatch

permissions:
  contents: write

jobs:
  define_matrix:
    name: Define Build Matrix
    runs-on: ubuntu-latest
    outputs:
      matrix: ${{ steps.set-matrix.outputs.matrix }}
    defaults:
      run:
        shell: pwsh

    steps:
      - name: Define Job Output
        id: set-matrix
        run: |
          $matrix = @{
              'os' = @('ubuntu-latest', 'windows-latest')
              'pyver' = @("3.9", "3.10", "3.11", "3.12")
              'cuda' = @("12.1.1", "12.2.2", "12.3.2", "12.4.1")
              'releasetag' = @("basic")
          }

          $matrixOut = ConvertTo-Json $matrix -Compress
          Write-Output ('matrix=' + $matrixOut) >> $env:GITHUB_OUTPUT

  build_wheels:
    name: Build Wheel ${{ matrix.os }} ${{ matrix.pyver }} ${{ matrix.cuda }} ${{ matrix.releasetag == 'wheels' && 'AVX2' || matrix.releasetag }}
    needs: define_matrix
    runs-on: ${{ matrix.os }}
    strategy:
      matrix: ${{ fromJSON(needs.define_matrix.outputs.matrix) }}
    defaults:
      run:
        shell: pwsh
    env:
      CUDAVER: ${{ matrix.cuda }}
      AVXVER: ${{ matrix.releasetag }}

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: "recursive"

      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.pyver }}
          cache: 'pip'

      - name: Setup Mamba
        uses: conda-incubator/setup-miniconda@v3.0.4
        with:
          activate-environment: "build"
          python-version: ${{ matrix.pyver }}
          miniforge-variant: Mambaforge
          miniforge-version: latest
          use-mamba: true
          add-pip-as-python-dependency: true
          auto-activate-base: false

      - name: VS Integration Cache
        id: vs-integration-cache
        if: runner.os == 'Windows'
        uses: actions/cache@v4.0.2
        with:
          path: ./MSBuildExtensions
          key: cuda-${{ matrix.cuda }}-vs-integration

      - name: Get Visual Studio Integration
        if: runner.os == 'Windows' && steps.vs-integration-cache.outputs.cache-hit != 'true'
        run: |
          if ($env:CUDAVER -eq '12.1.1') {$x = '12.1.0'} else {$x = $env:CUDAVER}
          $links = (Invoke-RestMethod 'https://raw.githubusercontent.com/Jimver/cuda-toolkit/master/src/links/windows-links.ts').Trim().split().where({$_ -ne ''})
          for ($i=$q=0;$i -lt $links.count -and $q -lt 2;$i++) {if ($links[$i] -eq "'$x',") {$q++}}
          Invoke-RestMethod $links[$i].Trim("'") -OutFile 'cudainstaller.zip'
          & 'C:\Program Files\7-Zip\7z.exe' e cudainstaller.zip -oMSBuildExtensions -r *\MSBuildExtensions\* > $null
          Remove-Item 'cudainstaller.zip'

      - name: Install Visual Studio Integration
        if: runner.os == 'Windows'
        run: |
          $y = (gi '.\MSBuildExtensions').fullname + '\*'
          (gi 'C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Microsoft\VC\*\BuildCustomizations').fullname.foreach({cp $y $_})
          $cupath = 'CUDA_PATH_V' + $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.','_')
          echo "$cupath=$env:CONDA_PREFIX" >> $env:GITHUB_ENV

      - name: Install Dependencies
        env:
          MAMBA_DOWNLOAD_FAILFAST: "0"
          MAMBA_NO_LOW_SPEED_LIMIT: "1"
        run: |
          $cudaVersion = $env:CUDAVER
          mamba install -y 'cuda' -c nvidia/label/cuda-$cudaVersion
          python -m pip install build wheel

      - name: Build Wheel
        run: |
          $cudaVersion = $env:CUDAVER.Remove($env:CUDAVER.LastIndexOf('.')).Replace('.','')
          $env:CUDA_PATH = $env:CONDA_PREFIX
          $env:CUDA_HOME = $env:CONDA_PREFIX
          $env:CUDA_TOOLKIT_ROOT_DIR = $env:CONDA_PREFIX
          if ($IsLinux) {
            $env:LD_LIBRARY_PATH = $env:CONDA_PREFIX + '/lib:' + $env:LD_LIBRARY_PATH
          }
          $env:VERBOSE = '1'
          $env:CMAKE_ARGS = '-DLLAMA_CUBLAS=on -DCMAKE_CUDA_ARCHITECTURES=all'
          $env:CMAKE_ARGS = "-DLLAMA_CUDA_FORCE_MMQ=ON $env:CMAKE_ARGS"
          # if ($env:AVXVER -eq 'AVX') {
          $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DLLAMA_AVX2=off -DLLAMA_FMA=off -DLLAMA_F16C=off'
          # }
          # if ($env:AVXVER -eq 'AVX512') {
          #  $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DLLAMA_AVX512=on'
          # }
          # if ($env:AVXVER -eq 'basic') {
          #  $env:CMAKE_ARGS = $env:CMAKE_ARGS + ' -DLLAMA_AVX=off -DLLAMA_AVX2=off -DLLAMA_FMA=off -DLLAMA_F16C=off'
          # }
          python -m build --wheel
          # write the build tag to the output
          Write-Output "CUDA_VERSION=$cudaVersion" >> $env:GITHUB_ENV

      - uses: softprops/action-gh-release@v2
        with:
          files: dist/*
          # Set tag_name to <tag>-cu<cuda_version>
          tag_name: ${{ github.ref_name }}-cu${{ env.CUDA_VERSION }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


Build Wheels (Metal)

name: Build Wheels (Metal)

on: workflow_dispatch

permissions:
  contents: write

jobs:
  build_wheels:
    name: Build wheels on ${{ matrix.os }}
    runs-on: ${{ matrix.os }}
    strategy:
      matrix:
        os: [macos-12, macos-13, macos-14]

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: "recursive"

      # Used to host cibuildwheel
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
          cache: 'pip'

      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install -e .[all]

      - name: Build wheels
        uses: pypa/cibuildwheel@v2.19.1
        env:
          # disable repair
          CIBW_REPAIR_WHEEL_COMMAND: ""
          CIBW_ARCHS: "arm64"
          CIBW_ENVIRONMENT: CMAKE_ARGS="-DCMAKE_OSX_ARCHITECTURES=arm64 -DCMAKE_APPLE_SILICON_PROCESSOR=arm64 -DLLAMA_METAL=on"
          CIBW_BUILD: "cp39-* cp310-* cp311-* cp312-*"
        with:
          package-dir: .
          output-dir: wheelhouse2

      - uses: actions/upload-artifact@v4
        with:
          name: wheels-mac_${{ matrix.os }}
          path: ./wheelhouse2/*.whl

  release:
    name: Release
    needs: [build_wheels]
    runs-on: ubuntu-latest

    steps:
      - uses: actions/download-artifact@v4
        with:
          merge-multiple: true
          path: dist2

      - uses: softprops/action-gh-release@v2
        with:
          files: dist2/*
          # set release name to <tag>-metal
          tag_name: ${{ github.ref_name }}-metal
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}


Wheels Index

name: Wheels Index

on:
  # Trigger on any new release
  release:
    types: [published]

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
permissions:
  contents: read
  pages: write
  id-token: write

# Allow only one concurrent deployment, skipping runs queued between the run in-progress and latest queued.
# However, do NOT cancel in-progress runs as we want to allow these production deployments to complete.
concurrency:
  group: "pages"
  cancel-in-progress: false

jobs:
  # Single deploy job since we're just deploying
  deploy:
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Setup Pages
        uses: actions/configure-pages@v5
      - name: Build
        run: |
          ./scripts/releases-to-pep-503.sh index/whl/cpu '^[v]?[0-9]+\.[0-9]+\.[0-9]+$'
          ./scripts/releases-to-pep-503.sh index/whl/cu121 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu121$'
          ./scripts/releases-to-pep-503.sh index/whl/cu122 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu122$'
          ./scripts/releases-to-pep-503.sh index/whl/cu123 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu123$'
          ./scripts/releases-to-pep-503.sh index/whl/cu124 '^[v]?[0-9]+\.[0-9]+\.[0-9]+-cu124$'
          ./scripts/releases-to-pep-503.sh index/whl/metal '^[v]?[0-9]+\.[0-9]+\.[0-9]+-metal$'
      - name: Upload artifact
        uses: actions/upload-pages-artifact@v3
        with:
          # Upload entire repository
          path: 'index'
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v4


Publish to TestPyPI

# Based on: https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/

name: Publish to TestPyPI

on:
  workflow_dispatch:
    inputs:
      dev_version:
        description: 'Dev version N'
        required: true

jobs:
  build-n-publish:
    name: Build and publish
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4
      with:
        submodules: "recursive"
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: "3.11"
        cache: 'pip'
    - name: Append Dev Version to __version__
      run: |
        DEV_VERSION=${{ github.event.inputs.dev_version }}
        CURRENT_VERSION=$(awk -F= '/__version__ =/ {print $2}' llama_cpp/__init__.py | tr -d ' "')
        NEW_VERSION="${CURRENT_VERSION}.dev${DEV_VERSION}"
        sed -i 's/__version__ = \".*\"/__version__ = \"'"${NEW_VERSION}"'\"/' llama_cpp/__init__.py
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip build
        python -m pip install -e .[all]
    - name: Build source distribution
      run: |
        python -m build --sdist
    - name: Publish to Test PyPI
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        password: ${{ secrets.TEST_PYPI_API_TOKEN }}
        repository-url: https://test.pypi.org/legacy/


Publish to PyPI

name: Publish to PyPI

# Based on: https://packaging.python.org/en/latest/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/

on: workflow_dispatch

jobs:
  build-n-publish:
    name: Build and publish
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4
      with:
        submodules: "recursive"
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: "3.9"
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip build
        python -m pip install -e .[all]
    - name: Build source distribution
      run: |
        python -m build --sdist
    - name: Publish distribution to PyPI
      # TODO: move to tag based releases
      # if: startsWith(github.ref, 'refs/tags')
      uses: pypa/gh-action-pypi-publish@release/v1
      with:
        password: ${{ secrets.PYPI_API_TOKEN }}


Tests for PyPI package

name: Tests for PyPI package

on: workflow_dispatch

jobs:
  build-linux:

    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]

    steps:
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install --verbose llama-cpp-python[all]
      - name: Test with pytest
        run: |
          python -c "import llama_cpp"

  build-windows:

    runs-on: windows-latest
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]

    steps:
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install --verbose llama-cpp-python[all]
      - name: Test with pytest
        run: |
          python -c "import llama_cpp"

  build-macos:

    runs-on: macos-latest
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]

    steps:
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install --verbose llama-cpp-python[all]
      - name: Test with pytest
        run: |
          python -c "import llama_cpp"


Tests

name: Tests

on:
  pull_request:
    branches:
      - main
  push:
    branches:
      - main

jobs:
  build-linux:

    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: "recursive"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install .[all] -v
      - name: Test with pytest
        run: |
          python -m pytest

  build-windows:

    runs-on: windows-latest
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: "recursive"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install .[all] -v
      - name: Test with pytest
        run: |
          python -m pytest

  build-macos:

    runs-on: macos-latest
    strategy:
      matrix:
        python-version: ["3.9", "3.10", "3.11", "3.12"]

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: "recursive"
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: 'pip'
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install .[all] --verbose
      - name: Test with pytest
        run: |
          python -m pytest

  # build-linux-opencl:

  #   runs-on: ubuntu-latest

  #   steps:
  #     - uses: actions/checkout@v4
  #       with:
  #         submodules: "recursive"
  #     - name: Set up Python 3.8
  #       uses: actions/setup-python@v5
  #       with:
  #         python-version: "3.8"
  #     - name: Set up OpenCL & CLBlast
  #       run: |
  #         wget -O- https://apt.repos.intel.com/intel-gpg-keys/GPG-PUB-KEY-INTEL-SW-PRODUCTS.PUB | gpg --dearmor | sudo tee /usr/share/keyrings/oneapi-archive-keyring.gpg > /dev/null
  #         echo "deb [signed-by=/usr/share/keyrings/oneapi-archive-keyring.gpg] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
  #         sudo apt-get update
  #         sudo apt-get install -y --no-install-recommends llvm intel-oneapi-runtime-opencl intel-oneapi-runtime-compilers libclblast-dev
  #     - name: Install dependencies
  #       run: |
  #         python -m pip install --upgrade pip
  #         CMAKE_ARGS="-DLLAMA_CLBLAST=on" python -m pip install .[all] --verbose
  #     - name: Test with pytest
  #       run: |
  #         python -m pytest

  build-macos-metal:

    runs-on: macos-latest

    steps:
      - uses: actions/checkout@v4
        with:
          submodules: "recursive"
      - name: Set up Python 3.9
        uses: actions/setup-python@v5
        with:
          python-version: "3.9"
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          CMAKE_ARGS="-DLLAMA_METAL=on" python -m pip install .[all] --verbose
      - name: Test with pytest
        run: |
          python -m pytest


Add A TestFilght Link

name: Add A TestFilght Link

on:
  workflow_dispatch:
    inputs:
      app_name:
        description: '应用名称（建议填上，程序可能无法通过链接判断 TestFlight 测试的应用名称）【中间不要有英文双引号或英文竖线分隔符"|"】'
        required: false
        default: 'None'
      testflight_link:
        description: 'Testflight 公共测试的完整链接(非 TestFilght 链接请添加到 scripts/data/signup.md)'
        required: true
      table:
        description: '链接分类(可选值：`macos`, `ios`, `ios_game`, `chinese`)'
        required: true

jobs:
  main:
    runs-on: ubuntu-latest
    if: github.repository == 'pluwen/awesome-testflight-link'
    steps:
      - uses: actions/checkout@v3.3.0
      - name: Set up Python
        uses: actions/setup-python@v4.5.0
        with:
          python-version: '3.11.1'
          architecture: x64
      - name: Init Python Env
        run: pip install -r scripts/requirements.txt
      - name: Add A TestFilght Link
        run: python scripts/add_link.py "${{ github.event.inputs.testflight_link }}" "${{ github.event.inputs.table }}" "${{ github.event.inputs.app_name }}"

      - name: Git commit && push
        run: |
          git_diff=`git diff`
          if [ -z "$git_diff" ]; then echo "Nothing Changed";exit; fi
          git config --global user.email "github_bot@noreply.github.com"
          git config --global user.name "github_bot"
          git add .
          git commit -m "Add A New Link"
          git push


Del A TestFilght Link

name: Del A TestFilght Link

on:
  workflow_dispatch:
    inputs:
      testflight_link:
        description: 'Testflight 公共测试的完整链接'
        required: true
      table:
        description: '链接分类(可选值：`macos`, `ios`, `ios_game`, `chinese`)'
        required: true

jobs:
  main:
    runs-on: ubuntu-latest
    if: github.repository == 'pluwen/awesome-testflight-link'
    steps:
      - uses: actions/checkout@v3.3.0
      - name: Set up Python
        uses: actions/setup-python@v4.5.0
        with:
          python-version: '3.11.1'
          architecture: x64
      - name: Del A TestFilght Link
        run: python scripts/del_link.py "${{ github.event.inputs.testflight_link }}" "${{ github.event.inputs.table }}"

      - name: Git commit && push
        run: |
          git_diff=`git diff`
          if [ -z "$git_diff" ]; then echo "Nothing Changed";exit; fi
          git config --global user.email "github_bot@noreply.github.com"
          git config --global user.name "github_bot"
          git add .
          git commit -m "Del A TestFilght Link"
          git push


Order TestFilght Link Status

name: Order TestFilght Link Status

on:
  push:
    branches:
      - main
  workflow_dispatch:
    inputs:
      python_version:
        description: '指定 Python 版本 3.6+ （不建议修改）'
        required: false
        default: '3.11.1'

jobs:
  main:
    runs-on: ubuntu-latest
    env:
      python_version: ${{ github.event.inputs.python_version }}
    steps:
      - uses: actions/checkout@v3.3.0
      - name: Check env
        run: if [ -z "$python_version" ]; then python_version='3.11.1'; fi
      - name: Set up Python
        uses: actions/setup-python@v4.5.0
        with:
          python-version: ${{ env.python_version }}
          architecture: x64

      - name: Checkout main -> status_ordered
        run: |
          git config --global user.email "github_bot@noreply.github.com"
          git config --global user.name "github_bot"
          git pull --unshallow
          git checkout status_ordered
          git checkout origin/main db/sqlite3.db

      - name: Order TestFilght Link Status
        run: python scripts/order_status.py

      - name: Git commit && push
        run: |
          git_diff=`git diff`
          if [ -z "$git_diff" ]; then echo "Nothing Changed";exit; fi
          git add .
          git commit -m "Github Action Auto Updated - `date`"
          git push origin status_ordered


Update TestFilght Link Status

name: Update TestFilght Link Status

on:
  schedule:
    - cron: '0 3 * * *'
  workflow_dispatch:
    inputs:
      python_version:
        description: '指定 Python 版本 3.6+ （不建议修改）'
        required: false
        default: '3.11.1'

jobs:
  main:
    runs-on: ubuntu-latest
    if: github.repository == 'pluwen/awesome-testflight-link'
    env:
      python_version: ${{ github.event.inputs.python_version }}
    steps:
      - uses: actions/checkout@v3.3.0
      - name: Check env
        run: if [ -z "$python_version" ]; then python_version='3.11.1'; fi
      - name: Set up Python
        uses: actions/setup-python@v4.5.0
        with:
          python-version: ${{ env.python_version }}
          architecture: x64
      - name: Init Python Env
        run: pip install -r scripts/requirements.txt
      - name: Update TestFilght Link Status
        run: python scripts/update_status.py

      - name: Git commit && push
        run: |
          git_diff=`git diff`
          if [ -z "$git_diff" ]; then echo "Nothing Changed";exit; fi
          git config --global user.email "github_bot@noreply.github.com"
          git config --global user.name "github_bot"
          git add .
          git commit -m "Github Action Auto Updated - `date`"
          git push

  forked-repo:
    runs-on: ubuntu-latest
    if: github.repository != 'pluwen/awesome-testflight-link'

    steps:
      - uses: actions/checkout@v3.3.0

      - name: Git commit && push
        run: |
          git config --global user.email "github_bot@noreply.github.com"
          git config --global user.name "github_bot"
          git remote add upstream https://github.com/pluwen/awesome-testflight-link.git
          git fetch upstream
          git checkout main
          git merge upstream/main --allow-unrelated-histories
          git push origin main


Build Frontend

# Build the frontend from source, and open a pull request to commit the build artifacts.
# This allows Python users to use the React frontend without installing and running Node.
name: Build Frontend

on:
  push:
    branches:
      - 'main'
    paths:
      - 'helm-frontend/**'
      - '.github/workflows/build-frontend.yml'

jobs:
  build:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    - name: Use Node.js
      uses: actions/setup-node@v4
      with:
        node-version: '18'
    - name: Install Yarn
      working-directory: ./helm-frontend
      run: npm install --global yarn
    - name: Install dependencies
      working-directory: ./helm-frontend
      run: yarn install
    - name: Build app
      working-directory: ./helm-frontend
      run: yarn build --outDir '../src/helm/benchmark/static_build' --emptyOutDir
    - name: Write README.md
      run: echo -e '# Frontend Build\n\nThis directory is automatically generated by GitHub Actions and contains a static site built from helm-frontend. Do not modify this directory!' > src/helm/benchmark/static_build/README.md
    - name: Git add
      run: git add --force src/helm/benchmark/static_build
    - name: Create pull request
      uses: peter-evans/create-pull-request@v6
      with:
        commit-message: Build frontend
        branch: actions/build-frontend
        delete-branch: true
        title: 'Build frontend'
        body: Auto-generated from GitHub Actions.


Frontend

name: Frontend

on:
  push:
    branches:
      - '*'
    paths:
      - 'helm-frontend/**'
  pull_request:
    branches:
      - '*'
    paths:
      - 'helm-frontend/**'

jobs:
  test:
    runs-on: ubuntu-latest
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    - name: Use Node.js
      uses: actions/setup-node@v4
      with:
        node-version: '18'
    - name: Install Yarn
      run: npm install --global yarn
    - name: Install dependencies
      working-directory: ./helm-frontend
      run: yarn install
    - name: Run lint
      working-directory: ./helm-frontend
      run: yarn lint
    - name: Run check format
      working-directory: ./helm-frontend
      run: yarn format:check
    - name: Run tests
      working-directory: ./helm-frontend
      run: yarn test

  build:
    runs-on: ubuntu-latest
    # Deploy to only run on pushes to master
    # if: github.event_name == 'push' && github.ref == 'refs/heads/main'
    if: github.event_name == 'push' && github.ref == 'refs/heads/react_frontend'
    needs: test
    environment:
      name: github-pages
    env:
      VITE_HELM_BENCHMARKS_ENDPOINT: ${{ vars.VITE_HELM_BENCHMARKS_ENDPOINT }}
      VITE_HELM_BENCHMARKS_SUITE: ${{ vars.VITE_HELM_BENCHMARKS_SUITE }}
    steps:
    - name: Checkout code
      uses: actions/checkout@v4
    - name: Use Node.js
      uses: actions/setup-node@v4
      with:
        node-version: '18'
    - name: Install Yarn
      working-directory: ./helm-frontend
      run: npm install --global yarn
    - name: Install dependencies
      working-directory: ./helm-frontend
      run: yarn install
    - name: Build app
      working-directory: ./helm-frontend
      run: yarn build
    - name: Upload artifact
      uses: actions/upload-pages-artifact@v3
      with:
        path: ./src/helm/benchmark/static_build/

  deploy:
    runs-on: ubuntu-latest
    needs: build
    permissions:
      pages: write
      id-token: write
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    steps:
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v3


Manage Python Cache

# Workflows for manually managing the Python cache.
#
# HELM's dependencies (e.g. PyTorch, transformers) can be very large,
# and the cache may contain multiple versions of these dependencies,
# so we may have to purge the cache from time to time.

name: Manage Python Cache

on: workflow_dispatch

jobs:
  info:
    name: Show pip cache info and package list
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10"]
    steps:
      - name: Check out repository
        uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip
      - name: Get pip cache info
        run: pip cache info && pip cache list

  purge:
    name: Purge and rebuild pip cache
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10"]
    steps:
      - name: Check out repository
        uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip
      - name: Get pip cache info before purging
        run: pip cache info && pip cache list
      - name: Purge pip cache
        run: pip cache purge
      - name: Get pip cache info after purging
        run: pip cache info && pip cache list
      - name: Install HELM
        run: pip install -e .[all]
      - name: Get pip cache info after installing HELM
        run: pip cache info && pip cache list


Publish Python package to PyPI

# This workflow will upload a Python Package using Twine when a release is created
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python#publishing-to-package-registries

name: Publish Python package to PyPI

on:
  release:
    types: [published]

permissions:
  contents: read

jobs:
  pypi-publish:
    name: Publish Python package to PyPI
    runs-on: ubuntu-latest
    environment:
      name: pypi
      url: https://pypi.org/p/crfm-helm
    permissions:
      id-token: write
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: 3.8
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install build
    - name: Build package
      run: python -m build
    - name: Publish package
      uses: pypa/gh-action-pypi-publish@release/v1


Test

name: Test
on:
  push:
    branches: [ main ]
    paths-ignore:
      - 'helm-frontend/**'
  pull_request:
    paths-ignore:
      - 'helm-frontend/**'

jobs:
  install:
    # Tests that the HELM CLI commands work when only installing required dependencies
    # without optional extra dependencies.
    name: Run HELM with minimal dependencies only
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10"]
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip
      - run: python3 -m pip install --upgrade build
      - run: python3 -m build
      - run: python3 -m pip install dist/crfm_helm-*.whl
      - run: helm-run --run-entries simple1:model=simple/model1 --max-eval-instances 10 --suite test
      - run: helm-summarize --suite test
      - run: helm-server --help

  install_openvino:
    # Tests that the Optimum Intel command works when only installing required dependencies
    name: Run Optimum Intel with minimal dependencies only
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10"]
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip
      - run: python3 -m pip install --upgrade build
      - run: python3 -m build
      - run: python3 -m pip install "$(ls dist/crfm_helm-*.whl)[openvino]"
      - run: helm-run --run-entries boolq:model=hf-internal-testing/tiny-random-MistralForCausalLM --enable-huggingface-models hf-internal-testing/tiny-random-MistralForCausalLM --suite v1 --max-eval-instances 10 --openvino

  test:
    name: Run all tests
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10"]
    steps:
      - name: Clear free space
        run: |
            sudo rm -rf /opt/ghc
            df -h
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip
      # Installs dependencies and performs static code checks
      - run: python3 -m pip install virtualenv && python3 -m virtualenv -p python3 venv
      - run: source venv/bin/activate && ./install-dev.sh
      - run: source venv/bin/activate && ./pre-commit.sh
      - name: Run tests
        run: source venv/bin/activate && pytest --durations=20
        env:
          TEST: ${{ matrix.test }}
          VERSION: ${{ github.head_ref || 'main' }}
      - name: Run entire pipeline quickly without any data
        # Checking RunSpecs with openai/davinci should be comprehensive enough
        run: source venv/bin/activate && helm-run --suite test -m 100 --skip-instances --models-to-run openai/davinci --exit-on-error


Scenario tests

name: Scenario tests
on:
  push:
    branches: [ main ]
    paths:
      - 'src/helm/benchmark/scenarios/test_*_scenario.py'
  pull_request:
    paths:
      - 'src/helm/benchmark/scenarios/test_*_scenario.py'

jobs:
  test:
    name: Run scenario tests
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.8", "3.9", "3.10"]
    steps:
      - name: Clear free space
        run: |
            sudo rm -rf /opt/ghc
            df -h
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip
      # Installs dependencies and performs static code checks
      - name: Install HELM
        run: ./install-dev.sh && ./pre-commit.sh
      - name: Run scenario tests
        run: python3 -m pytest -m scenarios


Update requirements.txt

# This GitHub Actions workflow automatically updates the pinned versions
# in requirements.txt for all Python dependencies (including transitive dependencies)
# whenever setup.cfg is modified.

name: Update requirements.txt

on:
  push:
    branches:
      - "main"
    paths:
      - "setup.cfg"
      - '.github/workflows/update-requirements.yml'

jobs:
  update-requirements:
    name: Update requirements.txt
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version: ["3.8"]
    steps:
      - name: Check out repository
        uses: actions/checkout@v4
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}
          cache: pip
      - name: Upgrade pip
        run: pip install --upgrade pip
      - name: Install PyTorch manually
        # PyTorch has to be installed manually in a separate step with --no-cache-dir
        # to avoid pip getting killed because PyTorch is too big
        # See: https://stackoverflow.com/a/54329850
        run: pip install torch==2.1.2 torchvision==0.16.2 --index-url https://download.pytorch.org/whl/cu121 --no-cache-dir
      - name: Upgrade dependencies for Dependabot alerts
        run: pip install -e .[all,dev] urllib3==1.26.18 requests==2.31.0 Jinja2==3.1.3 certifi==2024.2.2 aiohttp==3.9.3
      - name: Verify dependencies
        run: pip check
      - name: Write header to requirements.txt
        run: echo '# This file is automatically generated by GitHub Actions and contains pinned versions for all transitive Python dependencies. Do not modify this file!' > requirements.txt
      - name: Write dependencies to requirements.txt
        # grep is used to exclude dependencies installed in editable mode
        # and dependencies installed from outside PyPI.
        # sed is used to loosen the version matches for PyTorch
        # so that any CUDA version matches.
        run: pip freeze | grep -v -E "( @ |^-e)" | sed -E 's/==([A-Za-z0-9.]+)\+[A-Za-z0-9.]+/~=\1/' >> requirements.txt
        # Need to manually run tests here because the pull request opened later will not
        # run the test workflow.
        #
        # From https://docs.github.com/en/actions/security-guides/automatic-token-authentication#using-the-github_token-in-a-workflow
        #
        #     ...events triggered by the GITHUB_TOKEN... will not create a new workflow run.
        #     This prevents you from accidentally creating recursive workflow runs. For example,
        #     if a workflow run pushes code using the repository's GITHUB_TOKEN, a new workflow
        #     will not run even when the repository contains a workflow configured to run when
        #     push events occur.
      - name: Run linter
        run: ./pre-commit.sh
      - name: Run tests
        run: pytest
      - name: Run helm-run
        run: helm-run --suite test --run-entries simple1:model=simple/model1 --max-eval-instances 10 --exit-on-error
      - name: Create pull request
        uses: peter-evans/create-pull-request@v6
        with:
          commit-message: Update requirements.txt
          title: "Update requirements.txt"
          branch: actions/update-requirements
          delete-branch: true
          body: Auto-generated from GitHub Actions.


Cargo Publish

name: Cargo Publish

on:
  push:
    tags-ignore:
      # We don't publish pre-releases for Rust. Crates.io is just a source
      # distribution, so we don't need to publish pre-releases.
      - 'v*-beta*'
      - '*-v*' # for example, python-vX.Y.Z

env:
  # This env var is used by Swatinem/rust-cache@v2 for the cache
  # key, so we set it to make sure it is always consistent.
  CARGO_TERM_COLOR: always
  # Up-to-date compilers needed for fp16kernels.
  CC: gcc-12
  CXX: g++-12

jobs:
  build:
    runs-on: ubuntu-22.04
    timeout-minutes: 30
    # Only runs on tags that matches the make-release action
    if: startsWith(github.ref, 'refs/tags/v')
    steps:
      - uses: actions/checkout@v4
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: rust
      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler libssl-dev
      - name: Publish the package
        run: |
          cargo publish -p lancedb --all-features --token ${{ secrets.CARGO_REGISTRY_TOKEN }}


PR Checks

name: PR Checks

on:
  pull_request_target:
    types: [opened, edited, synchronize, reopened]

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  labeler:
    permissions:
      pull-requests: write
    name: Label PR
    runs-on: ubuntu-latest
    steps:
      - uses: srvaroa/labeler@master
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  commitlint:
    permissions:
      pull-requests: write
    name: Verify PR title / description conforms to semantic-release
    runs-on: ubuntu-latest
    steps:
      - uses: actions/setup-node@v3
        with:
          node-version: "18"
      # These rules are disabled because Github will always ensure there
      # is a blank line between the title and the body and Github will
      # word wrap the description field to ensure a reasonable max line
      # length.
      - run: npm install @commitlint/config-conventional
      - run: >
          echo 'module.exports = {
            "rules": {
              "body-max-line-length": [0, "always", Infinity],
              "footer-max-line-length": [0, "always", Infinity],
              "body-leading-blank": [0, "always"]
            }
          }' > .commitlintrc.js
      - run: npx commitlint --extends @commitlint/config-conventional --verbose <<< $COMMIT_MSG
        env:
          COMMIT_MSG: >
            ${{ github.event.pull_request.title }}

            ${{ github.event.pull_request.body }}
      - if: failure()
        uses: actions/github-script@v6
        with:
          script: |
            const message = `**ACTION NEEDED**

              Lance follows the [Conventional Commits specification](https://www.conventionalcommits.org/en/v1.0.0/) for release automation.

              The PR title and description are used as the merge commit message.\
              Please update your PR title and description to match the specification.

              For details on the error please inspect the "PR Title Check" action.
              `
            // Get list of current comments
            const comments = await github.paginate(github.rest.issues.listComments, {
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number
            });
            // Check if this job already commented
            for (const comment of comments) {
              if (comment.body === message) {
                return // Already commented
              }
            }
            // Post the comment about Conventional Commits
            github.rest.issues.createComment({
              owner: context.repo.owner,
              repo: context.repo.repo,
              issue_number: context.issue.number,
              body: message
            })
            core.setFailed(message)


Deploy docs to Pages

name: Deploy docs to Pages

on:
  push:
    branches: ["main"]

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

# Sets permissions of the GITHUB_TOKEN to allow deployment to GitHub Pages
permissions:
  contents: read
  pages: write
  id-token: write

# Allow one concurrent deployment
concurrency:
  group: "pages"
  cancel-in-progress: true

jobs:
  # Single deploy job since we're just deploying
  build:
    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}
    runs-on: buildjet-8vcpu-ubuntu-2204
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install dependecies needed for ubuntu
        run: |
          sudo apt install -y protobuf-compiler libssl-dev
          rustup update && rustup default
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.10"
          cache: "pip"
          cache-dependency-path: "docs/requirements.txt"
      - name: Build Python
        working-directory: python
        run: |
          python -m pip install -e .
          python -m pip install -r ../docs/requirements.txt
      - name: Set up node
        uses: actions/setup-node@v3
        with:
          node-version: 20
          cache: 'npm'
          cache-dependency-path: node/package-lock.json
      - uses: Swatinem/rust-cache@v2
      - name: Install node dependencies
        working-directory: node
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler libssl-dev
      - name: Build node
        working-directory: node
        run: |
          npm ci
          npm run build
          npm run tsc
      - name: Create markdown files
        working-directory: node
        run: |
          npx typedoc --plugin typedoc-plugin-markdown --out ../docs/src/javascript src/index.ts
      - name: Build docs
        working-directory: docs
        run: |
          PYTHONPATH=. mkdocs build
      - name: Setup Pages
        uses: actions/configure-pages@v2
      - name: Upload artifact
        uses: actions/upload-pages-artifact@v1
        with:
          path: "docs/site"
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v1


Documentation Code Testing

name: Documentation Code Testing

on:
  push:
    branches:
      - main
    paths:
      - docs/**
      - .github/workflows/docs_test.yml
  pull_request:
    paths:
      - docs/**
      - .github/workflows/docs_test.yml

  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:

env:
  # Disable full debug symbol generation to speed up CI build and keep memory down
  # "1" means line tables only, which is useful for panic tracebacks.
  RUSTFLAGS: "-C debuginfo=1 -C target-cpu=haswell -C target-feature=+f16c,+avx2,+fma"
  RUST_BACKTRACE: "1"

jobs:
  test-python:
    name: Test doc python code
    runs-on: "buildjet-8vcpu-ubuntu-2204"
    steps:
    - name: Checkout
      uses: actions/checkout@v4
    - name: Print CPU capabilities
      run: cat /proc/cpuinfo
    - name: Install dependecies needed for ubuntu
      run: |
        sudo apt install -y protobuf-compiler libssl-dev
        rustup update && rustup default
    - name: Set up Python
      uses: actions/setup-python@v5
      with:
        python-version: 3.11
        cache: "pip"
        cache-dependency-path: "docs/test/requirements.txt"
    - name: Rust cache
      uses: swatinem/rust-cache@v2
    - name: Build Python
      working-directory: docs/test
      run:
        python -m pip install -r requirements.txt
    - name: Create test files
      run: |
        cd docs/test
        python md_testing.py
    - name: Test
      run: |
        cd docs/test/python
        for d in *; do cd "$d"; echo "$d".py; python "$d".py; cd ..; done
  test-node:
    name: Test doc nodejs code
    runs-on: "buildjet-8vcpu-ubuntu-2204"
    timeout-minutes: 60
    strategy:
      fail-fast: false
    steps:
    - name: Checkout
      uses: actions/checkout@v4
      with:
        fetch-depth: 0
        lfs: true
    - name: Print CPU capabilities
      run: cat /proc/cpuinfo
    - name: Set up Node
      uses: actions/setup-node@v4
      with:
        node-version: 20
    - name: Install dependecies needed for ubuntu
      run: |
        sudo apt install -y protobuf-compiler libssl-dev
        rustup update && rustup default
    - name: Rust cache
      uses: swatinem/rust-cache@v2
    - name: Install node dependencies
      run: |
        sudo swapoff -a
        sudo fallocate -l 8G /swapfile
        sudo chmod 600 /swapfile
        sudo mkswap /swapfile
        sudo swapon /swapfile
        sudo swapon --show
        cd node
        npm ci
        npm run build-release
        cd ../docs
        npm install
    - name: Test
      env:
        LANCEDB_URI: ${{ secrets.LANCEDB_URI }}
        LANCEDB_DEV_API_KEY: ${{ secrets.LANCEDB_DEV_API_KEY }}
      run: |
        cd docs
        npm t


Build and Run Java JNI Tests

name: Build and Run Java JNI Tests
on:
  push:
    branches:
      - main
  pull_request:
    paths:
      - java/**
      - rust/**
      - .github/workflows/java.yml
env:
  # This env var is used by Swatinem/rust-cache@v2 for the cache
  # key, so we set it to make sure it is always consistent.
  CARGO_TERM_COLOR: always
  # Disable full debug symbol generation to speed up CI build and keep memory down
  # "1" means line tables only, which is useful for panic tracebacks.
  RUSTFLAGS: "-C debuginfo=1"
  RUST_BACKTRACE: "1"
  # according to: https://matklad.github.io/2021/09/04/fast-rust-builds.html
  # CI builds are faster with incremental disabled.
  CARGO_INCREMENTAL: "0"
  CARGO_BUILD_JOBS: "1"
jobs:
  linux-build:
    runs-on: ubuntu-22.04
    name: ubuntu-22.04 + Java 11 & 17
    defaults:
      run:
        working-directory: ./java
    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: java/core/lancedb-jni
      - name: Run cargo fmt
        run: cargo fmt --check
        working-directory: ./java/core/lancedb-jni
      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler libssl-dev
      - name: Install Java 17
        uses: actions/setup-java@v4
        with:
          distribution: temurin
          java-version: 17
          cache: "maven"
      - run: echo "JAVA_17=$JAVA_HOME" >> $GITHUB_ENV
      - name: Install Java 11
        uses: actions/setup-java@v4
        with:
          distribution: temurin
          java-version: 11
          cache: "maven"
      - name: Java Style Check
        run: mvn checkstyle:check
      # Disable because of issues in lancedb rust core code
      # - name: Rust Clippy
      #   working-directory: java/core/lancedb-jni
      #   run: cargo clippy --all-targets -- -D warnings
      - name: Running tests with Java 11
        run: mvn clean test
      - name: Running tests with Java 17
        run: |
          export JAVA_TOOL_OPTIONS="$JAVA_TOOL_OPTIONS \
          -XX:+IgnoreUnrecognizedVMOptions \
          --add-opens=java.base/java.lang=ALL-UNNAMED \
          --add-opens=java.base/java.lang.invoke=ALL-UNNAMED \
          --add-opens=java.base/java.lang.reflect=ALL-UNNAMED \
          --add-opens=java.base/java.io=ALL-UNNAMED \
          --add-opens=java.base/java.net=ALL-UNNAMED \
          --add-opens=java.base/java.nio=ALL-UNNAMED \
          --add-opens=java.base/java.util=ALL-UNNAMED \
          --add-opens=java.base/java.util.concurrent=ALL-UNNAMED \
          --add-opens=java.base/java.util.concurrent.atomic=ALL-UNNAMED \
          --add-opens=java.base/jdk.internal.ref=ALL-UNNAMED \
          --add-opens=java.base/sun.nio.ch=ALL-UNNAMED \
          --add-opens=java.base/sun.nio.cs=ALL-UNNAMED \
          --add-opens=java.base/sun.security.action=ALL-UNNAMED \
          --add-opens=java.base/sun.util.calendar=ALL-UNNAMED \
          --add-opens=java.security.jgss/sun.security.krb5=ALL-UNNAMED \
          -Djdk.reflect.useDirectMethodHandle=false \
          -Dio.netty.tryReflectionSetAccessible=true"
          JAVA_HOME=$JAVA_17 mvn clean test


Create release commit

name: Create release commit

# This workflow increments versions, tags the version, and pushes it.
# When a tag is pushed, another workflow is triggered that creates a GH release
# and uploads the binaries. This workflow is only for creating the tag.

# This script will enforce that a minor version is incremented if there are any
# breaking changes since the last minor increment. However, it isn't able to
# differentiate between breaking changes in Node versus Python. If you wish to
# bypass this check, you can manually increment the version and push the tag.
on:
  workflow_dispatch:
    inputs:
      dry_run:
        description: 'Dry run (create the local commit/tags but do not push it)'
        required: true
        default: false
        type: boolean
      type:
        description: 'What kind of release is this?'
        required: true
        default: 'preview'
        type: choice
        options:
          - preview
          - stable
      python:
        description: 'Make a Python release'
        required: true
        default: true
        type: boolean
      other:
        description: 'Make a Node/Rust release'
        required: true
        default: true
        type: boolean
      bump-minor:
        description: 'Bump minor version'
        required: true
        default: false
        type: boolean

jobs:
  make-release:
    # Creates tag and GH release. The GH release will trigger the build and release jobs.
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - name: Output Inputs
        run: echo "${{ toJSON(github.event.inputs) }}"
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
          # It's important we use our token here, as the default token will NOT
          # trigger any workflows watching for new tags. See:
          # https://docs.github.com/en/actions/using-workflows/triggering-a-workflow#triggering-a-workflow-from-a-workflow
          token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
      - name: Set git configs for bumpversion
        shell: bash
        run: |
          git config user.name 'Lance Release'
          git config user.email 'lance-dev@lancedb.com'
      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Bump Python version
        if: ${{ inputs.python }}
        working-directory: python
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          # Need to get the commit before bumping the version, so we can
          # determine if there are breaking changes in the next step as well.
          echo "COMMIT_BEFORE_BUMP=$(git rev-parse HEAD)" >> $GITHUB_ENV

          pip install bump-my-version PyGithub packaging
          bash ../ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} python-v
      - name: Bump Node/Rust version
        if: ${{ inputs.other }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
        run: |
          pip install bump-my-version PyGithub packaging
          bash ci/bump_version.sh ${{ inputs.type }} ${{ inputs.bump-minor }} v $COMMIT_BEFORE_BUMP
      - name: Push new version tag
        if: ${{ !inputs.dry_run }}
        uses: ad-m/github-push-action@master
        with:
          # Need to use PAT here too to trigger next workflow. See comment above.
          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}
          branch: ${{ github.ref }}
          tags: true
      - uses: ./.github/workflows/update_package_lock
        if: ${{ !inputs.dry_run && inputs.other }}
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}


Node

name: Node

on:
  push:
    branches:
      - main
  pull_request:
    paths:
      - node/**
      - rust/ffi/node/**
      - .github/workflows/node.yml
      - docker-compose.yml

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  # Disable full debug symbol generation to speed up CI build and keep memory down
  # "1" means line tables only, which is useful for panic tracebacks.
  #
  # Use native CPU to accelerate tests if possible, especially for f16
  # target-cpu=haswell fixes failing ci build
  RUSTFLAGS: "-C debuginfo=1 -C target-cpu=haswell -C target-feature=+f16c,+avx2,+fma"
  RUST_BACKTRACE: "1"

jobs:
  linux:
    name: Linux (Node ${{ matrix.node-version }})
    timeout-minutes: 30
    strategy:
      matrix:
        node-version: [ "18", "20" ]
    runs-on: "ubuntu-22.04"
    defaults:
      run:
        shell: bash
        working-directory: node
    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        lfs: true
    - uses: actions/setup-node@v3
      with:
        node-version: ${{ matrix.node-version }}
        cache: 'npm'
        cache-dependency-path: node/package-lock.json
    - uses: Swatinem/rust-cache@v2
    - name: Install dependencies
      run: |
        sudo apt update
        sudo apt install -y protobuf-compiler libssl-dev
    - name: Build
      run: |
        npm ci
        npm run build
        npm run pack-build
        npm install --no-save ./dist/lancedb-vectordb-*.tgz
        # Remove index.node to test with dependency installed
        rm index.node
    - name: Test
      run: npm run test
  macos:
    timeout-minutes: 30
    runs-on: "macos-13"
    defaults:
      run:
        shell: bash
        working-directory: node
    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        lfs: true
    - uses: actions/setup-node@v3
      with:
        node-version: 20
        cache: 'npm'
        cache-dependency-path: node/package-lock.json
    - uses: Swatinem/rust-cache@v2
    - name: Install dependencies
      run: brew install protobuf
    - name: Build
      run: |
        npm ci
        npm run build
        npm run pack-build
        npm install --no-save ./dist/lancedb-vectordb-*.tgz
        # Remove index.node to test with dependency installed
        rm index.node
    - name: Test
      run: |
        npm run test
  aws-integtest:
    timeout-minutes: 45
    runs-on: "ubuntu-22.04"
    defaults:
      run:
        shell: bash
        working-directory: node
    env:
      AWS_ACCESS_KEY_ID: ACCESSKEY
      AWS_SECRET_ACCESS_KEY: SECRETKEY
      AWS_DEFAULT_REGION: us-west-2
      # this one is for s3
      AWS_ENDPOINT: http://localhost:4566
      # this one is for dynamodb
      DYNAMODB_ENDPOINT: http://localhost:4566
      ALLOW_HTTP: true
    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        lfs: true
    - uses: actions/setup-node@v3
      with:
        node-version: 20
        cache: 'npm'
        cache-dependency-path: node/package-lock.json
    - name: start local stack
      run: docker compose -f ../docker-compose.yml up -d --wait
    - name: create s3
      run: aws s3 mb s3://lancedb-integtest --endpoint $AWS_ENDPOINT
    - name: create ddb
      run: |
        aws dynamodb create-table \
          --table-name lancedb-integtest \
          --attribute-definitions '[{"AttributeName": "base_uri", "AttributeType": "S"}, {"AttributeName": "version", "AttributeType": "N"}]' \
          --key-schema '[{"AttributeName": "base_uri", "KeyType": "HASH"}, {"AttributeName": "version", "KeyType": "RANGE"}]' \
          --provisioned-throughput '{"ReadCapacityUnits": 10, "WriteCapacityUnits": 10}' \
          --endpoint-url $DYNAMODB_ENDPOINT
    - uses: Swatinem/rust-cache@v2
    - name: Install dependencies
      run: |
        sudo apt update
        sudo apt install -y protobuf-compiler libssl-dev
    - name: Build
      run: |
        npm ci
        npm run build
        npm run pack-build
        npm install --no-save ./dist/lancedb-vectordb-*.tgz
        # Remove index.node to test with dependency installed
        rm index.node
    - name: Test
      run: npm run integration-test


NodeJS (NAPI)

name: NodeJS (NAPI)

on:
  push:
    branches:
      - main
  pull_request:
    paths:
      - nodejs/**
      - .github/workflows/nodejs.yml
      - docker-compose.yml

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  # Disable full debug symbol generation to speed up CI build and keep memory down
  # "1" means line tables only, which is useful for panic tracebacks.
  RUSTFLAGS: "-C debuginfo=1"
  RUST_BACKTRACE: "1"

jobs:
  lint:
    name: Lint
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash
        working-directory: nodejs
    env:
      # Need up-to-date compilers for kernels
      CC: gcc-12
      CXX: g++-12
    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        lfs: true
    - uses: actions/setup-node@v3
      with:
        node-version: 20
        cache: 'npm'
        cache-dependency-path: nodejs/package-lock.json
    - uses: Swatinem/rust-cache@v2
    - name: Install dependencies
      run: |
        sudo apt update
        sudo apt install -y protobuf-compiler libssl-dev
    - name: Lint
      run: |
        cargo fmt --all -- --check
        cargo clippy --all --all-features -- -D warnings
        npm ci
        npm run lint-ci
  linux:
    name: Linux (NodeJS ${{ matrix.node-version }})
    timeout-minutes: 30
    strategy:
      matrix:
        node-version: [ "18", "20" ]
    runs-on: "ubuntu-22.04"
    defaults:
      run:
        shell: bash
        working-directory: nodejs
    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        lfs: true
    - uses: actions/setup-node@v3
      with:
        node-version: ${{ matrix.node-version }}
        cache: 'npm'
        cache-dependency-path: node/package-lock.json
    - uses: Swatinem/rust-cache@v2
    - name: Install dependencies
      run: |
        sudo apt update
        sudo apt install -y protobuf-compiler libssl-dev
        npm install -g @napi-rs/cli
    - name: Build
      run: |
        npm ci
        npm run build
    - name: Setup localstack
      working-directory: .
      run: docker compose up --detach --wait
    - name: Test
      env:
        S3_TEST: "1"
      run: npm run test
  macos:
    timeout-minutes: 30
    runs-on: "macos-14"
    defaults:
      run:
        shell: bash
        working-directory: nodejs
    steps:
    - uses: actions/checkout@v4
      with:
        fetch-depth: 0
        lfs: true
    - uses: actions/setup-node@v3
      with:
        node-version: 20
        cache: 'npm'
        cache-dependency-path: node/package-lock.json
    - uses: Swatinem/rust-cache@v2
    - name: Install dependencies
      run: |
        brew install protobuf
        npm install -g @napi-rs/cli
    - name: Build
      run: |
        npm ci
        npm run build
    - name: Test
      run: |
        npm run test


NPM Publish

name: NPM Publish

on:
  push:
    tags:
      - "v*"

jobs:
  node:
    runs-on: ubuntu-latest
    # Only runs on tags that matches the make-release action
    if: startsWith(github.ref, 'refs/tags/v')
    defaults:
      run:
        shell: bash
        working-directory: node
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - uses: actions/setup-node@v3
        with:
          node-version: 20
          cache: "npm"
          cache-dependency-path: node/package-lock.json
      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler libssl-dev
      - name: Build
        run: |
          npm ci
          npm run tsc
          npm pack
      - name: Upload Linux Artifacts
        uses: actions/upload-artifact@v4
        with:
          name: node-package
          path: |
            node/vectordb-*.tgz

  node-macos:
    strategy:
      matrix:
        config:
          - arch: x86_64-apple-darwin
            runner: macos-13
          - arch: aarch64-apple-darwin
            # xlarge is implicitly arm64.
            runner: macos-14
    runs-on: ${{ matrix.config.runner }}
    # Only runs on tags that matches the make-release action
    if: startsWith(github.ref, 'refs/tags/v')
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install system dependencies
        run: brew install protobuf
      - name: Install npm dependencies
        run: |
          cd node
          npm ci
      - name: Build MacOS native node modules
        run: bash ci/build_macos_artifacts.sh ${{ matrix.config.arch }}
      - name: Upload Darwin Artifacts
        uses: actions/upload-artifact@v4
        with:
          name: node-native-darwin-${{ matrix.config.arch }}
          path: |
            node/dist/lancedb-vectordb-darwin*.tgz

  nodejs-macos:
    strategy:
      matrix:
        config:
          - arch: x86_64-apple-darwin
            runner: macos-13
          - arch: aarch64-apple-darwin
            # xlarge is implicitly arm64.
            runner: macos-14
    runs-on: ${{ matrix.config.runner }}
    # Only runs on tags that matches the make-release action
    if: startsWith(github.ref, 'refs/tags/v')
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install system dependencies
        run: brew install protobuf
      - name: Install npm dependencies
        run: |
          cd nodejs
          npm ci
      - name: Build MacOS native nodejs modules
        run: bash ci/build_macos_artifacts_nodejs.sh ${{ matrix.config.arch }}
      - name: Upload Darwin Artifacts
        uses: actions/upload-artifact@v4
        with:
          name: nodejs-native-darwin-${{ matrix.config.arch }}
          path: |
            nodejs/dist/*.node

  node-linux:
    name: node-linux (${{ matrix.config.arch}}-unknown-linux-gnu
    runs-on: ${{ matrix.config.runner }}
    # Only runs on tags that matches the make-release action
    if: startsWith(github.ref, 'refs/tags/v')
    strategy:
      fail-fast: false
      matrix:
        config:
          - arch: x86_64
            runner: ubuntu-latest
          - arch: aarch64
            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
            runner: warp-ubuntu-latest-arm64-4x
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      # To avoid OOM errors on ARM, we create a swap file.
      - name: Configure aarch64 build
        if: ${{ matrix.config.arch == 'aarch64' }}
        run: |
          free -h
          sudo fallocate -l 16G /swapfile
          sudo chmod 600 /swapfile
          sudo mkswap /swapfile
          sudo swapon /swapfile
          echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab
          # print info
          swapon --show
          free -h
      - name: Build Linux Artifacts
        run: |
          bash ci/build_linux_artifacts.sh ${{ matrix.config.arch }}
      - name: Upload Linux Artifacts
        uses: actions/upload-artifact@v4
        with:
          name: node-native-linux-${{ matrix.config.arch }}
          path: |
            node/dist/lancedb-vectordb-linux*.tgz

  nodejs-linux:
    name: nodejs-linux (${{ matrix.config.arch}}-unknown-linux-gnu
    runs-on: ${{ matrix.config.runner }}
    # Only runs on tags that matches the make-release action
    if: startsWith(github.ref, 'refs/tags/v')
    strategy:
      fail-fast: false
      matrix:
        config:
          - arch: x86_64
            runner: ubuntu-latest
          - arch: aarch64
            # For successful fat LTO builds, we need a large runner to avoid OOM errors.
            runner: buildjet-16vcpu-ubuntu-2204-arm
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      # Buildjet aarch64 runners have only 1.5 GB RAM per core, vs 3.5 GB per core for
      # x86_64 runners. To avoid OOM errors on ARM, we create a swap file.
      - name: Configure aarch64 build
        if: ${{ matrix.config.arch == 'aarch64' }}
        run: |
          free -h
          sudo fallocate -l 16G /swapfile
          sudo chmod 600 /swapfile
          sudo mkswap /swapfile
          sudo swapon /swapfile
          echo "/swapfile swap swap defaults 0 0" >> sudo /etc/fstab
          # print info
          swapon --show
          free -h
      - name: Build Linux Artifacts
        run: |
          bash ci/build_linux_artifacts_nodejs.sh ${{ matrix.config.arch }}
      - name: Upload Linux Artifacts
        uses: actions/upload-artifact@v4
        with:
          name: nodejs-native-linux-${{ matrix.config.arch }}
          path: |
            nodejs/dist/*.node
      # The generic files are the same in all distros so we just pick
      # one to do the upload.
      - name: Upload Generic Artifacts
        if: ${{ matrix.config.arch == 'x86_64' }}
        uses: actions/upload-artifact@v4
        with:
          name: nodejs-dist
          path: |
            nodejs/dist/*
            !nodejs/dist/*.node

  node-windows:
    runs-on: windows-2022
    # Only runs on tags that matches the make-release action
    if: startsWith(github.ref, 'refs/tags/v')
    strategy:
      fail-fast: false
      matrix:
        target: [x86_64-pc-windows-msvc]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Protoc v21.12
        working-directory: C:\
        run: |
          New-Item -Path 'C:\protoc' -ItemType Directory
          Set-Location C:\protoc
          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
          7z x protoc.zip
          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
        shell: powershell
      - name: Install npm dependencies
        run: |
          cd node
          npm ci
      - name: Build Windows native node modules
        run: .\ci\build_windows_artifacts.ps1 ${{ matrix.target }}
      - name: Upload Windows Artifacts
        uses: actions/upload-artifact@v4
        with:
          name: node-native-windows
          path: |
            node/dist/lancedb-vectordb-win32*.tgz

  nodejs-windows:
    runs-on: windows-2022
    # Only runs on tags that matches the make-release action
    if: startsWith(github.ref, 'refs/tags/v')
    strategy:
      fail-fast: false
      matrix:
        target: [x86_64-pc-windows-msvc]
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Install Protoc v21.12
        working-directory: C:\
        run: |
          New-Item -Path 'C:\protoc' -ItemType Directory
          Set-Location C:\protoc
          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
          7z x protoc.zip
          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
        shell: powershell
      - name: Install npm dependencies
        run: |
          cd nodejs
          npm ci
      - name: Build Windows native node modules
        run: .\ci\build_windows_artifacts_nodejs.ps1 ${{ matrix.target }}
      - name: Upload Windows Artifacts
        uses: actions/upload-artifact@v4
        with:
          name: nodejs-native-windows
          path: |
            nodejs/dist/*.node

  release:
    needs: [node, node-macos, node-linux, node-windows]
    runs-on: ubuntu-latest
    # Only runs on tags that matches the make-release action
    if: startsWith(github.ref, 'refs/tags/v')
    steps:
      - uses: actions/download-artifact@v4
        with:
          pattern: node-*
      - name: Display structure of downloaded files
        run: ls -R
      - uses: actions/setup-node@v3
        with:
          node-version: 20
          registry-url: "https://registry.npmjs.org"
      - name: Publish to NPM
        env:
          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
        run: |
          # Tag beta as "preview" instead of default "latest". See lancedb
          # npm publish step for more info.
          if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
            PUBLISH_ARGS="--tag preview"
          fi

          mv */*.tgz .
          for filename in *.tgz; do
            npm publish $PUBLISH_ARGS $filename
          done

  release-nodejs:
    needs: [nodejs-macos, nodejs-linux, nodejs-windows]
    runs-on: ubuntu-latest
    # Only runs on tags that matches the make-release action
    if: startsWith(github.ref, 'refs/tags/v')
    defaults:
      run:
        shell: bash
        working-directory: nodejs
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - uses: actions/download-artifact@v4
        with:
          name: nodejs-dist
          path: nodejs/dist
      - uses: actions/download-artifact@v4
        name: Download arch-specific binaries
        with:
          pattern: nodejs-*
          path: nodejs/nodejs-artifacts
          merge-multiple: true
      - name: Display structure of downloaded files
        run: find .
      - uses: actions/setup-node@v3
        with:
          node-version: 20
          registry-url: "https://registry.npmjs.org"
      - name: Install napi-rs
        run: npm install -g @napi-rs/cli
      - name: Prepare artifacts
        run: npx napi artifacts -d nodejs-artifacts
      - name: Display structure of staged files
        run: find npm
      - name: Publish to NPM
        env:
          NODE_AUTH_TOKEN: ${{ secrets.LANCEDB_NPM_REGISTRY_TOKEN }}
        # By default, things are published to the latest tag. This is what is
        # installed by default if the user does not specify a version. This is
        # good for stable releases, but for pre-releases, we want to publish to
        # the "preview" tag so they can install with `npm install lancedb@preview`.
        # See: https://medium.com/@mbostock/prereleases-and-npm-e778fc5e2420
        run: |
          if [[ $GITHUB_REF =~ refs/tags/v(.*)-beta.* ]]; then
            npm publish --access public --tag preview
          else
            npm publish --access public
          fi

  update-package-lock:
    needs: [release]
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: main
          persist-credentials: false
          fetch-depth: 0
          lfs: true
      - uses: ./.github/workflows/update_package_lock
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}

  update-package-lock-nodejs:
    needs: [release-nodejs]
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: main
          persist-credentials: false
          fetch-depth: 0
          lfs: true
      - uses: ./.github/workflows/update_package_lock_nodejs
        with:
          github_token: ${{ secrets.GITHUB_TOKEN }}

  gh-release:
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Extract version
        id: extract_version
        env:
          GITHUB_REF: ${{ github.ref }}
        run: |
          set -e
          echo "Extracting tag and version from $GITHUB_REF"
          if [[ $GITHUB_REF =~ refs/tags/v(.*) ]]; then
            VERSION=${BASH_REMATCH[1]}
            TAG=v$VERSION
            echo "tag=$TAG" >> $GITHUB_OUTPUT
            echo "version=$VERSION" >> $GITHUB_OUTPUT
          else
            echo "Failed to extract version from $GITHUB_REF"
            exit 1
          fi
          echo "Extracted version $VERSION from $GITHUB_REF"
          if [[ $VERSION =~ beta ]]; then
            echo "This is a beta release"

            # Get last release (that is not this one)
            FROM_TAG=$(git tag --sort='version:refname' \
              | grep ^v \
              | grep -vF "$TAG" \
              | python ci/semver_sort.py v \
              | tail -n 1)
          else
            echo "This is a stable release"
            # Get last stable tag (ignore betas)
            FROM_TAG=$(git tag --sort='version:refname' \
              | grep ^v \
              | grep -vF "$TAG" \
              | grep -v beta \
              | python ci/semver_sort.py v \
              | tail -n 1)
          fi
          echo "Found from tag $FROM_TAG"
          echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
      - name: Create Release Notes
        id: release_notes
        uses: mikepenz/release-changelog-builder-action@v4
        with:
          configuration: .github/release_notes.json
          toTag: ${{ steps.extract_version.outputs.tag }}
          fromTag: ${{ steps.extract_version.outputs.from_tag }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Create GH release
        uses: softprops/action-gh-release@v2
        with:
          prerelease: ${{ contains('beta', github.ref) }}
          tag_name: ${{ steps.extract_version.outputs.tag }}
          token: ${{ secrets.GITHUB_TOKEN }}
          generate_release_notes: false
          name: Node/Rust LanceDB v${{ steps.extract_version.outputs.version }}
          body: ${{ steps.release_notes.outputs.changelog }}


PyPI Publish

name: PyPI Publish

on:
  push:
    tags:
      - 'python-v*'

jobs:
  linux:
    name: Python ${{ matrix.config.platform }} manylinux${{ matrix.config.manylinux }}
    timeout-minutes: 60
    strategy:
      matrix:
        config:
          - platform: x86_64
            manylinux: "2_17"
            extra_args: ""
          - platform: x86_64
            manylinux: "2_28"
            extra_args: "--features fp16kernels"
          - platform: aarch64
            manylinux: "2_24"
            extra_args: ""
          # We don't build fp16 kernels for aarch64, because it uses
          # cross compilation image, which doesn't have a new enough compiler.
    runs-on: "ubuntu-22.04"
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - uses: ./.github/workflows/build_linux_wheel
        with:
          python-minor-version: 8
          args: "--release --strip ${{ matrix.config.extra_args }}"
          arm-build: ${{ matrix.config.platform == 'aarch64' }}
          manylinux: ${{ matrix.config.manylinux }}
      - uses: ./.github/workflows/upload_wheel
        with:
          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
          fury_token: ${{ secrets.FURY_TOKEN }}
  mac:
    timeout-minutes: 60
    runs-on: ${{ matrix.config.runner }}
    strategy:
      matrix:
        config:
          - target: x86_64-apple-darwin
            runner: macos-13
          - target: aarch64-apple-darwin
            runner: macos-14
    env:
      MACOSX_DEPLOYMENT_TARGET: 10.15
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.12
      - uses: ./.github/workflows/build_mac_wheel
        with:
          python-minor-version: 8
          args: "--release --strip --target ${{ matrix.config.target }} --features fp16kernels"
      - uses: ./.github/workflows/upload_wheel
        with:
          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
          fury_token: ${{ secrets.FURY_TOKEN }}
  windows:
    timeout-minutes: 60
    runs-on: windows-latest
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v4
        with:
          python-version: 3.8
      - uses: ./.github/workflows/build_windows_wheel
        with:
          python-minor-version: 8
          args: "--release --strip"
          vcpkg_token: ${{ secrets.VCPKG_GITHUB_PACKAGES }}
      - uses: ./.github/workflows/upload_wheel
        with:
          pypi_token: ${{ secrets.LANCEDB_PYPI_API_TOKEN }}
          fury_token: ${{ secrets.FURY_TOKEN }}
  gh-release:
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Extract version
        id: extract_version
        env:
          GITHUB_REF: ${{ github.ref }}
        run: |
          set -e
          echo "Extracting tag and version from $GITHUB_REF"
          if [[ $GITHUB_REF =~ refs/tags/python-v(.*) ]]; then
            VERSION=${BASH_REMATCH[1]}
            TAG=python-v$VERSION
            echo "tag=$TAG" >> $GITHUB_OUTPUT
            echo "version=$VERSION" >> $GITHUB_OUTPUT
          else
            echo "Failed to extract version from $GITHUB_REF"
            exit 1
          fi
          echo "Extracted version $VERSION from $GITHUB_REF"
          if [[ $VERSION =~ beta ]]; then
            echo "This is a beta release"

            # Get last release (that is not this one)
            FROM_TAG=$(git tag --sort='version:refname' \
              | grep ^python-v \
              | grep -vF "$TAG" \
              | python ci/semver_sort.py python-v \
              | tail -n 1)
          else
            echo "This is a stable release"
            # Get last stable tag (ignore betas)
            FROM_TAG=$(git tag --sort='version:refname' \
              | grep ^python-v \
              | grep -vF "$TAG" \
              | grep -v beta \
              | python ci/semver_sort.py python-v \
              | tail -n 1)
          fi
          echo "Found from tag $FROM_TAG"
          echo "from_tag=$FROM_TAG" >> $GITHUB_OUTPUT
      - name: Create Python Release Notes
        id: python_release_notes
        uses: mikepenz/release-changelog-builder-action@v4
        with:
          configuration: .github/release_notes.json
          toTag: ${{ steps.extract_version.outputs.tag }}
          fromTag: ${{ steps.extract_version.outputs.from_tag }}
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
      - name: Create Python GH release
        uses: softprops/action-gh-release@v2
        with:
          prerelease: ${{ contains('beta', github.ref) }}
          tag_name: ${{ steps.extract_version.outputs.tag }}
          token: ${{ secrets.GITHUB_TOKEN }}
          generate_release_notes: false
          name: Python LanceDB v${{ steps.extract_version.outputs.version }}
          body: ${{ steps.python_release_notes.outputs.changelog }}


Python

name: Python

on:
  push:
    branches:
      - main
  pull_request:
    paths:
      - python/**
      - .github/workflows/python.yml

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

jobs:
  lint:
    name: "Lint"
    timeout-minutes: 30
    runs-on: "ubuntu-22.04"
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: Install ruff
        run: |
          pip install ruff==0.2.2
      - name: Format check
        run: ruff format --check .
      - name: Lint
        run: ruff .
  doctest:
    name: "Doctest"
    timeout-minutes: 30
    runs-on: "ubuntu-22.04"
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          cache: "pip"
      - name: Install protobuf
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: python
      - name: Install
        run: |
          pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests,dev,embeddings]
          pip install tantivy
          pip install mlx
      - name: Doctest
        run: pytest --doctest-modules python/lancedb
  linux:
    name: "Linux: python-3.${{ matrix.python-minor-version }}"
    timeout-minutes: 30
    strategy:
      matrix:
        python-minor-version: ["9", "11"]
    runs-on: "ubuntu-22.04"
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Install protobuf
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: 3.${{ matrix.python-minor-version }}
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: python
      - uses: ./.github/workflows/build_linux_wheel
      - uses: ./.github/workflows/run_tests
        with:
          integration: true
      # Make sure wheels are not included in the Rust cache
      - name: Delete wheels
        run: rm -rf target/wheels
  platform:
    name: "Mac: ${{ matrix.config.name }}"
    timeout-minutes: 30
    strategy:
      matrix:
        config:
          - name: x86
            runner: macos-13
          - name: Arm
            runner: macos-14
    runs-on: "${{ matrix.config.runner }}"
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: python
      - uses: ./.github/workflows/build_mac_wheel
      - uses: ./.github/workflows/run_tests
      # Make sure wheels are not included in the Rust cache
      - name: Delete wheels
        run: rm -rf target/wheels
  windows:
    name: "Windows: ${{ matrix.config.name }}"
    timeout-minutes: 30
    strategy:
      matrix:
        config:
          - name: x86
            runner: windows-latest
    runs-on: "${{ matrix.config.runner }}"
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: python
      - uses: ./.github/workflows/build_windows_wheel
      - uses: ./.github/workflows/run_tests
      # Make sure wheels are not included in the Rust cache
      - name: Delete wheels
        run: rm -rf target/wheels
  pydantic1x:
    timeout-minutes: 30
    runs-on: "ubuntu-22.04"
    defaults:
      run:
        shell: bash
        working-directory: python
    steps:
      - uses: actions/checkout@v4
        with:
          fetch-depth: 0
          lfs: true
      - name: Install dependencies
        run: |
          sudo apt update
          sudo apt install -y protobuf-compiler
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: 3.9
      - name: Install lancedb
        run: |
          pip install "pydantic<2"
          pip install --extra-index-url https://pypi.fury.io/lancedb/ -e .[tests]
          pip install tantivy
      - name: Run tests
        run: pytest -m "not slow and not s3_test" -x -v --durations=30 python/tests


Rust

name: Rust

on:
  push:
    branches:
      - main
  pull_request:
    paths:
      - Cargo.toml
      - rust/**
      - .github/workflows/rust.yml

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:
  # This env var is used by Swatinem/rust-cache@v2 for the cache
  # key, so we set it to make sure it is always consistent.
  CARGO_TERM_COLOR: always
  # Disable full debug symbol generation to speed up CI build and keep memory down
  # "1" means line tables only, which is useful for panic tracebacks.
  RUSTFLAGS: "-C debuginfo=1"
  RUST_BACKTRACE: "1"

jobs:
  lint:
    timeout-minutes: 30
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash
        working-directory: rust
    env:
      # Need up-to-date compilers for kernels
      CC: gcc-12
      CXX: g++-12
    steps:
    - uses: actions/checkout@v4
      with:
          fetch-depth: 0
          lfs: true
    - uses: Swatinem/rust-cache@v2
      with:
        workspaces: rust
    - name: Install dependencies
      run: |
          sudo apt update
          sudo apt install -y protobuf-compiler libssl-dev
    - name: Run format
      run: cargo fmt --all -- --check
    - name: Run clippy
      run: cargo clippy --all --all-features -- -D warnings
  linux:
    timeout-minutes: 30
    runs-on: ubuntu-22.04
    defaults:
      run:
        shell: bash
        working-directory: rust
    env:
      # Need up-to-date compilers for kernels
      CC: gcc-12
      CXX: g++-12
    steps:
    - uses: actions/checkout@v4
      with:
          fetch-depth: 0
          lfs: true
    - uses: Swatinem/rust-cache@v2
      with:
          workspaces: rust
    - name: Install dependencies
      run: |
          sudo apt update
          sudo apt install -y protobuf-compiler libssl-dev
    - name: Start S3 integration test environment
      working-directory: .
      run: docker compose up --detach --wait
    - name: Build
      run: cargo build --all-features
    - name: Run tests
      run: cargo test --all-features
    - name: Run examples
      run: cargo run --example simple
  macos:
    timeout-minutes: 30
    strategy:
      matrix:
        mac-runner: [ "macos-13", "macos-14" ]
    runs-on: "${{ matrix.mac-runner }}"
    defaults:
      run:
        shell: bash
        working-directory: rust
    steps:
      - uses: actions/checkout@v4
        with:
            fetch-depth: 0
            lfs: true
      - name: CPU features
        run: sysctl -a | grep cpu
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: rust
      - name: Install dependencies
        run: brew install protobuf
      - name: Build
        run: cargo build --all-features
      - name: Run tests
        # Run with everything except the integration tests.
        run: cargo test --features remote,fp16kernels
  windows:
    runs-on: windows-2022
    steps:
      - uses: actions/checkout@v4
      - uses: Swatinem/rust-cache@v2
        with:
          workspaces: rust
      - name: Install Protoc v21.12
        working-directory: C:\
        run: |
          New-Item -Path 'C:\protoc' -ItemType Directory
          Set-Location C:\protoc
          Invoke-WebRequest https://github.com/protocolbuffers/protobuf/releases/download/v21.12/protoc-21.12-win64.zip -OutFile C:\protoc\protoc.zip
          7z x protoc.zip
          Add-Content $env:GITHUB_PATH "C:\protoc\bin"
        shell: powershell
      - name: Run tests
        run: |
          $env:VCPKG_ROOT = $env:VCPKG_INSTALLATION_ROOT
          cargo build
          cargo test


Trigger vectordb-recipers workflow

name: Trigger vectordb-recipers workflow
on:
  push:
    branches: [ main ]
  pull_request:
    paths:
      - .github/workflows/trigger-vectordb-recipes.yml
  workflow_dispatch:

jobs:
  build:
    runs-on: ubuntu-latest

    steps:
      - name: Trigger vectordb-recipes workflow
        uses: actions/github-script@v6
        with:
          github-token: ${{ secrets.VECTORDB_RECIPES_ACTION_TOKEN }}
          script: |
            const result = await github.rest.actions.createWorkflowDispatch({
                owner: 'lancedb',
                repo: 'vectordb-recipes',
                workflow_id: 'examples-test.yml',
                ref: 'main'
            });
            console.log(result);


Update package-lock.json

name: Update package-lock.json

on:
  workflow_dispatch:

jobs:
  publish:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          ref: main
          persist-credentials: false
          fetch-depth: 0
          lfs: true
      - uses: ./.github/workflows/update_package_lock
        with:
          github_token: ${{ secrets.LANCEDB_RELEASE_TOKEN }}