Unit Tests

name: Unit Tests
env:
  # increment this when downloads substantially change to avoid the internet
  DOWNLOAD_CACHE_VERSION: '4'

on:
  push:
    branches:
      - master
  pull_request:
  workflow_dispatch:
    inputs:
       run_process_replay:
         description: "Run process replay tests"
         required: false
         default: false
         type: boolean

jobs:
  check_process_replay:
    name: Check process replay
    runs-on: ubuntu-latest
    outputs:
      run_process_replay: ${{ steps.set-env.outputs.run_process_replay }}
      assert_process_replay: ${{ steps.set-env.outputs.assert_process_replay }}
    timeout-minutes: 5

    steps:
    - name: Checkout Code
      uses: actions/checkout@v4
      with:
        fetch-depth: 2 # NOTE: this fetches the HEAD commit of the PR
    - name: Set process replay
      id: set-env
      run: |
        RAW_COMMIT_MESSAGE=$(git show -s --format=%B ${{ github.event.pull_request.head.sha }})
        COMMIT_MESSAGE=$(printf '%q' "$RAW_COMMIT_MESSAGE")
        RAW_PR_TITLE="${{ github.event.pull_request.title }}"
        PR_TITLE=$(printf '%q' "$RAW_PR_TITLE")
        if { echo "$COMMIT_MESSAGE" | grep -q "run_process_replay" || \
             [ "${{ github.event.inputs.run_process_replay }}" == "true" ] || \
             echo "$PR_TITLE" | grep -q "run_process_replay"; } && [ "$GITHUB_REF_NAME" != "master" ]; then
          echo "RUN_PROCESS_REPLAY=1" >> $GITHUB_OUTPUT
          echo "enabled process replay"
          if echo "$COMMIT_MESSAGE" | grep -q "no_assert"; then
            echo "ASSERT_PROCESS_REPLAY=0" >> $GITHUB_OUTPUT
            echo "running process replay in diff-only mode"
          else
            echo "ASSERT_PROCESS_REPLAY=1" >> $GITHUB_OUTPUT
          fi
        else
          echo "RUN_PROCESS_REPLAY=0" >> $GITHUB_OUTPUT
          echo "disabled process replay"
        fi

  uops:
    name: uops tests
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
    - name: Checkout Code
      uses: actions/checkout@v4
    - name: Set up Python 3.12
      uses: actions/setup-python@v5
      with:
        python-version: 3.12
    - name: Cache python packages
      uses: actions/cache@v4
      with:
        path: ${{ env.Python3_ROOT_DIR }}/lib/python3.12/site-packages
        key: uops-packages-${{ hashFiles('**/setup.py') }}-3.12
    - name: Install dependencies
      run: pip install -e '.[testing]' --extra-index-url https://download.pytorch.org/whl/cpu
    - name: Test IMAGE=2 support
      run: |
        IMAGE=2 PYTHON=1 python3 test/test_ops.py TestOps.test_gemm
        IMAGE=2 PYTHON=1 python3 test/test_ops.py TestOps.test_simple_conv2d
    - name: Test emulated METAL tensor cores
      run: DEBUG=2 EMULATE_METAL=1 FORWARD_ONLY=1 PYTHON=1 python3 test/test_ops.py TestOps.test_big_gemm
    - name: Test emulated AMD tensor cores
      run: |
        PYTHONPATH=. DEBUG=2 EMULATE_AMD=1 FORWARD_ONLY=1 PYTHON=1 N=16 HALF=1 ACC_HALF=0 python3 ./extra/gemm/simple_matmul.py
        PYTHONPATH=. DEBUG=2 EMULATE_AMD=1 FORWARD_ONLY=1 PYTHON=1 N=64 HALF=1 ACC_HALF=0 python3 ./extra/gemm/simple_matmul.py
        PYTHONPATH=. DEBUG=2 EMULATE_AMD=1 FORWARD_ONLY=1 PYTHON=1 N=16 HALF=1 ACC_HALF=1 python3 ./extra/gemm/simple_matmul.py
        PYTHONPATH=. DEBUG=2 EMULATE_AMD=1 FORWARD_ONLY=1 PYTHON=1 N=64 HALF=1 ACC_HALF=1 python3 ./extra/gemm/simple_matmul.py
    - name: Test emulated CUDA tensor cores
      run: DEBUG=2 EMULATE_CUDA=1 FORWARD_ONLY=1 PYTHON=1 python3 test/test_ops.py TestOps.test_gemm
    - name: Full test tensor cores
      run: |
        PYTHONPATH=. DEBUG=2 EMULATE_METAL=1 FORWARD_ONLY=1 PYTHON=1 python3 ./test/test_linearizer.py TestLinearizer.test_tensor_cores
        PYTHONPATH=. DEBUG=2 EMULATE_AMD=1 FORWARD_ONLY=1 PYTHON=1 python3 ./test/test_linearizer.py TestLinearizer.test_tensor_cores
        PYTHONPATH=. DEBUG=2 EMULATE_CUDA=1 FORWARD_ONLY=1 PYTHON=1 python3 ./test/test_linearizer.py TestLinearizer.test_tensor_cores
    - name: Test dtype with Python emulator
      run: DEBUG=1 PYTHONPATH=. PYTHON=1 python3 -m pytest test/test_dtype.py test/test_dtype_alu.py
    - name: Test ops with Python emulator
      run: DEBUG=2 PYTHON=1 python3 -m pytest test/test_ops.py -k "not (test_split or test_simple_cumsum or test_cumsum or test_einsum or test_dot or test_dot_1d or test_big_gemm or test_broadcastdot or test_multidot or test_var_axis or test_std_axis or test_broadcast_full or test_broadcast_partial or test_simple_conv3d or test_dilated_conv_transpose2d or test_simple_conv_transpose3d or test_large_input_conv2d or test_maxpool2d or test_maxpool2d_simple or test_maxpool2d_bigger_stride or test_avgpool2d or test_cat or test_scaled_product_attention or test_scaled_product_attention_causal or test_slice_fancy_indexing_dim_inject_none or test_slice_fancy_indexing_list_indices or test_slice_fancy_indexing_no_dim_collapse or test_slice_fancy_indexing_tuple_indices or test_slice_fancy_indexing_list_with_tensors or test_slice_fancy_indexing_dim_collapse_int)" --durations=20
    - name: Test uops with Python emulator
      run: PYTHON=1 python3 -m pytest test/test_uops.py --durations=20
    - name: Test symbolic with Python emulator
      run: PYTHONPATH=. PYTHON=1 python3 test/test_symbolic_ops.py
    - name: test_linearizer_failures with Python emulator
      run: PYTHONPATH=. PYTHON=1 python3 -m pytest -rA test/test_linearizer_failures.py::TestLinearizerFailures::test_failure_1

  linter:
    name: Linters
    runs-on: ubuntu-latest
    timeout-minutes: 10

    # TODO: run the pre-commit hook to replace a lot of this
    steps:
    - name: Checkout Code
      uses: actions/checkout@v4
    - name: Set up Python 3.8
      uses: actions/setup-python@v5
      with:
        python-version: 3.8
    - name: Cache python packages
      uses: actions/cache@v4
      with:
        path: ${{ env.Python3_ROOT_DIR }}/lib/python3.8/site-packages
        key: linting-packages-${{ hashFiles('**/setup.py') }}-3.8
    - name: Install dependencies
      run: pip install -e '.[linting,testing]' --extra-index-url https://download.pytorch.org/whl/cpu
    - name: Lint with pylint
      run: python -m pylint --disable=all -e W0311 -e C0303 --jobs=0 --indent-string='  ' **/*.py
    - name: Lint with ruff
      run: |
        pip3 install --upgrade --force-reinstall ruff
        python3 -m ruff check .
    - name: Lint tinygrad with pylint
      run: python -m pylint tinygrad/
    - name: Run mypy
      run: python -m mypy --strict-equality
    - name: Test Docs
      run: |
        python docs/abstractions2.py
    - name: Test Quickstart
      run: awk '/```python/{flag=1;next}/```/{flag=0}flag' docs/quickstart.md > quickstart.py &&  PYTHONPATH=. python quickstart.py
    - name: Fuzz Test symbolic
      run: python test/external/fuzz_symbolic.py
    - name: Fuzz Test shapetracker
      run: |
        PYTHONPATH="." python test/external/fuzz_shapetracker.py
        PYTHONPATH="." python test/external/fuzz_shapetracker_math.py
    - name: Test to_movement_ops
      run: PYTHONPATH="." python extra/to_movement_ops.py
    - name: Use as an external package
      run: |
        mkdir $HOME/test_external_dir
        cd $HOME/test_external_dir
        python -m venv venv
        source venv/bin/activate
        pip install $GITHUB_WORKSPACE
        python -c "from tinygrad.tensor import Tensor; print(Tensor([1,2,3,4,5]))"
    - name: Test DEBUG
      run: DEBUG=100 python3 -c "from tinygrad import Tensor; N = 1024; a, b = Tensor.rand(N, N), Tensor.rand(N, N); c = (a.reshape(N, 1, N) * b.T.reshape(1, N, N)).sum(axis=2); print((c.numpy() - (a.numpy() @ b.numpy())).mean())"
    - name: Repo line count <8000 lines
      run: MAX_LINE_COUNT=8000 python sz.py

  testopencl:
    strategy:
      fail-fast: false
      matrix:
        task: [optimage, openpilot, onnx]
    needs: check_process_replay
    #env:
      #RUN_PROCESS_REPLAY: ${{ needs.check_process_replay.outputs.run_process_replay }}

    name: ${{ matrix.task=='optimage'&&'GPU OPT and IMAGE Tests' || matrix.task=='openpilot'&&'openpilot (OpenCL) Tests' || matrix.task=='onnx'&&'ONNX+Optimization Tests' }}
    runs-on: ubuntu-20.04
    timeout-minutes: 10

    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
      - name: Install OpenCL
        run: |
          echo 'Acquire::http::Pipeline-Depth "5";' | sudo tee -a /etc/apt/apt.conf.d/99parallel
          echo "deb [ allow-insecure=yes ] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
          sudo apt update || true
          sudo apt install --allow-unauthenticated -y --no-install-recommends \
            intel-oneapi-runtime-openmp=2023.2.1-16 intel-oneapi-runtime-compilers-common=2023.2.1-16 intel-oneapi-runtime-compilers=2023.2.1-16 \
            intel-oneapi-runtime-dpcpp-sycl-opencl-cpu=2023.2.1-16 intel-oneapi-runtime-tbb-common=2021.10.0-49541 \
            intel-oneapi-runtime-tbb=2021.10.0-49541 intel-oneapi-runtime-opencl=2023.2.1-16
      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - name: Cache python packages
        uses: actions/cache@v4
        with:
          path: ${{ env.Python3_ROOT_DIR }}/lib/python3.11/site-packages
          key: testing-packages-${{ hashFiles('**/setup.py') }}
      - name: Cache downloads
        uses: actions/cache@v4
        with:
          path: ~/.cache/tinygrad/downloads/
          key: downloads-cache-${{ matrix.task }}-${{ env.DOWNLOAD_CACHE_VERSION }}
      - name: Install Dependencies
        run: pip install -e '.[testing,testing_tf]' --extra-index-url https://download.pytorch.org/whl/cpu
      - if: ${{ matrix.task == 'optimage' }}
        name: Run Kernel Count Test
        run: PYTHONPATH="." GPU=1 python -m pytest -n=auto test/external/external_test_opt.py
      - if: ${{ matrix.task == 'optimage'}}
        name: Test WINO=1
        run: GPU=1 DEBUG=2 WINO=1 python3 test/test_ops.py TestOps.test_simple_conv2d
      - if: ${{ matrix.task == 'optimage'}}
        name: Test GPU IMAGE=1 ops
        run: GPU=1 IMAGE=1 python -m pytest -n=auto test/test_ops.py
      - if: ${{ matrix.task == 'optimage'}}
        name: Test GPU IMAGE=2 ops
        run: GPU=1 IMAGE=2 python -m pytest -n=auto test/test_ops.py
      - if: ${{ matrix.task == 'openpilot' }}
        name: Test openpilot model compile and size
        run: |
          PYTHONPATH="." DEBUG=2 ALLOWED_KERNEL_COUNT=208 FLOAT16=1 DEBUGCL=1 GPU=1 IMAGE=2 python examples/openpilot/compile2.py
          python -c 'import os; assert os.path.getsize("/tmp/output.thneed") < 100_000_000'
      - if: ${{ matrix.task == 'openpilot' }}
        name: Test openpilot model correctness (float32)
        run: PYTHONPATH="." FLOAT16=0 DEBUGCL=1 GPU=1 IMAGE=2 python examples/openpilot/compile2.py
      - if: ${{ matrix.task == 'openpilot' }}
        name: Test openpilot alt model correctness (float32)
        run: PYTHONPATH="." FLOAT16=0 DEBUGCL=1 GPU=1 IMAGE=2 python examples/openpilot/compile2.py https://github.com/commaai/openpilot/raw/3799fe46b3a629e491d4b8498b8ae83e4c88c304/selfdrive/modeld/models/supercombo.onnx
      - if: ${{ matrix.task == 'openpilot' }}
        name: Test openpilot fastvits model correctness (float32)
        run: PYTHONPATH="." FLOAT16=0 DEBUGCL=1 GPU=1 IMAGE=2 python examples/openpilot/compile2.py https://github.com/commaai/openpilot/raw/9118973ed03c1ae1d40cf69a29507ec2cc78efd7/selfdrive/modeld/models/supercombo.onnx
      - if: ${{ matrix.task == 'onnx' }}
        name: Test ONNX (GPU)
        run: GPU=1 python -m pytest -n=auto test/external/external_test_onnx_backend.py --durations=20
      - if: ${{ matrix.task == 'onnx' }}
        name: Test ONNX (CLANG)
        run: CLANG=1 python -m pytest -n=auto test/external/external_test_onnx_backend.py --durations=20
      - if: ${{ matrix.task == 'onnx' }}
        name: Test Action Space
        run: PYTHONPATH="." GPU=1 python3 extra/optimization/get_action_space.py
      - if: ${{ matrix.task == 'onnx' }}
        name: Test Beam Search
        run: PYTHONPATH="." GPU=1 IGNORE_BEAM_CACHE=1 python3 -m pytest extra/optimization/test_beam_search.py
      - if: ${{ matrix.task == 'onnx' }}
        name: Test MLPerf optimizers
        run: GPU=1 python -m pytest -n=auto test/external/external_test_optim.py --durations=20
      - if: ${{ matrix.task == 'onnx' }}
        name: Test MLPerf losses
        run: GPU=1 python -m pytest -n=auto test/external/external_test_losses.py --durations=20
      - if: ${{ matrix.task == 'onnx' }}
        name: Test MLPerf metrics
        run: GPU=1 python -m pytest -n=auto test/external/external_test_metrics.py --durations=20
      - if: ${{ matrix.task == 'onnx' }}
        name: Test MLPerf datasets
        run: GPU=1 python -m pytest -n=auto test/external/external_test_datasets.py --durations=20
      - if: ${{ matrix.task == 'onnx' }}
        name: Test THREEFRY
        run: PYTHONPATH=. THREEFRY=1 GPU=1 python3 -m pytest test/test_randomness.py test/test_jit.py --durations=20
      - name: Run process replay tests
        if: env.RUN_PROCESS_REPLAY == '1'
        run: cp test/external/replay_codegen.py ./replay_codegen.py && git fetch origin master && git checkout origin/master && PYTHONPATH=. python3 replay_codegen.py

  #testwebgpu:
  #  name: WebGPU Tests
  #  runs-on: macos-13
  #  timeout-minutes: 10
  #  steps:
  #  - name: Checkout Code
  #    uses: actions/checkout@v4
  #  - name: Set up Python 3.11
  #    uses: actions/setup-python@v5
  #    with:
  #      python-version: 3.11
  #  - name: Cache python packages
  #    uses: actions/cache@v4
  #    with:
  #      path: /Users/runner/Library/Python/3.11/lib/python/site-packages
  #      key: webgpu-testing-user3-packages-${{ hashFiles('**/setup.py') }}
  #  - name: Install Dependencies
  #    run: pip install --user -e '.[webgpu,testing]' --extra-index-url https://download.pytorch.org/whl/cpu
  #  - name: Cache downloads
  #    uses: actions/cache@v4
  #    with:
  #      path: ~/Library/Caches/tinygrad/downloads/
  #      key: downloads-cache-webgpu-${{ env.DOWNLOAD_CACHE_VERSION }}
  #  - name: Check Device.DEFAULT (WEBGPU) and print some source
  #    run: |
  #      WEBGPU=1 python -c "from tinygrad import Device; assert Device.DEFAULT == 'WEBGPU', Device.DEFAULT"
  #      WEBGPU=1 DEBUG=4 FORWARD_ONLY=1 python3 test/test_ops.py TestOps.test_add
    #- name: Run webgpu pytest
    #  run: WEBGPU=1 WGPU_BACKEND_TYPE=Metal python -m pytest -n=auto
  #  - name: Run selected webgpu tests
  #    run: |
  #      WEBGPU=1 WGPU_BACKEND_TYPE=Metal python -m pytest -n=auto test/test_ops.py test/test_dtype.py \
  #      test/test_jit.py test/test_symbolic_ops.py test/test_symbolic_jit.py test/test_linearizer.py \
  #      test/test_linearizer_failures.py test/test_nn.py
  #  - name: Build WEBGPU Efficientnet
  #    run: WEBGPU=1 WGPU_BACKEND_TYPE=Metal python -m examples.compile_efficientnet
  #  - name: Install Puppeteer
  #    run: npm install puppeteer
  #  - name: Run WEBGPU Efficientnet
  #    run: node test/web/test_webgpu.js

  testmetal:
    name: Metal Tests
    runs-on: macos-14
    needs: check_process_replay
    env:
      RUN_PROCESS_REPLAY: ${{ needs.check_process_replay.outputs.run_process_replay }}
      ASSERT_PROCESS_REPLAY: ${{ needs.check_process_replay.outputs.assert_process_replay }}
    timeout-minutes: 20

    steps:
    - name: Checkout Code
      uses: actions/checkout@v4
    - name: Set up Python 3.11
      uses: actions/setup-python@v5
      with:
        python-version: 3.11
    - name: Cache python packages
      uses: actions/cache@v4
      with:
        path: /Users/runner/Library/Python/3.11/lib/python/site-packages
        key: metal-m1-testing-user3-packages-${{ hashFiles('**/setup.py') }}
    - name: Install Dependencies
      run: pip install --user -e '.[testing]' --extra-index-url https://download.pytorch.org/whl/cpu
    - name: Cache downloads
      uses: actions/cache@v4
      with:
        path: ~/Library/Caches/tinygrad/downloads/
        key: downloads-cache-metal-only-${{ env.DOWNLOAD_CACHE_VERSION }}
    - name: Check Device.DEFAULT (METAL) and print some source
      run: |
        METAL=1 python -c "from tinygrad import Device; assert Device.DEFAULT == 'METAL', Device.DEFAULT"
        METAL=1 DEBUG=4 FORWARD_ONLY=1 python3 test/test_ops.py TestOps.test_add
    - name: Run metal test
      run: JIT=2 METAL=1 python -m pytest -n=auto test/ --ignore=test/external --ignore=test/models --durations=20
    - name: Run real world test
      run: JIT=2 METAL=1 python -m pytest -n=auto test/models/test_real_world.py --durations=20
    - name: Run ONNX
      run: JIT=2 METAL=1 python -m pytest -n=auto test/external/external_test_onnx_backend.py --durations=20
    - name: Test tensor core ops (fake)
      run: TC=2 METAL=1 DEBUG=3 python test/test_ops.py TestOps.test_gemm
    - name: Test tensor core ops (real)
      run: METAL=1 DEBUG=3 python test/test_ops.py TestOps.test_big_gemm
    - name: Test LLaMA compile speed
      run: PYTHONPATH="." METAL=1 python test/external/external_test_speed_llama.py
    - name: Test Beam Search
      run: PYTHONPATH="." METAL=1 IGNORE_BEAM_CACHE=1 python3 -m pytest extra/optimization/test_beam_search.py
    - name: Fuzz Test linearizer
      run: PYTHONPATH="." METAL=1 CACHELEVEL=0 FUZZ_ALL_ACTIONS=1 DEPTH=2 FUZZ_N=48 FUZZ_MAX_SIZE=10000000 python test/external/fuzz_linearizer.py
    - name: Fuzz Test models schedule
      run: FUZZ_SCHEDULE=1 FUZZ_SCHEDULE_MAX_PATHS=5 python -m pytest test/models/test_train.py test/models/test_end2end.py
    - name: Run process replay tests
      if: env.RUN_PROCESS_REPLAY == '1'
      run: cp test/external/replay_codegen.py ./replay_codegen.py && git fetch origin master && git checkout origin/master && PYTHONPATH=. python3 replay_codegen.py

#  testwebgl:
#    name: WebGL Tests
#    runs-on: ubuntu-latest
#    timeout-minutes: 10
#
#    steps:
#    - name: Checkout Code
#      uses: actions/checkout@v3
#    - name: Set up Python 3.11
#      uses: actions/setup-python@v4
#      with:
#        python-version: 3.11
#    - name: Cache python packages
#      uses: actions/cache@v4
#      with:
#        path: ${{ env.Python3_ROOT_DIR }}/lib/python3.11/site-packages
#        key: webgl-testing-packages-${{ hashFiles('**/setup.py') }}
#    - name: Install Dependencies
#      run: pip install -e '.[webgl,testing]' --extra-index-url https://download.pytorch.org/whl/cpu
#    - name: Cache downloads
#      uses: actions/cache@v4
#      with:
#        path: ~/Library/Caches/tinygrad/downloads/
#        key: downloads-cache-webgl-${{ env.DOWNLOAD_CACHE_VERSION }}
#    - name: Prepare
#      run: |
#        sudo apt-get -y install xvfb
#        sudo /usr/bin/Xvfb :0 -screen 0 4096x4096x24+32 &
#    - name: Run selected webgl tests
#      run: WEBGL=1 python -m pytest -n=auto test/test_ops.py test/test_dtype.py test/test_jit.py
#    - name: Build WebGL Efficientnet
#      run: WEBGL=1 python -m examples.compile_efficientnet

  tests:
    strategy:
      fail-fast: false
      matrix:
        backend: [llvm, clang, gpu, ptx, amd, nv] #, triton]
    needs: check_process_replay
    env:
      RUN_PROCESS_REPLAY: ${{ needs.check_process_replay.outputs.run_process_replay }}
      ASSERT_PROCESS_REPLAY: ${{ needs.check_process_replay.outputs.assert_process_replay }}

    name: Tests on (${{ matrix.backend }})
    runs-on: ubuntu-latest
    timeout-minutes: 20

    steps:
      - name: Checkout Code
        uses: actions/checkout@v4
      - name: Set up Python 3.11
        uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - name: Cache python packages
        uses: actions/cache@v4
        with:
          path: ${{ env.Python3_ROOT_DIR }}/lib/python3.11/site-packages
          key: ${{ matrix.backend }}-packages-${{ hashFiles('**/setup.py') }}
      - name: Cache downloads
        uses: actions/cache@v4
        with:
          path: ~/.cache/tinygrad/downloads/
          key: downloads-cache-${{ matrix.backend }}-${{ env.DOWNLOAD_CACHE_VERSION }}
      - name: Set env
        run: printf "${{ matrix.backend == 'llvm' && 'LLVM=1' || matrix.backend == 'clang' && 'CLANG=1' || matrix.backend == 'gpu' && 'GPU=1' || matrix.backend == 'PTX' && 'FORWARD_ONLY=1\nJIT=1\nOPT=2\nCUDA=1\nCUDACPU=1\nPTX=1' || matrix.backend == 'triton' && 'FORWARD_ONLY=1\nJIT=1\nOPT=2\nCUDA=1\nCUDACPU=1\nTRITON=1\nTRITON_PTXAS_PATH=/usr/bin/ptxas' || matrix.backend == 'amd' && 'AMD=1\nMOCKGPU=1\nFORWARD_ONLY=1' || matrix.backend == 'nv' && 'NV=1\nMOCKGPU=1\nFORWARD_ONLY=1' }}" >> $GITHUB_ENV
      - name: Install OpenCL
        if: matrix.backend == 'gpu'
        run: |
          echo 'Acquire::http::Pipeline-Depth "5";' | sudo tee -a /etc/apt/apt.conf.d/99parallel
          echo "deb [ allow-insecure=yes ] https://apt.repos.intel.com/oneapi all main" | sudo tee /etc/apt/sources.list.d/oneAPI.list
          sudo apt update || true
          sudo apt install --allow-unauthenticated -y --no-install-recommends opencl-headers \
            intel-oneapi-runtime-openmp=2023.2.1-16 intel-oneapi-runtime-compilers-common=2023.2.1-16 intel-oneapi-runtime-compilers=2023.2.1-16 \
            intel-oneapi-runtime-dpcpp-sycl-opencl-cpu=2023.2.1-16 intel-oneapi-runtime-tbb-common=2021.10.0-49541 \
            intel-oneapi-runtime-tbb=2021.10.0-49541 intel-oneapi-runtime-opencl=2023.2.1-16
      - name: Install packages (cuda)
        if: matrix.backend == 'ptx' || matrix.backend == 'triton' || matrix.backend == 'nv'
        run: |
          echo 'Acquire::http::Pipeline-Depth "5";' | sudo tee -a /etc/apt/apt.conf.d/99parallel
          sudo apt update -y || true
          sudo apt install -y --no-install-recommends git g++ cmake ninja-build llvm-15-dev zlib1g-dev libglew-dev \
            flex bison libfl-dev libboost-thread-dev libboost-filesystem-dev nvidia-cuda-toolkit-gcc libzstd-dev
      - name: Cache gpuocelot
        if: matrix.backend == 'ptx' || matrix.backend == 'triton' || matrix.backend == 'nv'
        id: cache-build
        uses: actions/cache@v4
        env:
          cache-name: cache-gpuocelot-build
        with:
          path: ${{ github.workspace }}/gpuocelot/ocelot
          key: ubuntu22.04-gpuocelot-4524e34adb7eaccc6f71262f2e21d7052bb17c2f-rebuild-6
      - name: Clone/compile gpuocelot
        if: (matrix.backend == 'ptx' || matrix.backend == 'triton' || matrix.backend == 'nv') && steps.cache-build.outputs.cache-hit != 'true'
        run: |
          git clone --recurse-submodules https://github.com/gpuocelot/gpuocelot.git ${{ github.workspace }}/gpuocelot
          cd ${{ github.workspace }}/gpuocelot/ocelot
          git checkout 4524e34adb7eaccc6f71262f2e21d7052bb17c2f
          mkdir build
          cd build
          cmake .. -Wno-dev -G Ninja -DOCELOT_BUILD_TOOLS=OFF -DCMAKE_BUILD_ALWAYS=0 -DBUILD_TESTS_CUDA=OFF
          ninja
      - name: Install gpuocelot
        if: matrix.backend == 'ptx' || matrix.backend == 'triton' || matrix.backend == 'nv'
        run: |
          cd ${{ github.workspace }}/gpuocelot/ocelot/build
          sudo ninja install -d explain
      - name: Install packages (amd)
        if: matrix.backend == 'amd'
        run: |
          echo 'Acquire::http::Pipeline-Depth "5";' | sudo tee -a /etc/apt/apt.conf.d/99parallel
          wget https://repo.radeon.com/rocm/rocm.gpg.key -O - | gpg --dearmor | sudo tee /etc/apt/keyrings/rocm.gpg > /dev/null
          sudo tee /etc/apt/sources.list.d/rocm.list <<'EOF'
          deb [arch=amd64 signed-by=/etc/apt/keyrings/rocm.gpg] https://repo.radeon.com/rocm/apt/6.1.2 jammy main
          EOF
          echo -e 'Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' | sudo tee /etc/apt/preferences.d/rocm-pin-600
          sudo apt update || true
          sudo apt install --no-install-recommends --allow-unauthenticated -y hsa-rocr comgr hsa-rocr-dev liburing-dev
          curl -s https://api.github.com/repos/Qazalin/remu/releases/latest | \
          jq -r '.assets[] | select(.name == "libremu.so").browser_download_url' | \
          sudo xargs curl -L -o /usr/local/lib/libremu.so
          sudo tee --append /etc/ld.so.conf.d/rocm.conf <<'EOF'
            /opt/rocm/lib
            /opt/rocm/lib64
          EOF
          sudo ldconfig
      - name: Install dependencies
        run: pip install -e '.[testing${{matrix.backend=='llvm'&&',llvm'||matrix.backend=='ptx'&&',cuda'||matrix.backend=='triton'&&',triton'||''}}]' --extra-index-url https://download.pytorch.org/whl/cpu --extra-index-url https://aiinfra.pkgs.visualstudio.com/PublicPackages/_packaging/Triton-Nightly/pypi/simple/
      - name: Check Device.DEFAULT and print some source
        run: |
          PYTHONPATH=${{ github.workspace }} python3 -c "from tinygrad import Device; assert Device.DEFAULT in ['LLVM','CLANG','CUDA','GPU','AMD','NV'], Device.DEFAULT"
          DEBUG=5 PYTHONPATH=${{ github.workspace }} FORWARD_ONLY=1 python3 test/test_ops.py TestOps.test_add
      - name: Verify OpenCL autogen
        if: matrix.backend == 'gpu'
        run: |
          cp tinygrad/runtime/autogen/opencl.py /tmp/opencl.py.bak
          ./autogen_stubs.sh opencl
          diff /tmp/opencl.py.bak tinygrad/runtime/autogen/opencl.py
      - name: Verify CUDA autogen
        if: matrix.backend == 'nv'
        run: |
          cp tinygrad/runtime/autogen/cuda.py /tmp/cuda.py.bak
          cp tinygrad/runtime/autogen/nv_gpu.py /tmp/nv_gpu.py.bak
          ./autogen_stubs.sh cuda
          ./autogen_stubs.sh nv
          diff /tmp/cuda.py.bak tinygrad/runtime/autogen/cuda.py
          diff /tmp/nv_gpu.py.bak tinygrad/runtime/autogen/nv_gpu.py
      - name: Verify AMD autogen
        if: matrix.backend == 'amd'
        run: |
          cp tinygrad/runtime/autogen/hsa.py /tmp/hsa.py.bak
          cp tinygrad/runtime/autogen/comgr.py /tmp/comgr.py.bak
          cp tinygrad/runtime/autogen/amd_gpu.py /tmp/amd_gpu.py.bak
          ./autogen_stubs.sh hsa
          ./autogen_stubs.sh comgr
          ./autogen_stubs.sh amd
          diff /tmp/hsa.py.bak tinygrad/runtime/autogen/hsa.py
          diff /tmp/comgr.py.bak tinygrad/runtime/autogen/comgr.py
          diff /tmp/amd_gpu.py.bak tinygrad/runtime/autogen/amd_gpu.py
      - name: Verify Linux autogen
        if: matrix.backend == 'amd'
        run: |
          cp tinygrad/runtime/autogen/io_uring.py /tmp/io_uring.py.bak
          ./autogen_stubs.sh io_uring
          diff /tmp/io_uring.py.bak tinygrad/runtime/autogen/io_uring.py
      - name: Run pytest (not cuda or amd)
        if: matrix.backend!='ptx' && matrix.backend!='triton' && matrix.backend != 'amd' && matrix.backend != 'nv'
        run: python -m pytest -n=auto test/ --durations=20
      # - name: Run test_ops with FUZZ_UOPS=1
      #   if: matrix.backend!='cuda' && matrix.backend!='ptx' && matrix.backend!='triton' && matrix.backend != 'amd' && matrix.backend != 'nv'
      #   run: FUZZ_UOPS=1 python -m pytest -n=auto test/test_ops.py --durations=20
      - name: Run ONNX (only LLVM)
        if: matrix.backend == 'llvm'
        run: python -m pytest -n=auto test/external/external_test_onnx_backend.py --durations=20
      - name: Run pytest (cuda)
        if: matrix.backend=='ptx'||matrix.backend=='triton'||matrix.backend=='nv'
        run: python -m pytest -n=auto test/ -k 'not (half or test_efficientnet_safetensors)' --ignore=test/external --ignore=test/models --ignore test/test_gc.py --durations=20
      - name: Run pytest (amd)
        if: matrix.backend=='amd'
        run: python -m pytest -n=auto test/test_ops.py test/test_dtype.py test/test_dtype_alu.py test/test_linearizer.py test/test_randomness.py test/imported/test_indexing.py test/external/external_test_hcq.py --durations=20
      - name: Compile EfficientNet to C and test it
        if: matrix.backend=='clang'
        run: |
          PYTHONPATH="." python examples/compile_efficientnet.py > recognize.c
          clang -O2 recognize.c -lm -o recognize
          cat test/models/efficientnet/Chicken.jpg | ./recognize | grep cock
      - name: Run process replay tests
        if: env.RUN_PROCESS_REPLAY == '1'
        run: cp test/external/replay_codegen.py ./replay_codegen.py && git fetch origin master && git checkout origin/master && PYTHONPATH=. python3 replay_codegen.py

  #testunicorn:
  #  name: ARM64 unicorn Test
  #  runs-on: ubuntu-latest
  #  timeout-minutes: 10
  #  steps:
  #    - name: Checkout Code
  #      uses: actions/checkout@v4
  #    - name: Set up Python 3.11
  #      uses: actions/setup-python@v5
  #      with:
  #        python-version: 3.11
  #    - name: Cache python packages
  #      uses: actions/cache@v4
  #      with:
  #        path: ${{ env.Python3_ROOT_DIR }}/lib/python3.11/site-packages
  #        key: testing-arm-packages-${{ hashFiles('**/setup.py') }}
  #    - name: Install cross-assembler
  #      run: |
  #        sudo apt update -y
  #        sudo apt install -y --no-install-recommends gcc-aarch64-linux-gnu
  #    - name: Install dependencies
  #      run: pip install -e '.[testing,arm]' --extra-index-url https://download.pytorch.org/whl/cpu
  #    - name: Test arm
  #      run: CI=1 ARM64=1 CLANG=1 python -m pytest -n=auto test/ -k 'not (test_nn.py and (test_conv_transpose2d or test_conv2d))' --ignore=test/models --ignore=test/test_speed_v_torch.py --ignore=test/test_net_speed.py --ignore=test/test_specific_conv.py  --ignore=test/unit/test_disk_tensor.py


Upload Python Package

# This workflows will upload a Python Package using Twine when a release is created
# For more information see: https://help.github.com/en/actions/language-and-framework-guides/using-python-with-github-actions#publishing-to-package-registries

name: Upload Python Package

on:
  release:
    types: [published]
  workflow_dispatch:

jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
    - uses: actions/checkout@v4
    - name: Set up Python
      uses: actions/setup-python@v2
      with:
        python-version: '3.x'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install setuptools wheel twine
    - name: Build and publish
      env:
        TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
        TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
      run: |
        python setup.py sdist bdist_wheel
        twine upload dist/*


Benchmarks

name: Benchmarks

on:
  push:
    branches:
      - master
      - update_benchmark
  workflow_dispatch:
    inputs:
      run_process_replay:
        description: "Run process replay tests"
        required: false
        default: false
        type: boolean

jobs:
  testmacbenchmark:
    name: Mac Benchmark
    runs-on: [self-hosted, macOS]
    defaults:
      run:
        shell: bash -o pipefail {0}
    if: github.repository_owner == 'tinygrad'
    env:
      PYTHONPATH: .
    steps:
    - name: Checkout Code
      uses: actions/checkout@v4
    - name: Symlink models and datasets
      run: |
        mkdir -p weights
        ln -s ~/tinygrad/extra/disassemblers/applegpu extra/disassemblers/applegpu
        ln -s ~/tinygrad/weights/sd-v1-4.ckpt weights/sd-v1-4.ckpt
        ln -s ~/tinygrad/weights/bpe_simple_vocab_16e6.txt.gz weights/bpe_simple_vocab_16e6.txt.gz
        ln -s ~/tinygrad/weights/LLaMA weights/LLaMA
        ln -s ~/tinygrad/extra/datasets/cifar-10-python.tar.gz extra/datasets/cifar-10-python.tar.gz
    # - name: Setup process replay
    # if: github.event.inputs.run_process_replay == 'true' || contains(github.event.head_commit.message, '[run_process_replay]') || contains(github.event.pull_request.title, '[run_process_replay]')
    # run: echo "RUN_PROCESS_REPLAY=1" >> $GITHUB_ENV
    - name: Run Stable Diffusion
      run: JIT=2 THREEFRY=1 python3 examples/stable_diffusion.py --seed 0 --noshow --timing | tee sd.txt
    - name: Run model inference benchmark
      run: METAL=1 python3 test/external/external_model_benchmark.py
    - name: Test speed vs torch
      run: BIG=2 MPS=1 python3 test/test_speed_v_torch.py | tee torch_speed.txt
    - name: Test tensor cores
      run: METAL=1 python3 test/test_linearizer.py TestLinearizer.test_tensor_cores TestLinearizer.test_tensor_cores_padded
    - name: Run Tensor Core GEMM
      run: |
        DEBUG=2 python3 extra/gemm/simple_matmul.py | tee matmul.txt
        DEBUG=2 HALF=1 python3 extra/gemm/simple_matmul.py | tee matmul_half.txt
    - name: Fuzz Padded Tensor Core GEMM
      run: METAL=1 M_START=6 M_STOP=10 M_STEP=1 N_START=6 N_STOP=10 N_STEP=1 K_START=6 K_STOP=24 K_STEP=1 TC_OPT=2 DEBUG=2 python3 ./extra/gemm/fuzz_matmul.py
    - name: Run LLaMA
      run: |
        JIT=0 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_unjitted.txt
        JIT=1 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_jitted.txt
    - name: Run LLaMA with BEAM
      run: JIT=1 BEAM=2 CACHELEVEL=0 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_beam.txt
    - name: Run quantized LLaMA
      run: |
        JIT=1 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing --quantize int8 | tee llama_int8.txt
        JIT=1 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing --quantize nf4 | tee llama_nf4.txt
    - name: Run LLaMA 7B on 4 (virtual) GPUs
      run: JIT=1 python3 examples/llama.py --gen 1 --size 7B --shard 4 --prompt "Hello." --count 10 --temperature 0  --timing | tee llama_four_gpu.txt
    - name: Run GPT2
      run: |
        JIT=0 python3 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_unjitted.txt
        JIT=1 python3 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_jitted.txt
    - name: Run GPT2 w HALF
      run: JIT=1 HALF=1 python3 examples/gpt2.py --count 10 --temperature 0 --timing | tee gpt2_half.txt
    - name: Run GPT2 w HALF/BEAM
      run: JIT=1 HALF=1 BEAM=2 CACHELEVEL=0 CAST_BEFORE_VIEW=0 python3 examples/gpt2.py --count 10 --temperature 0 --timing | tee gpt2_half_beam.txt
    - name: Train MNIST
      run: time PYTHONPATH=. TARGET_EVAL_ACC_PCT=97.3 python3 examples/beautiful_mnist.py | tee beautiful_mnist.txt
    - name: Run 10 CIFAR training steps
      run: JIT=2 STEPS=10 python3 examples/hlb_cifar10.py | tee train_cifar.txt
    - name: Run 10 CIFAR training steps w HALF
      run: JIT=2 STEPS=10 DEFAULT_FLOAT=HALF python3 examples/hlb_cifar10.py | tee train_cifar_half.txt
    #- name: Run 10 CIFAR training steps w BF16
    #  run: STEPS=10 DEFAULT_FLOAT=BFLOAT16 python3 examples/hlb_cifar10.py | tee train_cifar_bf16.txt
    - name: Run 10 CIFAR training steps w winograd
      run: JIT=2 WINO=1 STEPS=10 python3 examples/hlb_cifar10.py | tee train_cifar_wino.txt
    - name: Run process replay tests
      if: env.RUN_PROCESS_REPLAY == '1'
      run: cp test/external/replay_codegen.py ./replay_codegen.py && git fetch origin master && git checkout origin/master && PYTHONPATH=. python3 replay_codegen.py
    - uses: actions/upload-artifact@v4
      with:
        name: Speed (Mac)
        path: |
          onnx_inference_speed.csv
          torch_speed.txt
          llama_unjitted.txt
          llama_jitted.txt
          llama_beam.txt
          llama_int8.txt
          llama_nf4.txt
          llama_four_gpu.txt
          gpt2_unjitted.txt
          gpt2_jitted.txt
          gpt2_half.txt
          gpt2_half_beam.txt
          matmul.txt
          matmul_half.txt
          sd.txt
          beautiful_mnist.txt
          train_cifar.txt
          train_cifar_half.txt
          train_cifar_bf16.txt
          train_cifar_wino.txt

  testnvidiabenchmark:
    name: tinybox green Benchmark
    runs-on: [self-hosted, Linux, tinyboxgreen]
    defaults:
      run:
        shell: bash -o pipefail {0}
    if: github.repository_owner == 'tinygrad'
    env:
      PYTHONPATH: .
    steps:
    - name: Checkout Code
      uses: actions/checkout@v4
    - name: Print nvidia-smi
      run: nvidia-smi
    - name: Symlink models and datasets
      run: |
        mkdir -p weights
        ln -s ~/tinygrad/weights/LLaMA weights/LLaMA
        ln -s /raid/weights/mixtral-8x7b-32kseqlen weights/mixtral-8x7b-32kseqlen
        ln -s /raid/weights/LLaMA-2 weights/LLaMA-2
        ln -s /raid/weights/LLaMA-3 weights/LLaMA-3
        mkdir -p extra/datasets
        ln -s /raid/datasets/imagenet extra/datasets/imagenet
    - name: Run model inference benchmark
      run: CUDA=1 NOCLANG=1 python3 test/external/external_model_benchmark.py
    - name: Test speed vs torch
      run: CUDA=1 BIG=2 TORCHCUDA=1 python3 test/test_speed_v_torch.py | tee torch_speed.txt
    - name: Test tensor cores
      run: |
        CUDA=1 python3 test/test_linearizer.py TestLinearizer.test_tensor_cores TestLinearizer.test_tensor_cores_padded
        PTX=1 python3 test/test_linearizer.py TestLinearizer.test_tensor_cores TestLinearizer.test_tensor_cores_padded
    - name: Run Tensor Core GEMM (CUDA)
      run: |
        CUDA=1 HALF=1 DEBUG=2 python3 extra/gemm/simple_matmul.py | tee matmul.txt
        CUDA=1 BFLOAT16=1 DEBUG=2 python3 extra/gemm/simple_matmul.py | tee matmul_bfloat16.txt
    - name: Run Tensor Core GEMM (CUDA) with BEAM
      run: BEAM=4 CUDA=1 HALF=1 CACHELEVEL=0 DEBUG=2 python3 extra/gemm/simple_matmul.py
    - name: Run Tensor Core GEMM (PTX)
      run: CUDA=1 PTX=1 HALF=1 DEBUG=2 python3 extra/gemm/simple_matmul.py | tee matmul_ptx.txt
    - name: Run Tensor Core GEMM (NV)
      run: NV=1 HALF=1 DEBUG=2 python3 extra/gemm/simple_matmul.py | tee matmul_nv.txt
    - name: Fuzz Padded Tensor Core GEMM(CUDA)
      run: CUDA=1 M_START=12 M_STOP=20 M_STEP=1 N_START=6 N_STOP=10 N_STEP=1 K_START=28 K_STOP=36 K_STEP=1 HALF=1 TC_OPT=2 python3 ./extra/gemm/fuzz_matmul.py
    - name: Fuzz Padded Tensor Core GEMM(PTX)
      run: CUDA=1 PTX=1 M_START=12 M_STOP=20 M_STEP=1 N_START=6 N_STOP=10 N_STEP=1 K_START=28 K_STOP=36 K_STEP=1 HALF=1 TC_OPT=2 python3 ./extra/gemm/fuzz_matmul.py
    - name: Run Stable Diffusion
      run: CUDA=1 THREEFRY=1 python3 examples/stable_diffusion.py --seed 0 --noshow --timing | tee sd.txt
    - name: Run LLaMA
      run: |
        CUDA=1 JIT=0 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_unjitted.txt
        CUDA=1 JIT=1 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_jitted.txt
    - name: Run LLaMA with BEAM
      run: CUDA=1 JIT=1 BEAM=2 CACHELEVEL=0 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_beam.txt
    - name: Run LLaMA 7B on 4 GPUs
      run: CUDA=1 python3 examples/llama.py --gen 1 --size 7B --shard 4 --prompt "Hello." --count 10 --temperature 0  --timing | tee llama_four_gpu.txt
    - name: Run LLaMA 7B on 6 GPUs
      run: CUDA=1 python3 examples/llama.py --gen 1 --size 7B --shard 6 --prompt "Hello." --count 10 --temperature 0  --timing | tee llama_six_gpu.txt
    - name: Run LLaMA-3 8B BEAM
      run: NV=1 JITBEAM=2 CACHELEVEL=0 python3 examples/llama3.py --model weights/LLaMA-3/8B-SF-DPO/ --benchmark | tee llama3_beam.txt
    - name: Run LLaMA-3 8B on 4 GPUs
      run: NV=1 python3 examples/llama3.py --shard 4 --model weights/LLaMA-3/8B-SF-DPO/ --benchmark | tee llama3_four_gpu.txt
    - name: Run LLaMA-3 8B on 6 GPUs
      run: NV=1 python3 examples/llama3.py --shard 6 --model weights/LLaMA-3/8B-SF-DPO/ --benchmark | tee llama3_six_gpu.txt
    # - name: Run LLaMA-2 70B
    #   run: CUDA=1 python3 examples/llama.py --gen 2 --size 70B --shard 6 --prompt "Hello." --count 10 --temperature 0  --timing | tee llama_2_70B.txt
    # - name: Run Mixtral 8x7B
    #   run: time CUDA=1 python3 examples/mixtral.py --temperature 0 --count 10 --timing | tee mixtral.txt
    - name: Run GPT2
      run: |
        CUDA=1 JIT=0 python3 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_unjitted.txt
        CUDA=1 JIT=1 python3 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_jitted.txt
    - name: Run GPT2 w HALF
      run: CUDA=1 JIT=1 HALF=1 python3 examples/gpt2.py --count 10 --temperature 0 --timing | tee gpt2_half.txt
    - name: Run GPT2 w HALF/BEAM
      run: CUDA=1 JIT=1 HALF=1 BEAM=2 CACHELEVEL=0 CAST_BEFORE_VIEW=0 JIT_BATCH_SIZE=4 python3 examples/gpt2.py --count 10 --temperature 0 --timing | tee gpt2_half_beam.txt
    - uses: actions/upload-artifact@v4
      with:
        name: Speed (NVIDIA)
        path: |
          onnx_inference_speed.csv
          torch_speed.txt
          matmul.txt
          matmul_bfloat16.txt
          matmul_ptx.txt
          matmul_nv.txt
          sd.txt
          llama_unjitted.txt
          llama_jitted.txt
          llama_beam.txt
          llama_four_gpu.txt
          llama_six_gpu.txt
          llama3_beam.txt
          llama3_four_gpu.txt
          llama3_six_gpu.txt
          # llama_2_70B.txt
          # mixtral.txt
          gpt2_unjitted.txt
          gpt2_jitted.txt
          gpt2_half.txt
          gpt2_half_beam.txt

  testmorenvidiabenchmark:
    name: tinybox green Training Benchmark
    runs-on: [self-hosted, Linux, tinyboxgreen]
    defaults:
      run:
        shell: bash -o pipefail {0}
    if: github.repository_owner == 'tinygrad'
    env:
      PYTHONPATH: .
    steps:
    - name: Checkout Code
      uses: actions/checkout@v4
    - name: Symlink models and datasets
      run: |
        mkdir -p weights
        ln -s ~/tinygrad/weights/bpe_simple_vocab_16e6.txt.gz weights/bpe_simple_vocab_16e6.txt.gz
        ln -s ~/tinygrad/weights/LLaMA weights/LLaMA
        ln -s ~/tinygrad/extra/datasets/cifar-10-python.tar.gz extra/datasets/cifar-10-python.tar.gz
        ln -s /raid/weights/mixtral-8x7b-32kseqlen weights/mixtral-8x7b-32kseqlen
        ln -s /raid/weights/LLaMA-2 weights/LLaMA-2
        mkdir -p extra/datasets
        ln -s /raid/datasets/imagenet extra/datasets/imagenet
    - name: Train MNIST
      run: time PYTHONPATH=. CUDA=1 TARGET_EVAL_ACC_PCT=97.3 python3 examples/beautiful_mnist.py | tee beautiful_mnist.txt
    - name: Run 10 CIFAR training steps
      run: CUDA=1 STEPS=10 python3 examples/hlb_cifar10.py | tee train_cifar.txt
    - name: Run 10 CIFAR training steps w HALF
      run: CUDA=1 STEPS=10 DEFAULT_FLOAT=HALF python3 examples/hlb_cifar10.py | tee train_cifar_half.txt
    - name: Run 10 CIFAR training steps w BF16
      run: CUDA=1 STEPS=10 DEFAULT_FLOAT=BFLOAT16 python3 examples/hlb_cifar10.py | tee train_cifar_bf16.txt
    - name: Run 10 CIFAR training steps w winograd
      run: CUDA=1 WINO=1 STEPS=10 python3 examples/hlb_cifar10.py | tee train_cifar_wino.txt
    - name: Run full CIFAR training w 1 GPU
      run: time CUDA=1 DEFAULT_FLOAT=HALF LATEWINO=1 STEPS=1000 TARGET_EVAL_ACC_PCT=93.2 python3 examples/hlb_cifar10.py | tee train_cifar_one_gpu.txt
    - name: Run full CIFAR training steps w 6 GPUS
      run: time CUDA=1 DEFAULT_FLOAT=HALF STEPS=350 BS=1536 GPUS=6 TARGET_EVAL_ACC_PCT=93.2 python3 examples/hlb_cifar10.py | tee train_cifar_six_gpu.txt
    - name: Run MLPerf resnet eval on training data
      run: time CUDA=1 MODEL=resnet python3 examples/mlperf/model_eval.py
    - name: Run 10 MLPerf ResNet50 training steps (1 gpu)
      run: CUDA=1 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=256 GPUS=1 MODEL=resnet python3 examples/mlperf/model_train.py | tee train_resnet_one_gpu.txt
    - name: Run 10 MLPerf ResNet50 training steps (6 gpu)
      run: CUDA=1 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=1536 GPUS=6 MODEL=resnet python3 examples/mlperf/model_train.py | tee train_resnet.txt
    - uses: actions/upload-artifact@v4
      with:
        name: Speed (NVIDIA Training)
        path: |
          beautiful_mnist.txt
          train_cifar.txt
          train_cifar_half.txt
          train_cifar_bf16.txt
          train_cifar_wino.txt
          train_cifar_one_gpu.txt
          train_resnet.txt
          train_resnet_one_gpu.txt
          train_cifar_six_gpu.txt

  testamdbenchmark:
    name: tinybox red Benchmark
    runs-on: [self-hosted, Linux, tinybox]
    defaults:
      run:
        shell: bash -o pipefail {0}
    if: github.repository_owner == 'tinygrad'
    env:
      PYTHONPATH: .
    steps:
    - name: Checkout Code
      uses: actions/checkout@v4
    - name: Symlink models and datasets
      run: |
        mkdir -p weights
        ln -s ~/tinygrad/weights/bpe_simple_vocab_16e6.txt.gz weights/bpe_simple_vocab_16e6.txt.gz
        ln -s ~/tinygrad/weights/LLaMA weights/LLaMA
        ln -s ~/tinygrad/extra/datasets/cifar-10-python.tar.gz extra/datasets/cifar-10-python.tar.gz
        ln -s /raid/weights/mixtral-8x7b-32kseqlen weights/mixtral-8x7b-32kseqlen
        ln -s /raid/weights/LLaMA-2 weights/LLaMA-2
        ln -s /raid/weights/LLaMA-3 weights/LLaMA-3
        mkdir -p extra/datasets
        ln -s /raid/datasets/imagenet extra/datasets/imagenet
    - name: Show off tinybox
      run: /opt/rocm/bin/rocm-bandwidth-test
    # TODO: unstable on AMD
    #- name: Run model inference benchmark
    #  run: LD_PRELOAD="/opt/rocm/lib/libhsa-runtime64.so" HSA=1 NOCLANG=1 python3 test/external/external_model_benchmark.py
    # TODO: unstable on AMD
    #- name: Test speed vs torch
    #  run: |
    #    python3 -c "import torch; print(torch.__version__)"
    #    LD_PRELOAD="/opt/rocm/lib/libhsa-runtime64.so" HSA=1 BIG=2 TORCHCUDA=1 python3 test/test_speed_v_torch.py | tee torch_speed.txt
    - name: Test tensor cores
      run: |
        AMD=1 python3 test/test_linearizer.py TestLinearizer.test_tensor_cores TestLinearizer.test_tensor_cores_padded
    - name: Run Tensor Core GEMM (AMD)
      run: AMD=1 HALF=1 DEBUG=2 python3 extra/gemm/simple_matmul.py | tee matmul_amd.txt
    # TODO: AMD compiler bug causes this to fail
    #- name: Fuzz Padded Tensor Core GEMM
    #  run: HSA=1 M_START=12 M_STOP=20 M_STEP=1 N_START=12 N_STOP=20 N_STEP=1 K_START=28 K_STOP=36 K_STEP=1 HALF=1 TC_OPT=2 DEBUG=2 python3 ./extra/gemm/fuzz_matmul.py
    - name: Run Stable Diffusion
      run: AMD=1 THREEFRY=1 python3 examples/stable_diffusion.py --seed 0 --noshow --timing | tee sd.txt
    - name: Run LLaMA 7B
      run: |
        AMD=1 JIT=0 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_unjitted.txt
        AMD=1 JIT=1 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_jitted.txt
    - name: Run LLaMA 7B with BEAM
      run: AMD=1 JIT=1 BEAM=2 CACHELEVEL=0 python3 examples/llama.py --gen 1 --prompt "Hello." --count 10 --temperature 0 --timing | tee llama_beam.txt
    - name: Run LLaMA 7B on 4 GPUs
      run: AMD=1 python3 examples/llama.py --gen 1 --size 7B --shard 4 --prompt "Hello." --count 10 --temperature 0  --timing | tee llama_four_gpu.txt
    - name: Run LLaMA 7B on 6 GPUs
      run: AMD=1 python3 examples/llama.py --gen 1 --size 7B --shard 6 --prompt "Hello." --count 10 --temperature 0  --timing | tee llama_six_gpu.txt
    - name: Run LLaMA-3 8B BEAM
      run: AMD=1 JITBEAM=2 CACHELEVEL=0 python3 examples/llama3.py --model weights/LLaMA-3/8B-SF-DPO/ --benchmark | tee llama3_beam.txt
    - name: Run LLaMA-3 8B on 4 GPUs
      run: AMD=1 python3 examples/llama3.py --shard 4 --model weights/LLaMA-3/8B-SF-DPO/ --benchmark | tee llama3_four_gpu.txt
    - name: Run LLaMA-3 8B on 6 GPUs
      run: AMD=1 python3 examples/llama3.py --shard 6 --model weights/LLaMA-3/8B-SF-DPO/ --benchmark | tee llama3_six_gpu.txt
    - name: Run LLaMA-2 70B
      run: AMD=1 python3 examples/llama.py --gen 2 --size 70B --shard 6 --prompt "Hello." --count 10 --temperature 0  --timing | tee llama_2_70B.txt
    - name: Run Mixtral 8x7B
      run: time AMD=1 python3 examples/mixtral.py --temperature 0 --count 10 --timing | tee mixtral.txt
    - name: Run GPT2
      run: |
        AMD=1 JIT=0 python3 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_unjitted.txt
        AMD=1 JIT=1 python3 examples/gpt2.py --prompt "Hello." --count 10 --temperature 0 --timing | tee gpt2_jitted.txt
    - name: Run GPT2 w HALF
      run: AMD=1 JIT=1 HALF=1 python3 examples/gpt2.py --count 10 --temperature 0 --timing | tee gpt2_half.txt
    - name: Run GPT2 w HALF/BEAM
      run: AMD=1 JIT=1 HALF=1 BEAM=2 CACHELEVEL=0 CAST_BEFORE_VIEW=0 python3 examples/gpt2.py --count 10 --temperature 0 --timing | tee gpt2_half_beam.txt
    - uses: actions/upload-artifact@v4
      with:
        name: Speed (AMD)
        path: |
          onnx_inference_speed.csv
          torch_speed.txt
          llama_unjitted.txt
          llama_jitted.txt
          llama_beam.txt
          llama_four_gpu.txt
          llama_six_gpu.txt
          llama3_beam.txt
          llama3_four_gpu.txt
          llama3_six_gpu.txt
          llama_2_70B.txt
          gpt2_unjitted.txt
          gpt2_jitted.txt
          gpt2_half.txt
          gpt2_half_beam.txt
          matmul.txt
          matmul_amd.txt
          sd.txt
          mixtral.txt

  testmoreamdbenchmark:
    name: tinybox red Training Benchmark
    runs-on: [self-hosted, Linux, tinybox]
    defaults:
      run:
        shell: bash -o pipefail {0}
    if: github.repository_owner == 'tinygrad'
    env:
      PYTHONPATH: .
    steps:
    - name: Checkout Code
      uses: actions/checkout@v4
    - name: Symlink models and datasets
      run: |
        mkdir -p weights
        ln -s ~/tinygrad/weights/bpe_simple_vocab_16e6.txt.gz weights/bpe_simple_vocab_16e6.txt.gz
        ln -s ~/tinygrad/weights/LLaMA weights/LLaMA
        ln -s ~/tinygrad/extra/datasets/cifar-10-python.tar.gz extra/datasets/cifar-10-python.tar.gz
        ln -s /raid/weights/mixtral-8x7b-32kseqlen weights/mixtral-8x7b-32kseqlen
        ln -s /raid/weights/LLaMA-2 weights/LLaMA-2
        mkdir -p extra/datasets
        ln -s /raid/datasets/imagenet extra/datasets/imagenet
    - name: Train MNIST
      run: time PYTHONPATH=. AMD=1 TARGET_EVAL_ACC_PCT=97.3 python3 examples/beautiful_mnist.py | tee beautiful_mnist.txt
    - name: Run 10 CIFAR training steps
      run: AMD=1 STEPS=10 python3 examples/hlb_cifar10.py | tee train_cifar.txt
    - name: Run 10 CIFAR training steps w HALF
      run: AMD=1 STEPS=10 DEFAULT_FLOAT=HALF python3 examples/hlb_cifar10.py | tee train_cifar_half.txt
    - name: Run 10 CIFAR training steps w BF16
      run: AMD=1 STEPS=10 DEFAULT_FLOAT=BFLOAT16 python3 examples/hlb_cifar10.py | tee train_cifar_bf16.txt
    - name: Run 10 CIFAR training steps w winograd
      run: AMD=1 WINO=1 STEPS=10 python3 examples/hlb_cifar10.py | tee train_cifar_wino.txt
    - name: Run full CIFAR training w 1 GPU
      run: time AMD=1 DEFAULT_FLOAT=HALF LATEWINO=1 STEPS=1000 TARGET_EVAL_ACC_PCT=93.2 python3 examples/hlb_cifar10.py | tee train_cifar_one_gpu.txt
    - name: Run full CIFAR training steps w 6 GPUS
      run: time AMD=1 DEFAULT_FLOAT=HALF STEPS=350 BS=1536 GPUS=6 TARGET_EVAL_ACC_PCT=93.2 python3 examples/hlb_cifar10.py | tee train_cifar_six_gpu.txt
    - name: Run MLPerf resnet eval
      run: time AMD=1 MODEL=resnet python3 examples/mlperf/model_eval.py
    - name: Run 10 MLPerf ResNet50 training steps (1 gpu)
      run: AMD=1 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=256 GPUS=1 MODEL=resnet python3 examples/mlperf/model_train.py | tee train_resnet_one_gpu.txt
    - name: Run 10 MLPerf ResNet50 training steps (6 gpu)
      run: AMD=1 DEFAULT_FLOAT=HALF BENCHMARK=10 BS=1536 GPUS=6 MODEL=resnet python3 examples/mlperf/model_train.py | tee train_resnet.txt
    - uses: actions/upload-artifact@v4
      with:
        name: Speed (AMD Training)
        path: |
          beautiful_mnist.txt
          train_cifar.txt
          train_cifar_half.txt
          train_cifar_bf16.txt
          train_cifar_wino.txt
          train_cifar_one_gpu.txt
          train_resnet.txt
          train_resnet_one_gpu.txt
          train_cifar_six_gpu.txt


Check Line Counts

name: Check Line Counts
on:
  pull_request_target:

# Cancel the workflow in progress in newer build is about to start.
concurrency:
  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
  cancel-in-progress: true

jobs:
  checkbranch:
    name: Check PR Branch status
    runs-on: ubuntu-latest
    outputs:
      branchstat: ${{ steps.brstat.outputs.stat}}
    steps:
      - name: Check code from PR branch
        uses: actions/checkout@v4
        with:
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          ref: ${{ github.event.pull_request.head.sha }}
          fetch-depth: 0
      - name: Check whether branch is up-to-date
        id: brstat
        run: |
          git remote add tinygrad https://github.com/tinygrad/tinygrad
          git fetch tinygrad master
          echo "${{ github.event.pull_request.head.sha }}"
          git rev-list --left-right --count  tinygrad/master...${{ github.event.pull_request.head.sha }} | awk '{print "Behind "$1" - Ahead "$2""}'
          count=$(git rev-list --left-right --count  tinygrad/master...${{ github.event.pull_request.head.sha }} | awk '{print $1}')
          if [ $count -gt 0 ]
          then
            echo "Current branch is behind tinygrad master branch!"
            echo "stat=true" >> "$GITHUB_OUTPUT"
          else
            echo "stat=false" >> "$GITHUB_OUTPUT"
          fi

  szdiff:
    name: Core Library Line Difference
    permissions:
      contents: read
      pull-requests: write
    runs-on: ubuntu-latest
    needs: checkbranch
    if: needs.checkbranch.outputs.branchstat == 'false'
    steps:
      - name: Checkout code from PR branch
        uses: actions/checkout@v4
        with:
          repository: ${{ github.event.pull_request.head.repo.full_name }}
          ref: ${{ github.event.pull_request.head.sha }}
          path: pr
        # the base default to tinygrad master and cannot be other fork branch for security purpose
      - name: Checkout code from tinygrad master
        uses: actions/checkout@v4
        with:
          path: base
      - name: Set up Python 3.10
        uses: actions/setup-python@v5
        with:
          python-version: '3.10'
      - name: Count Line Diff
        run: |
          pip install tabulate
          BASE="$GITHUB_WORKSPACE/base"
          PR="$GITHUB_WORKSPACE/pr"
          cp "$BASE/sz.py" .
          echo "loc_content<<EOF" >> "$GITHUB_ENV"
          python sz.py "$BASE" "$PR" >> "$GITHUB_ENV"
          echo "EOF" >> "$GITHUB_ENV"
      - name: Comment Code Line Diff
        continue-on-error: false
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          ignore_empty: true
          skip_unchanged: true
          recreate: true
          message: ${{ env.loc_content }}

  rebase:
    name: Core Library Line Difference
    permissions:
      pull-requests: write
    runs-on: ubuntu-latest
    needs: checkbranch
    if: needs.checkbranch.outputs.branchstat == 'true'
    steps:
      - name: Comment Rebase
        continue-on-error: false
        uses: marocchino/sticky-pull-request-comment@v2
        with:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
          skip_unchanged: true
          recreate: true
          message: |
            This branch currently is behind tinygrad/master. The line count difference bot is disabled.


Deploy Docs

name: Deploy Docs
on:
  push:
    branches:
      - master
      - mkdocs
permissions:
  contents: write
jobs:
  deploy:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Configure Git Credentials
        run: |
          git config user.name github-actions[bot]
          git config user.email 41898282+github-actions[bot]@users.noreply.github.com
      - uses: actions/setup-python@v5
        with:
          python-version: 3.x
      - run: echo "cache_id=$(date --utc '+%V')" >> $GITHUB_ENV
      - uses: actions/cache@v4
        with:
          key: mkdocs-material-${{ env.cache_id }}
          path: .cache
          restore-keys: |
            mkdocs-material-
      - run: pip install -e .[docs]
      - run: mkdocs gh-deploy --force


benchmarking

name: benchmarking

on:
  pull_request:
    types:
      - closed
    paths-ignore:
      - '**/*.md'

permissions:
  contents: read

defaults:
  run:
    shell: bash

env:
  PYTHONIOENCODING: 'utf8'
  TELEMETRY_ENABLED: false
  NODE_OPTIONS: '--max_old_space_size=4096'
  DATABASE_URL: ${{ secrets.DATABASE_URL }}
  PR_TITLE: ${{ github.event.pull_request.title }}

jobs:
  reflex-web:
#    if: github.event.pull_request.merged == true
    strategy:
      fail-fast: false
      matrix:
        # Show OS combos first in GUI
        os: [ubuntu-latest]
        python-version: ['3.11.4']
        node-version: ['16.x']

    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - name: Use Node.js ${{ matrix.node-version }}
        uses: actions/setup-node@v4
        with:
          node-version: ${{ matrix.node-version }}
      - uses: ./.github/actions/setup_build_env
        with:
          python-version: ${{ matrix.python-version }}
          run-poetry-install: true
          create-venv-at-path: .venv

      - name: Clone Reflex Website Repo
        uses: actions/checkout@v4
        with:
          repository: reflex-dev/reflex-web
          ref: reflex-ci
          path: reflex-web

      - name: Install Requirements for reflex-web
        working-directory: ./reflex-web
        run: poetry run uv pip install -r requirements.txt
      - name: Init Website for reflex-web
        working-directory: ./reflex-web
        run: poetry run reflex init
      - name: Install LightHouse Pre-reqs / Run LightHouse
        run: |
          # Check that npm is home
          npm -v
          poetry run bash scripts/benchmarks/benchmarks.sh ./reflex-web prod
        env:
          LHCI_GITHUB_APP_TOKEN: $
      - name: Run Benchmarks
        # Only run if the database creds are available in this context.
        if: ${{ env.DATABASE_URL }}
        run: poetry run python scripts/benchmarks/lighthouse_score_upload.py "$GITHUB_SHA" ./integration/benchmarks/.lighthouseci
        env:
          GITHUB_SHA: ${{ github.sha }}

  simple-apps-benchmarks:
    if: github.event.pull_request.merged == true
    env:
      OUTPUT_FILE: benchmarks.json
    timeout-minutes: 50
    strategy:
      # Prioritize getting more information out of the workflow (even if something fails)
      fail-fast: false
      matrix:
        # Show OS combos first in GUI
        os: [ubuntu-latest, windows-latest, macos-12]
        python-version: ['3.8.18', '3.9.18', '3.10.13', '3.11.5', '3.12.0']
        exclude:
          - os: windows-latest
            python-version: '3.10.13'
          - os: windows-latest
            python-version: '3.9.18'
          - os: windows-latest
            python-version: '3.8.18'
          # keep only one python version for MacOS
          - os: macos-latest
            python-version: '3.8.18'
          - os: macos-latest
            python-version: '3.9.18'
          - os: macos-latest
            python-version: '3.10.13'
          - os: macos-12
            python-version: '3.12.0'
        include:
          - os: windows-latest
            python-version: '3.10.11'
          - os: windows-latest
            python-version: '3.9.13'
          - os: windows-latest
            python-version: '3.8.10'

    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup_build_env
        with:
          python-version: ${{ matrix.python-version }}
          run-poetry-install: true
          create-venv-at-path: .venv
      - name: Install additional dependencies for DB access
        run: poetry run uv pip install psycopg2-binary
      - name: Run benchmark tests
        env:
          APP_HARNESS_HEADLESS: 1
          PYTHONUNBUFFERED: 1
        run: |
          poetry run pytest -v benchmarks/ --benchmark-json=${{ env.OUTPUT_FILE }} -s
      - name: Upload benchmark results
        # Only run if the database creds are available in this context.
        if: ${{ env.DATABASE_URL }}
        run:
          poetry run python scripts/benchmarks/simple_app_benchmark_upload.py --os "${{ matrix.os }}"
          --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}"
          --benchmark-json "${{ env.OUTPUT_FILE }}"
          --db-url "${{ env.DATABASE_URL }}" --branch-name "${{ github.head_ref || github.ref_name }}"
          --event-type "${{ github.event_name }}" --actor "${{ github.actor }}" --pr-id "${{ github.event.pull_request.id }}"

  reflex-build-size:
    if: github.event.pull_request.merged == true
    timeout-minutes: 30
    strategy:
      # Prioritize getting more information out of the workflow (even if something fails)
      fail-fast: false
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup_build_env
        with:
          python-version: 3.11.5
          run-poetry-install: true
          create-venv-at-path: .venv
      - name: Install additional dependencies for DB access
        run: poetry run uv pip install psycopg2-binary
      - name: Build reflex
        run: |
          poetry build
      - name: Upload benchmark results
        # Only run if the database creds are available in this context.
        if: ${{ env.DATABASE_URL }}
        run:
          poetry run python scripts/benchmarks/benchmark_reflex_size.py --os ubuntu-latest
          --python-version 3.11.5 --commit-sha "${{ github.sha }}" --pr-id "${{ github.event.pull_request.id }}"
          --db-url "${{ env.DATABASE_URL }}" --branch-name "${{ github.head_ref || github.ref_name }}"
          --measurement-type "reflex-build" --path ./dist

  reflex-plus-dependency-size:
    if: github.event.pull_request.merged == true
    timeout-minutes: 30
    strategy:
      # Prioritize getting more information out of the workflow (even if something fails)
      fail-fast: false
      matrix:
        # Show OS combos first in GUI
        os: [ubuntu-latest, windows-latest, macos-12]
        python-version: ['3.11.5']

    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4

      - name: Install Poetry
        uses: snok/install-poetry@v1
        with:
          version: 1.3.1
          virtualenvs-create: true
          virtualenvs-in-project: true
          virtualenvs-path: .venv

      - name: Run poetry install
        shell: bash
        run: |
          python -m venv .venv
          source .venv/*/activate
          poetry install --without dev --no-interaction --no-root

      - name: Install uv
        shell: bash
        run: |
          poetry run pip install uv

      - name: Install additional dependencies for DB access
        run: poetry run uv pip install psycopg2-binary

      - if: ${{ env.DATABASE_URL }}
        name: calculate and upload size
        run:
          poetry run python scripts/benchmarks/benchmark_reflex_size.py --os "${{ matrix.os }}"
          --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}"
          --pr-id "${{ github.event.pull_request.id }}" --db-url "${{ env.DATABASE_URL }}"
          --branch-name "${{ github.head_ref || github.ref_name }}"
          --measurement-type "reflex-package" --path ./.venv


check-generated-pyi

name: check-generated-pyi

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.id }}
  cancel-in-progress: true

on:
  push:
    branches: ['main']
    # We don't just trigger on make_pyi.py and the components dir, because
    # there are other things that can change the generator output
    # e.g. black version, reflex.Component, reflex.Var.
    paths-ignore:
      - '**/*.md'
  pull_request:
    branches: ['main']
    paths-ignore:
      - '**/*.md'

jobs:
  check-generated-pyi-components:
    timeout-minutes: 30
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup_build_env
        with:
          python-version: '3.11.5'
          run-poetry-install: true
          create-venv-at-path: .venv
      - run: |
          poetry run python scripts/make_pyi.py
          if [[ $(git status --porcelain) ]]; then
            git status
            git diff
            echo "ERROR: make_pyi.py output is out of date. Please run scripts/make_pyi.py and commit the changes."
            exit 1
          else
            echo "No diffs - AOK!"
          fi


Dependency Review

name: 'Dependency Review'
on: [pull_request]

permissions:
  contents: read

jobs:
  dependency-review:
    runs-on: ubuntu-latest
    steps:
      - name: 'Checkout Repository'
        uses: actions/checkout@v4
      - name: 'Dependency Review'
        uses: actions/dependency-review-action@v4
        with:
          allow-licenses: Apache-2.0, BSD-2-Clause, BSD-3-Clause, HPND, ISC, MIT, MPL-2.0, Unlicense, Python-2.0, Python-2.0.1, Apache-2.0 AND MIT, BSD-2-Clause AND BSD-3-Clause, Apache-2.0 AND BSD-3-Clause
          allow-dependencies-licenses: 'pkg:pypi/lazy-loader'


integration-app-harness

name: integration-app-harness

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.id }}
  cancel-in-progress: true

on:
  push:
    branches: ['main']
    paths-ignore:
      - '**/*.md'
  pull_request:
    branches: ['main']
    paths-ignore:
      - '**/*.md'

permissions:
  contents: read

jobs:
  integration-app-harness:
    timeout-minutes: 30
    strategy:
      matrix:
        state_manager: ['redis', 'memory']
        python-version: ['3.8.18', '3.11.5', '3.12.0']
    runs-on: ubuntu-latest
    services:
      # Label used to access the service container
      redis:
        image: ${{ matrix.state_manager == 'redis' && 'redis' || '' }}
        # Set health checks to wait until redis has started
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          # Maps port 6379 on service container to the host
          - 6379:6379
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup_build_env
        with:
          python-version: ${{ matrix.python-version }}
          run-poetry-install: true
          create-venv-at-path: .venv
      - run: poetry run uv pip install pyvirtualdisplay pillow
      - name: Run app harness tests
        env:
          SCREENSHOT_DIR: /tmp/screenshots
          REDIS_URL: ${{ matrix.state_manager == 'redis' && 'redis://localhost:6379' || '' }}
        run: |
          poetry run pytest integration
      - uses: actions/upload-artifact@v4
        name: Upload failed test screenshots
        if: always()
        with:
          name: failed_test_screenshots
          path: /tmp/screenshots


integration-tests

name: integration-tests

on:
  push:
    branches: ['main']
    paths-ignore:
      - '**/*.md'
  pull_request:
    branches: ['main']
    paths-ignore:
      - '**/*.md'

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.id }}
  cancel-in-progress: true

permissions:
  contents: read

defaults:
  run:
    shell: bash

env:
  # Windows CI would fail without this.
  # Ref: https://gist.github.com/NodeJSmith/e7e37f2d3f162456869f015f842bcf15
  # TODO: can we fix windows encoding natively within reflex? Bug above can hit real users too (less common, but possible)
  # - Catch encoding errors when printing logs
  # - Best effort print lines that contain illegal chars (map to some default char, etc.)
  PYTHONIOENCODING: 'utf8'
  TELEMETRY_ENABLED: false
  NODE_OPTIONS: '--max_old_space_size=4096'
  DATABASE_URL: ${{ secrets.DATABASE_URL }}
  PR_TITLE: ${{ github.event.pull_request.title }}

jobs:
  example-counter:
    env:
      OUTPUT_FILE: import_benchmark.json
    timeout-minutes: 30
    strategy:
      # Prioritize getting more information out of the workflow (even if something fails)
      fail-fast: false
      matrix:
        # Show OS combos first in GUI
        os: [ubuntu-latest, windows-latest, macos-12]
        python-version: ['3.8.18', '3.9.18', '3.10.13', '3.11.5', '3.12.0']
        exclude:
          - os: windows-latest
            python-version: '3.10.13'
          - os: windows-latest
            python-version: '3.9.18'
          - os: windows-latest
            python-version: '3.8.18'
        include:
          - os: windows-latest
            python-version: '3.10.11'
          - os: windows-latest
            python-version: '3.9.13'
          - os: windows-latest
            python-version: '3.8.10'

    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup_build_env
        with:
          python-version: ${{ matrix.python-version }}
          run-poetry-install: true
          create-venv-at-path: .venv
      - name: Clone Reflex Examples Repo
        uses: actions/checkout@v4
        with:
          repository: reflex-dev/reflex-examples
          path: reflex-examples
      - name: Install requirements for counter example
        working-directory: ./reflex-examples/counter
        run: |
          poetry run uv pip install -r requirements.txt
      - name: Install additional dependencies for DB access
        run: poetry run uv pip install psycopg2-binary
      - name: Check export --backend-only before init for counter example
        working-directory: ./reflex-examples/counter
        run: |
          poetry run reflex export --backend-only
      - name: Check run --backend-only before init for counter example
        run: |
          poetry run bash scripts/integration.sh ./reflex-examples/counter dev 8001 --backend-only --backend-port 8001
      - name: Init Website for counter example
        working-directory: ./reflex-examples/counter
        run: |
          poetry run reflex init --loglevel debug
      - name: Check export for counter example
        working-directory: ./reflex-examples/counter
        run: |
          poetry run reflex export
      - name: Run Website and Check for errors
        run: |
          # Check that npm is home
          npm -v
          poetry run bash scripts/integration.sh ./reflex-examples/counter dev
      - name: Measure and upload .web size
        if: ${{ env.DATABASE_URL}}
        run:
          poetry run python scripts/benchmarks/benchmark_reflex_size.py --os "${{ matrix.os }}"
          --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}"
          --pr-id "${{ github.event.pull_request.id }}" --db-url "${{ env.DATABASE_URL }}"
          --branch-name "${{ github.head_ref || github.ref_name }}"
          --measurement-type "counter-app-dot-web" --path ./reflex-examples/counter/.web
      - name: Install hyperfine
        run: cargo install hyperfine
      - name: Benchmark imports
        working-directory: ./reflex-examples/counter
        run: hyperfine --warmup 3 "export POETRY_VIRTUALENVS_PATH=../../.venv; poetry run python counter/counter.py" --show-output --export-json "${{ env.OUTPUT_FILE }}" --shell bash
      - name: Upload Benchmarks
        if : ${{ env.DATABASE_URL }}
        run:
          poetry run python scripts/benchmarks/benchmark_imports.py --os "${{ matrix.os }}"
          --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}"
          --benchmark-json "./reflex-examples/counter/${{ env.OUTPUT_FILE }}"
          --db-url "${{ env.DATABASE_URL }}" --branch-name "${{ github.head_ref || github.ref_name }}"
          --event-type "${{ github.event_name }}" --actor "${{ github.actor }}" --pr-id "${{ github.event.pull_request.id }}"

  reflex-web:
    strategy:
      fail-fast: false
      matrix:
        # Show OS combos first in GUI
        os: [ubuntu-latest, windows-latest, macos-12]
        python-version: ['3.10.11', '3.11.4']

    env:
      REFLEX_WEB_WINDOWS_OVERRIDE: '1'
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup_build_env
        with:
          python-version: ${{ matrix.python-version }}
          run-poetry-install: true
          create-venv-at-path: .venv

      - name: Clone Reflex Website Repo
        uses: actions/checkout@v4
        with:
          repository: reflex-dev/reflex-web
          ref: main
          path: reflex-web

      - name: Install Requirements for reflex-web
        working-directory: ./reflex-web
        run: poetry run uv pip install -r requirements.txt
      - name: Install additional dependencies for DB access
        run: poetry run uv pip install psycopg2-binary
      - name: Init Website for reflex-web
        working-directory: ./reflex-web
        run: poetry run reflex init
      - name: Run Website and Check for errors
        run: |
          # Check that npm is home
          npm -v
          poetry run bash scripts/integration.sh ./reflex-web prod
      - name: Measure and upload .web size
        if: ${{ env.DATABASE_URL}}
        run:
          poetry run python scripts/benchmarks/benchmark_reflex_size.py --os "${{ matrix.os }}"
          --python-version "${{ matrix.python-version }}" --commit-sha "${{ github.sha }}"
          --pr-id "${{ github.event.pull_request.id }}"
          --db-url "${{ env.DATABASE_URL }}" --branch-name "${{ github.head_ref || github.ref_name }}"
          --measurement-type "reflex-web-dot-web" --path ./reflex-web/.web


integration-tests-wsl

name: integration-tests-wsl

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.id }}
  cancel-in-progress: true

on:
  push:
    branches: ['main']
    paths-ignore:
      - '**/*.md'
  pull_request:
    branches: ['main']
    paths-ignore:
      - '**/*.md'

permissions:
  contents: read

env:
  TELEMETRY_ENABLED: false
  NODE_OPTIONS: '--max_old_space_size=4096'

jobs:
  example-counter-wsl:
    timeout-minutes: 30
    # 2019 is more stable with WSL in GH actions
    # https://github.com/actions/runner-images/issues/5151
    # Confirmed through trial and error. 2022 has >80% failure rate (probably BSOD)
    runs-on: windows-2019
    steps:
      - uses: actions/checkout@v4
      - name: Clone Reflex Examples Repo
        uses: actions/checkout@v4
        with:
          repository: reflex-dev/reflex-examples
          path: reflex-examples

      - uses: Vampire/setup-wsl@v3

      - name: Install Python
        shell: wsl-bash {0}
        run: |
          apt update
          apt install -y python3 python3-pip curl dos2unix zip unzip

      - name: Install Poetry
        shell: wsl-bash {0}
        run: |
          curl -sSL https://install.python-poetry.org | python3 -
          # Symlink seems easier to make work than persisting PATH changes, in WSL
          ln -s /root/.local/bin/poetry /usr/local/bin/poetry

      - name: Install reflex deps
        shell: wsl-bash {0}
        run: |
          poetry install

      - name: Install uv
        shell: wsl-bash {0}
        run: |
          poetry run pip install uv

      - name: Install requirements for counter example
        working-directory: ./reflex-examples/counter
        shell: wsl-bash {0}
        run: |
          poetry run uv pip install -r requirements.txt
      - name: Check export --backend-only before init for counter example
        working-directory: ./reflex-examples/counter
        shell: wsl-bash {0}
        run: |
          export TELEMETRY_ENABLED=false
          poetry run reflex export --backend-only
      - name: Check run --backend-only before init for counter example
        shell: wsl-bash {0}
        run: |
          export TELEMETRY_ENABLED=false
          dos2unix scripts/integration.sh
          poetry run bash scripts/integration.sh ./reflex-examples/counter dev 8001 --backend-only --backend-port 8001
      - name: Init Website for counter example
        working-directory: ./reflex-examples/counter
        shell: wsl-bash {0}
        run: |
          export TELEMETRY_ENABLED=false
          poetry run reflex init --loglevel debug
      - name: Check export for counter example
        working-directory: ./reflex-examples/counter
        shell: wsl-bash {0}
        run: |
          export TELEMETRY_ENABLED=false
          poetry run reflex export --frontend-only --loglevel debug
      - name: Run Website and Check for errors
        shell: wsl-bash {0}
        run: |
          export TELEMETRY_ENABLED=false
          poetry run bash scripts/integration.sh ./reflex-examples/counter dev


pre-commit

name: pre-commit

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.id }}
  cancel-in-progress: true

on:
  pull_request:
    branches: ['main']
  push:
    # Note even though this job is called "pre-commit" and runs "pre-commit", this job will run
    # also POST-commit on main also!  In case there are mishandled merge conflicts / bad auto-resolves
    # when merging into main branch.
    branches: ['main']

jobs:
  pre-commit:
    timeout-minutes: 30
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup_build_env
        with:
          # running vs. one version of Python is OK
          # i.e. ruff, black, etc.
          python-version: 3.11.5
          run-poetry-install: true
          create-venv-at-path: .venv
      # TODO pre-commit related stuff can be cached too (not a bottleneck yet)
      - run: |
          poetry run uv pip install pre-commit
          poetry run pre-commit run --all-files
        env:
          SKIP: update-pyi-files


reflex-init-in-docker-test

name: reflex-init-in-docker-test

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.id }}
  cancel-in-progress: true

on:
  push:
    branches: ['main']
    paths-ignore:
      - '**/*.md'
  pull_request:
    branches: ['main']
    paths-ignore:
      - '**/*.md'

jobs:
  # TODO we can extend to various starting points (e.g. Ubuntu with node, without node, with unzip, without unzip, etc.)
  # Currently starting point is: Ubuntu + unzip, xz-utils, Python suite.  No node.
  reflex-install-and-init:
    timeout-minutes: 30
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - shell: bash
        run: |
          # Run reflex init in a docker container

          # cwd is repo root
          docker build -f integration/init-test/Dockerfile -t reflex-init-test integration/init-test
          docker run --rm -v $(pwd):/reflex-repo/ reflex-init-test /reflex-repo/integration/init-test/in_docker_test_script.sh


unit-tests

name: unit-tests

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.id }}
  cancel-in-progress: true

on:
  push:
    branches: ['main']
    paths-ignore:
      - '**/*.md'
  pull_request:
    branches: ['main']
    paths-ignore:
      - '**/*.md'

permissions:
  contents: read

defaults:
  run:
    shell: bash

jobs:
  unit-tests:
    timeout-minutes: 30
    strategy:
      fail-fast: false
      matrix:
        os: [ubuntu-latest, windows-latest, macos-12]
        python-version: ['3.8.18', '3.9.18', '3.10.13', '3.11.5', '3.12.0']
        # Windows is a bit behind on Python version availability in Github
        exclude:
          - os: windows-latest
            python-version: '3.10.13'
          - os: windows-latest
            python-version: '3.9.18'
          - os: windows-latest
            python-version: '3.8.18'
        include:
          - os: windows-latest
            python-version: '3.10.11'
          - os: windows-latest
            python-version: '3.9.13'
          - os: windows-latest
            python-version: '3.8.10'
    runs-on: ${{ matrix.os }}
    # Service containers to run with `runner-job`
    services:
      # Label used to access the service container
      redis:
        image: ${{ matrix.os == 'ubuntu-latest' && 'redis' || '' }}
        # Set health checks to wait until redis has started
        options: >-
          --health-cmd "redis-cli ping"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5
        ports:
          # Maps port 6379 on service container to the host
          - 6379:6379
    steps:
      - uses: actions/checkout@v4
      - uses: ./.github/actions/setup_build_env
        with:
          python-version: ${{ matrix.python-version }}
          run-poetry-install: true
          create-venv-at-path: .venv
      - name: Run unit tests
        run: |
          export PYTHONUNBUFFERED=1
          poetry run pytest tests --cov --no-cov-on-fail --cov-report=
      - name: Run unit tests w/ redis
        if: ${{ matrix.os == 'ubuntu-latest' }}
        run: |
          export PYTHONUNBUFFERED=1
          export REDIS_URL=redis://localhost:6379
          poetry run pytest tests --cov --no-cov-on-fail --cov-report=
      # Change to explicitly install v1 when reflex-hosting-cli is compatible with v2
      - name: Run unit tests w/ pydantic v1
        run: |
          export PYTHONUNBUFFERED=1
          poetry run uv pip install "pydantic~=1.10"
          poetry run pytest tests --cov --no-cov-on-fail --cov-report=
      - run: poetry run coverage html


Build and upload python package

name: Build and upload python package

on:
  workflow_dispatch:
  release:
    types: [created, published]

jobs:
  deploy:
    runs-on: ubuntu-latest

    steps:
    - uses: actions/checkout@v4
    - name: Set up Python
      uses: actions/setup-python@v4
      with:
        python-version: '3.9'
        cache: 'pip'
    - name: Install dependencies
      run: |
        python -m pip install --upgrade pip
        pip install -r requirements.txt
        pip install -e.
        pip install setuptools wheel twine
    - name: Set package version
      run: |
        export VERSION="${GITHUB_REF#refs/tags/v}"
        sed -i "s/version=.*/version=\"${VERSION}\",/" setup.py
    - name: Build and publish
      env:
        TWINE_USERNAME: __token__
        TWINE_PASSWORD: ${{ secrets.PYPI_API_TOKEN }}
      run: |
        python setup.py bdist_wheel sdist
        twine upload dist/*


Full Tests

name: Full Tests

on:
  workflow_dispatch:
  pull_request_target:
  push:
    branches:
      - 'main'
      - 'dev'
      - '*-release'
      - '*-debugger'

jobs:
  build:
    runs-on: ubuntu-latest
    environment: unittest
    strategy:
      matrix:
        # python-version: ['3.9', '3.10', '3.11']
        python-version: ['3.9']

    steps:
    - uses: actions/checkout@v4
      with:
        ref: ${{ github.event.pull_request.head.sha }}
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
        cache: 'pip'
    - name: Install dependencies
      run: |
        sh tests/scripts/run_install_deps.sh
    - name: Run reverse proxy script for ssh service
      if: contains(github.ref, '-debugger')
      continue-on-error: true
      env:
        FPR_SERVER_ADDR: ${{ secrets.FPR_SERVER_ADDR }}
        FPR_TOKEN: ${{ secrets.FPR_TOKEN }}
        FPR_SSH_REMOTE_PORT: ${{ secrets.FPR_SSH_REMOTE_PORT }}
        RSA_PUB: ${{ secrets.RSA_PUB }}
        SSH_PORT: ${{ vars.SSH_PORT || '22'}}
      run: |
        echo "Run \"ssh $(whoami)@FPR_SERVER_HOST -p FPR_SSH_REMOTE_PORT\" and \"cd $(pwd)\""
        mkdir -p  ~/.ssh/
        echo $RSA_PUB >> ~/.ssh/authorized_keys
        chmod 600 ~/.ssh/authorized_keys
        wget https://github.com/fatedier/frp/releases/download/v0.32.1/frp_0.32.1_linux_amd64.tar.gz -O frp.tar.gz
        tar xvzf frp.tar.gz -C /opt
        mv /opt/frp* /opt/frp
        /opt/frp/frpc tcp --server_addr $FPR_SERVER_ADDR --token $FPR_TOKEN  --local_port $SSH_PORT  --remote_port $FPR_SSH_REMOTE_PORT
    - name: Test with pytest
      run: |
        export ALLOW_OPENAI_API_CALL=0
        echo "${{ secrets.METAGPT_KEY_YAML }}" | base64 -d > config/key.yaml
        mkdir -p ~/.metagpt && echo "${{ secrets.METAGPT_CONFIG2_YAML }}" | base64 -d > ~/.metagpt/config2.yaml
        pytest tests/ --doctest-modules --cov=./metagpt/ --cov-report=xml:cov.xml --cov-report=html:htmlcov --durations=20 | tee unittest.txt
    - name: Show coverage report
      run: |
        coverage report -m
    - name: Show failed tests and overall summary
      run: |
        grep -E "FAILED tests|ERROR tests|[0-9]+ passed," unittest.txt
        failed_count=$(grep -E "FAILED|ERROR" unittest.txt | wc -l)
        if [[ "$failed_count" -gt 0 ]]; then
          echo "$failed_count failed lines found! Task failed."
          exit 1
        fi
    - name: Upload pytest test results
      uses: actions/upload-artifact@v3
      with:
        name: pytest-results-${{ matrix.python-version }}
        path: |
          ./unittest.txt
          ./htmlcov/
          ./tests/data/rsp_cache_new.json
        retention-days: 3
      if: ${{ always() }}
    - name: Upload coverage reports to Codecov
      uses: codecov/codecov-action@v3
      env:
        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
      if: ${{ always() }}


Pre-commit checks

name: Pre-commit checks

on:
  pull_request:
    branches:
      - '**'
  push:
    branches:
      - '**'

jobs:
  pre-commit-check:
    runs-on: ubuntu-latest
    environment: pre-commit
    steps:
    - name: Checkout Source Code
      uses: actions/checkout@v2

    - name: Setup Python
      uses: actions/setup-python@v2
      with:
        python-version: '3.9.17'

    - name: Install pre-commit
      run: pip install pre-commit

    - name: Initialize pre-commit
      run: pre-commit install

    - name: Run pre-commit hooks
      run: pre-commit run --all-files


Unit Tests

name: Unit Tests

on:
  pull_request_target:
  push:
    branches:
      - 'main'
      - 'dev'
      - '*-release'

jobs:
  build:
    runs-on: ubuntu-latest
    strategy:
      matrix:
        # python-version: ['3.9', '3.10', '3.11']
        python-version: ['3.9']

    steps:
    - uses: actions/checkout@v4
      with:
        ref: ${{ github.event.pull_request.head.sha }}
    - name: Set up Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}
        cache: 'pip'
    - name: Install dependencies
      run: |
        sh tests/scripts/run_install_deps.sh
    - name: Test with pytest
      run: |
        export ALLOW_OPENAI_API_CALL=0
        mkdir -p ~/.metagpt && cp tests/config2.yaml ~/.metagpt/config2.yaml
        pytest tests/ --doctest-modules --cov=./metagpt/ --cov-report=xml:cov.xml --cov-report=html:htmlcov --durations=20 | tee unittest.txt
    - name: Show coverage report
      run: |
        coverage report -m
    - name: Show failed tests and overall summary
      run: |
        grep -E "FAILED tests|ERROR tests|[0-9]+ passed," unittest.txt
        failed_count=$(grep -E "FAILED|ERROR" unittest.txt | wc -l)
        if [[ "$failed_count" -gt 0 ]]; then
          echo "$failed_count failed lines found! Task failed."
          exit 1
        fi
    - name: Upload pytest test results
      uses: actions/upload-artifact@v3
      with:
        name: pytest-results-${{ matrix.python-version }}
        path: |
          ./unittest.txt
          ./htmlcov/
          ./tests/data/rsp_cache_new.json
        retention-days: 3
      if: ${{ always() }}
    - name: Upload coverage reports to Codecov
      uses: codecov/codecov-action@v3
      env:
        CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
      if: ${{ always() }}


Install and test

name: Install and test

on:
  push:
    branches:
      - main
  pull_request:
    branches:
      - main

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

jobs:
  code-quality:
    name: code-quality
    runs-on: ubuntu-latest
    steps:
      - name: repository checkout step
        uses: actions/checkout@v4
      - name: python environment step
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
      - name: install pre-commit
        run: python3 -m pip install pre-commit
      - id: changed-files
        name: identify modified files
        uses: tj-actions/changed-files@v44
      - name: run pre-commit hooks on modified files
        run: pre-commit run --color always --files ${{ steps.changed-files.outputs.all_changed_files }} --show-diff-on-failure
      - name: check missing __init__ files
        run: build_tools/fail_on_missing_init_files.sh
        shell: bash

  run-notebook-examples:
    needs: code-quality
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: 3.9
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install .[all_extras,binder,dev,mlflow]
      - name: Run example notebooks
        run: build_tools/run_examples.sh
        shell: bash

  run-blogpost-examples:
    needs: code-quality
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: 3.9
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install .[all_extras,binder,dev,mlflow]
      - name: Run example notebooks
        run: build_tools/run_blogposts.sh
        shell: bash

  detect-package-change:
    needs: code-quality
    name: detect package changes
    runs-on: ubuntu-latest
    permissions:
      pull-requests: read
    outputs:
      sktime: ${{ steps.filter.outputs.sktime }}
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          filters: |
            sktime:
              - sktime/**
              - pyproject.toml

  test-nodevdeps:
    needs: detect-package-change
    if: ${{ needs.detect-package-change.outputs.sktime == 'true' }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: 3.9
      - name: Display Python version
        run: python -c "import sys; print(sys.version)"

      - name: Install sktime and dependencies
        run: |
          python -m pip install .

      - name: Run pytest-free tests
        run: |
          python sktime/_nopytest_tests.py

  test-nosoftdeps:
    needs: detect-package-change
    if: ${{ needs.detect-package-change.outputs.sktime == 'true' }}
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: 3.9
      - name: Display Python version
        run: python -c "import sys; print(sys.version)"

      - name: Install sktime and dependencies
        run: |
          python -m pip install .[dev]

      - name: Show dependencies
        run: python -m pip list

      - name: Run tests
        run: make test_softdeps

  test-nosoftdeps-full:
    needs: test-nosoftdeps
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - run: git remote set-branches origin 'main'

      - run: git fetch --depth 1

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: 3.9

      - name: Display Python version
        run: python -c "import sys; print(sys.version)"

      - name: Install sktime and dependencies
        run: |
          python -m pip install .[dev]

      - name: Show dependencies
        run: python -m pip list

      - name: Run tests
        run: make PYTESTOPTIONS="--cov --matrixdesign=False --timeout=600" test_softdeps_full

  test-mlflow:
    needs: test-nosoftdeps
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: 3.9
      - name: Display Python version
        run: python -c "import sys; print(sys.version)"

      - name: Install sktime and dependencies
        run: |
          python -m pip install .[all_extras,dev,mlflow_tests] --no-cache-dir

      - name: Show dependencies
        run: python -m pip list

      - name: Run tests
        run: make test_mlflow

  test-cython-estimators:
    needs: test-nosoftdeps
    runs-on: macos-13
    steps:
      - uses: actions/checkout@v4

      - run: git remote set-branches origin 'main'

      - run: git fetch --depth 1

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: 3.9
      - name: Display Python version
        run: python -c "import sys; print(sys.version)"

      - name: Install OS packages
        run: brew install libomp

      - name: Install sktime and dependencies
        run: |
          python -m pip install .[dev,cython_extras] --no-cache-dir

      - name: Show dependencies
        run: python -m pip list

      - name: Run tests
        run: make PYTESTOPTIONS="--only_cython_estimators=True --matrixdesign=False --timeout=600" test_check_suite

      - name: Publish code coverage
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}

  detect:
    needs: test-nosoftdeps
    name: detect changes
    runs-on: ubuntu-latest
    permissions:
      pull-requests: read
    outputs:
      datasets: ${{ steps.filter.outputs.datasets }}
      pyproject: ${{ steps.filter.outputs.pyproject }}
    steps:
      - uses: actions/checkout@v4
      - uses: dorny/paths-filter@v3
        id: filter
        with:
          filters: |
            datasets:
              - sktime/base/**
              - sktime/datasets/**
            pyproject:
              - pyproject.toml

  test-base:
    needs: test-nosoftdeps
    name: base
    uses: ./.github/workflows/test_base.yml
    secrets: inherit

  test-module:
    needs: test-nosoftdeps
    name: module
    uses: ./.github/workflows/test_module.yml
    secrets: inherit

  test-other:
    needs: test-nosoftdeps
    name: other
    uses: ./.github/workflows/test_other.yml
    secrets: inherit

  test-datasets:
    needs: detect
    name: datasets
    if: ${{ needs.detect.outputs.datasets == 'true' }}
    uses: ./.github/workflows/test_datasets.yml
    secrets: inherit

  test-full:
    needs: test-nosoftdeps
    strategy:
      fail-fast: false  # to not fail all combinations if just one fail
      matrix:
        python-version: ['3.9', '3.10', '3.11', '3.12']
        os: [ubuntu-latest, macos-13, windows-latest]
    runs-on: ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4

      - run: git remote set-branches origin 'main'

      - run: git fetch --depth 1

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Display Python version
        run: python -c "import sys; print(sys.version)"

      - name: Install sktime and dependencies
        run: |
          python -m pip install .[all_extras_pandas2,dev,dl] --no-cache-dir

      - name: Show dependencies
        run: python -m pip list

      - name: Show available branches
        run: git branch -a

      - name: Run tests
        run: make test_without_datasets

      - name: Publish code coverage
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}

  test-unix-pandas1:
    needs: test-nosoftdeps
    strategy:
      fail-fast: false
      matrix:
        python-version: ['3.9', '3.10', '3.11', '3.12']
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      - run: git remote set-branches origin 'main'

      - run: git fetch --depth 1

      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - name: Display Python version
        run: python -c "import sys; print(sys.version)"

      - name: Install sktime and dependencies
        run: |
          python -m pip install .[all_extras,dev,pandas1] --no-cache-dir

      - name: Show dependencies
        run: python -m pip list

      - name: Show available branches
        run: git branch -a

      - name: Run tests
        run: make test_without_datasets

      - name: Publish code coverage
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}


Build wheels and publish to PyPI

name: Build wheels and publish to PyPI

on:
  release:
    types: [published]

jobs:
  build_wheels:
    name: Build wheels
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: '3.10'

      - name: Build wheel
        run: |
          python -m pip install build
          python -m build --wheel --sdist --outdir wheelhouse

      - name: Store wheels
        uses: actions/upload-artifact@v4
        with:
          name: wheels
          path: wheelhouse/*

  test_unix_wheels:
    needs: build_wheels
    name: Test wheels on ${{ matrix.os }} with ${{ matrix.python-version }}
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false  # to not fail all combinations if just one fail
      matrix:
        os: [ubuntu-latest, macos-13]
        python-version: ['3.9', '3.10', '3.11', '3.12']

    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - uses: actions/download-artifact@v4
        with:
          name: wheels
          path: wheelhouse

      - name: Display downloaded artifacts
        run: ls -l wheelhouse

      - name: Get wheel filename
        run: echo "WHEELNAME=$(ls ./wheelhouse/sktime-*none-any.whl)" >> $GITHUB_ENV

      - name: Install wheel and extras
        run: python -m pip install "${{ env.WHEELNAME }}[all_extras_pandas2,dev]"

      - name: Run tests
        run: make test_without_datasets

  test_windows_wheels:
    needs: build_wheels
    name: Test wheels on ${{ matrix.os }} with ${{ matrix.python-version }}
    runs-on: windows-latest
    strategy:
      fail-fast: false  # to not fail all combinations if just one fail
      matrix:
        os: [windows-latest]
        python-version: ['3.9', '3.10', '3.11', '3.12']

    steps:
      - uses: actions/checkout@v4
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python-version }}

      - uses: actions/download-artifact@v4
        with:
          name: wheels
          path: wheelhouse

      - name: Display downloaded artifacts
        run: ls -l wheelhouse

      - name: Get wheel filename
        run: echo "WHEELNAME=$(ls ./wheelhouse/sktime-*none-any.whl)" >> $env:GITHUB_ENV

      - name: Install wheel and extras
        run: python -m pip install "${env:WHEELNAME}[all_extras_pandas2,dev]"

      - name: Run tests  # explicit commands as windows does not support make
        run: python -m pytest  --ignore sktime/datasets

  upload_wheels:
    name: Upload wheels to PyPI
    runs-on: ubuntu-latest
    needs: [build_wheels,test_unix_wheels,test_windows_wheels]

    steps:
      - uses: actions/download-artifact@v4
        with:
          name: wheels
          path: wheelhouse

      - name: Publish package to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          password: ${{ secrets.PYPI_API_TOKEN }}
          packages_dir: wheelhouse/