LLM Binary Build

name: LLM Binary Build

# Cancel previous runs in the PR when you push new commits
# concurrency:
#   group: ${{ github.workflow }}-llm-binary-build-${{ github.event.pull_request.number || github.run_id }}
#   cancel-in-progress: false

permissions:
  contents: read

# Controls when the action will run.
on:
  # Triggers the workflow on push or pull request events but only for the main branch
  push:
    branches: [main]
    paths:
      - ".github/workflows/llm-binary-build.yml"
  pull_request:
    branches: [main]
    paths:
      - ".github/workflows/llm-binary-build.yml"
  workflow_dispatch:
    inputs:
      llmcpp-ref:
        description: 'Ref of llm.cpp code'
        default: ''
        required: false
        type: string
      platform:
        description: 'Platforms to built on'
        default: '["Windows", "Linux"]'
        required: false
        type: string
  workflow_call:
    inputs:
      llmcpp-ref:
        description: 'Ref of llm.cpp code'
        default: ''
        required: false
        type: string
      platform:
        description: 'Platforms to built on'
        default: 'Windows,Linux'
        required: false
        type: string

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  check-linux-avxvnni-artifact:
    if: ${{contains(inputs.platform, 'Linux')}}
    runs-on: ubuntu-latest
    outputs:
      if-exists: ${{steps.check_artifact.outputs.exists}}
    steps:
      - name: Check if built
        id: check_artifact
        uses: xSAVIKx/artifact-exists-action@v0
        with:
          name: linux-avxvnni

  linux-build-avxvnni:
    runs-on: [self-hosted, AVX2, almalinux8]
    needs: check-linux-avxvnni-artifact
    if: needs.check-linux-avxvnni-artifact.outputs.if-exists == 'false'
    steps:
      - name: Set access token
        run: |
          echo "github_access_token=${GITHUB_ACCESS_TOKEN}" >> "$GITHUB_ENV"
      - name: Install Build Environment
        shell: bash
        run: |
          export http_proxy=${HTTP_PROXY}
          export https_proxy=${HTTPS_PROXY}
          yum install -y gcc-toolset-11 cmake git
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
        with:
          repository: "intel-analytics/llm.cpp"
          ref: ${{ inputs.llmcpp-ref }}
          token: ${{ env.github_access_token }}
          submodules: "recursive"
      - name: Build binary
        shell: bash
        run: |
          scl enable gcc-toolset-11 "cmake -B build"
          scl enable gcc-toolset-11 "cmake --build build --config Release -j"
      - name: Move release binary
        shell: bash
        run: |
          mkdir release
          mv build/main-bloom release/main-bloom
          mv build/libbloom-api.so release/libbloom-api.so
          mv build/quantize-bloom release/quantize-bloom
          mv build/libbloom.so release/libbloom_avxvnni.so
          mv build/main-llama release/main-llama
          mv build/libllama-api.so release/libllama-api.so
          mv build/quantize-llama release/quantize-llama
          mv build/libllama.so release/libllama_avxvnni.so
          mv build/main-gptneox release/main-gptneox
          mv build/libgptneox-api.so release/libgptneox-api.so
          mv build/quantize-gptneox release/quantize-gptneox
          mv build/libgptneox.so release/libgptneox_avxvnni.so
          mv build/main-starcoder release/main-starcoder
          mv build/libstarcoder-api.so release/libstarcoder-api.so
          mv build/quantize-starcoder release/quantize-starcoder
          mv build/libstarcoder.so release/libstarcoder_avxvnni.so
      - name: Archive build files
        uses: actions/upload-artifact@v3
        with:
          name: linux-avxvnni
          path: |
            release
      - name: Clean up test environment
        shell: bash
        run: |
          make clean

  check-linux-avx512-artifact:
    if: ${{contains(inputs.platform, 'Linux')}}
    runs-on: ubuntu-latest
    outputs:
      if-exists: ${{steps.check_artifact.outputs.exists}}
    steps:
      - name: Check if built
        id: check_artifact
        uses: xSAVIKx/artifact-exists-action@v0
        with:
          name: linux-avx512

  linux-build-avx512:
    runs-on: [self-hosted, AVX512, almalinux8]
    needs: check-linux-avx512-artifact
    if: needs.check-linux-avx512-artifact.outputs.if-exists == 'false'
    steps:
      - name: Set access token
        run: |
          echo "github_access_token=${GITHUB_ACCESS_TOKEN}" >> "$GITHUB_ENV"
      - name: Install Build Environment
        shell: bash
        run: |
          export http_proxy=${HTTP_PROXY}
          export https_proxy=${HTTPS_PROXY}
          yum install -y gcc-toolset-11 cmake git
          conda remove -n python39 --all -y
          conda create -n python39 python=3.9 -y
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
        with:
          repository: "intel-analytics/llm.cpp"
          ref: ${{ inputs.llmcpp-ref }}
          token: ${{ env.github_access_token }}
          submodules: "recursive"
      - name: Build avx512 binary
        shell: bash
        run: |
          scl enable gcc-toolset-11 "cmake -DONLYAVX=OFF -DONLYAVX2=OFF -B build"
          scl enable gcc-toolset-11 "cmake --build build --config Release -j"
      - name: Move avx512 release binary
        shell: bash
        run: |
          mkdir avx512_release
          mv build/quantize-bloom avx512_release/quantize-bloom_avx512
          mv build/libbloom.so avx512_release/libbloom_avx512.so
          mv build/quantize-llama avx512_release/quantize-llama_avx512
          mv build/libllama.so avx512_release/libllama_avx512.so
          mv build/quantize-gptneox avx512_release/quantize-gptneox_avx512
          mv build/libgptneox.so avx512_release/libgptneox_avx512.so
          mv build/quantize-starcoder avx512_release/quantize-starcoder_avx512
          mv build/libstarcoder.so avx512_release/libstarcoder_avx512.so
      - name: Build avx2 binary
        shell: bash
        run: |
          scl enable gcc-toolset-11 "cmake -DONLYAVX=OFF -DONLYAVX2=ON -B build"
          scl enable gcc-toolset-11 "cmake --build build --config Release -j"
      - name: Move avx2 release binary
        shell: bash
        run: |
          mkdir avx2_release
          mv build/libbloom.so avx2_release/libbloom_avx2.so
          mv build/libllama.so avx2_release/libllama_avx2.so
          mv build/libgptneox.so avx2_release/libgptneox_avx2.so
          mv build/libstarcoder.so avx2_release/libstarcoder_avx2.so
      - name: Build avx binary
        shell: bash
        run: |
          scl enable gcc-toolset-11 "cmake -DONLYAVX=ON -DONLYAVX2=OFF -B build"
          scl enable gcc-toolset-11 "cmake --build build --config Release -j"
      - name: Move avx release binary
        shell: bash
        run: |
          mkdir avx_release
          mv build/libbloom.so avx_release/libbloom_avx.so
          mv build/libllama.so avx_release/libllama_avx.so
          mv build/libgptneox.so avx_release/libgptneox_avx.so
          mv build/libstarcoder.so avx_release/libstarcoder_avx.so
      - name: Archive avx512 build files
        uses: actions/upload-artifact@v3
        with:
          name: linux-avx512
          path: |
            avx512_release
      - name: Archive avx2 build files
        uses: actions/upload-artifact@v3
        with:
          name: linux-avx2
          path: |
            avx2_release
      - name: Archive avx build files
        uses: actions/upload-artifact@v3
        with:
          name: linux-avx
          path: |
            avx_release
      - name: Clean up test environment
        if: ${{ always() }}
        shell: bash
        run: |
          make clean
          conda remove -n python39 --all -y

  check-linux-amx-artifact:
    if: ${{contains(inputs.platform, 'Linux')}}
    runs-on: ubuntu-latest
    outputs:
      if-exists: ${{steps.check_artifact.outputs.exists}}
    steps:
      - name: Check if built
        id: check_artifact
        uses: xSAVIKx/artifact-exists-action@v0
        with:
          name: linux-amx

  linux-build-amx:
    runs-on: [self-hosted, amx, almalinux8]
    needs: check-linux-amx-artifact
    if: needs.check-linux-amx-artifact.outputs.if-exists == 'false'
    steps:
      - name: Set access token
        run: |
          echo "github_access_token=${GITHUB_ACCESS_TOKEN}" >> "$GITHUB_ENV"
      - name: Install Build Environment
        shell: bash
        run: |
          export http_proxy=${HTTP_PROXY}
          export https_proxy=${HTTPS_PROXY}
          yum install -y gcc-toolset-11 cmake git
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
        with:
          repository: "intel-analytics/llm.cpp"
          ref: ${{ inputs.llmcpp-ref }}
          token: ${{ env.github_access_token }}
          submodules: "recursive"
      - name: Build amx binary
        shell: bash
        run: |
          scl enable gcc-toolset-11 "cmake -DONLYAVX=OFF -DONLYAVX2=OFF -B build"
          scl enable gcc-toolset-11 "cmake --build build --config Release -j"
      - name: Move amx release binary
        shell: bash
        run: |
          mkdir amx_release
          mv build/quantize-bloom amx_release/quantize-bloom_amx
          mv build/libbloom.so amx_release/libbloom_amx.so
          mv build/quantize-llama amx_release/quantize-llama_amx
          mv build/libllama.so amx_release/libllama_amx.so
          mv build/quantize-gptneox amx_release/quantize-gptneox_amx
          mv build/libgptneox.so amx_release/libgptneox_amx.so
          mv build/quantize-starcoder amx_release/quantize-starcoder_amx
          mv build/libstarcoder.so amx_release/libstarcoder_amx.so
      - name: Archive amx build files
        uses: actions/upload-artifact@v3
        with:
          name: linux-amx
          path: |
            amx_release
      - name: Clean up test environment
        shell: bash
        run: |
          make clean

  check-windows-avx2-artifact:
    if: ${{contains(inputs.platform, 'Windows')}}
    runs-on: ubuntu-latest
    outputs:
      if-exists: ${{steps.check_artifact.outputs.exists}}
    steps:
      - name: Check if built
        id: check_artifact
        uses: xSAVIKx/artifact-exists-action@v0
        with:
          name: windows-avx2

  windows-build-avx2:
    runs-on: [self-hosted, Windows, AVX-VNNI-Build]
    needs: check-windows-avx2-artifact
    if: needs.check-windows-avx2-artifact.outputs.if-exists == 'false'
    steps:
      - name: Set access token
        run: |
          echo "github_access_token=$env:GITHUB_ACCESS_TOKEN" >> $env:GITHUB_ENV
          echo "github_access_token=$env:GITHUB_ACCESS_TOKEN"
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
        with:
          repository: "intel-analytics/llm.cpp"
          ref: ${{ inputs.llmcpp-ref }}
          token: ${{ env.github_access_token }}
          submodules: "recursive"
      - name: Add msbuild to PATH
        uses: microsoft/setup-msbuild@v1.1
        with:
          msbuild-architecture: x64
      - name: Add cmake to PATH
        uses: ilammy/msvc-dev-cmd@v1
      - name: Build binary
        shell: powershell
        run: |
          cmake .
          cmake --build . --config Release -j
      - name: Archive build files
        uses: actions/upload-artifact@v3
        with:
          name: windows-avx2
          path: |
            build/Release

  check-windows-avx-vnni-artifact:
    if: ${{contains(inputs.platform, 'Windows')}}
    runs-on: ubuntu-latest
    outputs:
      if-exists: ${{steps.check_artifact.outputs.exists}}
    steps:
      - name: Check if built
        id: check_artifact
        uses: xSAVIKx/artifact-exists-action@v0
        with:
          name: windows-avx-vnni

  windows-build-avx-vnni:
    runs-on: [self-hosted, Windows, AVX-VNNI-Build]
    needs: check-windows-avx-vnni-artifact
    if: needs.check-windows-avx-vnni-artifact.outputs.if-exists == 'false'
    steps:
      - name: Set access token
        run: |
          echo "github_access_token=$env:GITHUB_ACCESS_TOKEN" >> $env:GITHUB_ENV
          echo "github_access_token=$env:GITHUB_ACCESS_TOKEN"
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
        with:
          repository: "intel-analytics/llm.cpp"
          ref: ${{ inputs.llmcpp-ref }}
          token: ${{ env.github_access_token }}
          submodules: "recursive"
      - name: Add msbuild to PATH
        uses: microsoft/setup-msbuild@v1.1
        with:
          msbuild-architecture: x64
      - name: Add cmake to PATH
        uses: ilammy/msvc-dev-cmd@v1
      - name: Build binary
        shell: powershell
        run: |
          cmake -DAVXVNNI=ON .
          cmake --build . --config Release -j
      - name: Move release binary
        shell: powershell
        run: |
          if (Test-Path ./release) { rm -r -fo release }
          mkdir release
          # mv build/Release/main-bloom.exe release/main-bloom_vnni.exe
          mv build/Release/quantize-bloom.exe release/quantize-bloom_vnni.exe
          mv build/Release/bloom.dll release/libbloom_vnni.dll

          # mv build/Release/main-llama.exe release/main-llama_vnni.exe
          mv build/Release/quantize-llama.exe release/quantize-llama_vnni.exe
          mv build/Release/llama.dll release/libllama_vnni.dll

          # mv build/Release/main-gptneox.exe release/main-gptneox_vnni.exe
          mv build/Release/quantize-gptneox.exe release/quantize-gptneox_vnni.exe
          mv build/Release/gptneox.dll release/libgptneox_vnni.dll

          # mv build/Release/main-starcoder.exe release/main-starcoder_vnni.exe
          mv build/Release/quantize-starcoder.exe release/quantize-starcoder_vnni.exe
          mv build/Release/starcoder.dll release/libstarcoder_vnni.dll
      - name: Archive build files
        uses: actions/upload-artifact@v3
        with:
          name: windows-avx-vnni
          path: |
            release

  check-windows-avx-artifact:
    if: ${{contains(inputs.platform, 'Windows')}}
    runs-on: ubuntu-latest
    outputs:
      if-exists: ${{steps.check_artifact.outputs.exists}}
    steps:
      - name: Check if built
        id: check_artifact
        uses: xSAVIKx/artifact-exists-action@v0
        with:
          name: windows-avx

  windows-build-avx:
    runs-on: [self-hosted, Windows, AVX-VNNI-Build]
    needs: check-windows-avx-artifact
    if: needs.check-windows-avx-artifact.outputs.if-exists == 'false'
    steps:
      - name: Set access token
        run: |
          echo "github_access_token=$env:GITHUB_ACCESS_TOKEN" >> $env:GITHUB_ENV
          echo "github_access_token=$env:GITHUB_ACCESS_TOKEN"
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
        with:
          repository: "intel-analytics/llm.cpp"
          ref: ${{ inputs.llmcpp-ref }}
          token: ${{ env.github_access_token }}
          submodules: "recursive"
      - name: Add msbuild to PATH
        uses: microsoft/setup-msbuild@v1.1
        with:
          msbuild-architecture: x64
      - name: Add cmake to PATH
        uses: ilammy/msvc-dev-cmd@v1
      - name: Build binary
        shell: powershell
        run: |
          cmake -DONLYAVX=ON .
          cmake --build . --config Release -j
      - name: Move release binary
        shell: powershell
        run: |
          if (Test-Path ./release) { rm -r -fo release }
          mkdir release
          mv build/Release/bloom.dll release/libbloom_avx.dll

          mv build/Release/llama.dll release/libllama_avx.dll

          mv build/Release/gptneox.dll release/libgptneox_avx.dll

          mv build/Release/starcoder.dll release/libstarcoder_avx.dll
      - name: Archive build files
        uses: actions/upload-artifact@v3
        with:
          name: windows-avx
          path: |
            release


LLM Example Test

name: LLM Example Test

# Cancel previous runs in the PR when you push new commits
concurrency:
  group: ${{ github.workflow }}-llm-example-tests-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: true

permissions:
  contents: read

# Controls when the action will run.
on:
  # schedule:
  #   - cron: '00 13 * * *' # GMT time, 13:00 GMT == 21:00 China
  pull_request:
    branches: [ main ]
    paths:
      - '.github/workflows/llm_example_tests.yml'
      - '.github/workflows/llm-binary-build.yml'
      - '.github/actions/llm/example-test/action.yml'
      - '.github/actions/llm/setup-llm-env/action.yml'
      - '.github/actions/llm/remove-llm-env/action.yml'
      - '.github/actions/llm/download-llm-binary/action.yml'
      - 'python/llm/dev/test/run-example-tests.sh'
      - 'python/llm/example/**'
  workflow_dispatch:
  workflow_call:

env:
  INT4_CKPT_DIR: ./llm/ggml-actions/stable
  LLM_DIR: ./llm

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  llm-cpp-build:
    uses: ./.github/workflows/llm-binary-build.yml
  llm-example-test:
    needs: llm-cpp-build
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11"]
        instruction: ["AVX512"]
    runs-on: [ self-hosted, llm,"${{matrix.instruction}}", ubuntu-20.04-lts ]
    env:
      THREAD_NUM: 24
    steps:
      - uses: actions/checkout@ee0669bd1cc54295c223e0bb666b733df41de1c5 # actions/checkout@v2
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v2
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          python -m pip install --upgrade setuptools==58.0.4
          python -m pip install --upgrade wheel

      - name: Download llm binary
        uses: ./.github/actions/llm/download-llm-binary

      - name: Run LLM install (all) test
        uses: ./.github/actions/llm/setup-llm-env
        env:
          ANALYTICS_ZOO_ROOT: ${{ github.workspace }}

      - name: Run LLM example test
        uses: ./.github/actions/llm/example-test
        env:
          ANALYTICS_ZOO_ROOT: ${{ github.workspace }}

      # - name: Clean up test environment
      #   uses: ./.github/actions/llm/remove-llm-env
      #   env:
      #     ANALYTICS_ZOO_ROOT: ${{ github.workspace }}


LLM Unit Tests

name: LLM Unit Tests

# Cancel previous runs in the PR when you push new commits
concurrency:
  group: ${{ github.workflow }}-llm-unittest-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: true

permissions:
  contents: read

# Controls when the action will run.
on:
  # Triggers the workflow on push or pull request events but only for the main branch
  push:
    branches: [main]
    paths:
      - "python/llm/**"
      - ".github/workflows/llm_unit_tests.yml"
      - ".github/workflows/llm-binary-build.yml"
      - ".github/actions/llm/setup-llm-env/action.yml"
      - ".github/actions/llm/remove-llm-env/action.yml"
      - ".github/actions/llm/cli-test-linux/action.yml"
      - ".github/actions/llm/cli-test-windows/action.yml"
      - ".github/actions/llm/download-llm-binary/action.yml"
  pull_request:
    branches: [main]
    paths:
      - "python/llm/**"
      - ".github/workflows/llm_unit_tests.yml"
      - ".github/workflows/llm-binary-build.yml"
      - ".github/actions/llm/setup-llm-env/action.yml"
      - ".github/actions/llm/remove-llm-env/action.yml"
      - ".github/actions/llm/cli-test-linux/action.yml"
      - ".github/actions/llm/cli-test-windows/action.yml"
      - ".github/actions/llm/download-llm-binary/action.yml"
  workflow_dispatch:
  workflow_call:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  llm-cpp-build:
    uses: ./.github/workflows/llm-binary-build.yml
  setup-python-version:
    runs-on: ubuntu-latest
    outputs:
      python-version: ${{ steps.setup-python-version.outputs.python-version }}
    steps:
      - name: setup-python-version
        id: setup-python-version
        run: |
          if [ ${{ github.event_name }} == 'schedule' ]; then
            python_version='["3.9", "3.10", "3.11"]'
          else
            python_version='["3.11"]'
          fi
          list=$(echo ${python_version} | jq -c)
          echo "python-version=${list}" >> "$GITHUB_OUTPUT"
  llm-unit-test:
    needs: [setup-python-version, llm-cpp-build]
    strategy:
      fail-fast: false
      matrix:
        os: [windows, ubuntu-20.04-lts]
        python-version: ${{ fromJson(needs.setup-python-version.outputs.python-version) }}
        include:
          - os: windows
            instruction: AVX-VNNI-UT
          - os: ubuntu-20.04-lts
            instruction: avx512
    runs-on: [self-hosted, llm, "${{matrix.instruction}}", "${{matrix.os}}"]
    env:
      THREAD_NUM: 24
      ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
    steps:
      - name: Set model directories
        shell: bash
        run: |
          echo "DATASET_DIR=${{ github.workspace }}/../llm/datasets" >> "$GITHUB_ENV"
          echo "ORIGIN_DIR=${{ github.workspace }}/../llm/origin-models" >> "$GITHUB_ENV"
          echo "INT4_CKPT_DIR=${{ github.workspace }}/../llm/converted-models" >> "$GITHUB_ENV"
      - name: Create model directories
        shell: bash
        run: |
          if [ ! -d $DATASET_DIR ]; then
            mkdir -p $DATASET_DIR
          fi
          if [ ! -d $ORIGIN_DIR ]; then
            mkdir -p $ORIGIN_DIR
          fi
          if [ ! -d $INT4_CKPT_DIR ]; then
            mkdir -p $INT4_CKPT_DIR
          fi
      - name: Set environment variables
        shell: bash
        run: |
          echo "SPEECH_DATASET_PATH=${DATASET_DIR}/librispeech_asr_dummy" >> "$GITHUB_ENV"
          echo "COMMON_VOICE_PATH=${DATASET_DIR}/common_voice" >> "$GITHUB_ENV"

          echo "LLAMA_ORIGIN_PATH=${ORIGIN_DIR}/llama-7b-hf" >> "$GITHUB_ENV"
          echo "BLOOM_ORIGIN_PATH=${ORIGIN_DIR}/bloom-7b1" >> "$GITHUB_ENV"
          echo "ORIGINAL_CHATGLM2_6B_PATH=${ORIGIN_DIR}/chatglm2-6b" >> "$GITHUB_ENV"
          echo "ORIGINAL_CODESHELL_7B_PATH=${ORIGIN_DIR}/CodeShell-7B-Chat" >> "$GITHUB_ENV"
          echo "ORIGINAL_WHISPER_TINY_PATH=${ORIGIN_DIR}/whisper-tiny" >> "$GITHUB_ENV"
          echo "MISTRAL_ORIGIN_PATH=${ORIGIN_DIR}/Mistral-7B-v0.1" >> "$GITHUB_ENV"
          echo "LLAMA2_7B_ORIGIN_PATH=${ORIGIN_DIR}/Llama-2-7b-chat-hf" >> "$GITHUB_ENV"
          echo "VICUNA_7B_1_3_ORIGIN_PATH=${ORIGIN_DIR}/vicuna-7b-v1.3" >> "$GITHUB_ENV"

          echo "LLAMA_INT4_CKPT_PATH=${INT4_CKPT_DIR}/bigdl_llm_llama_7b_q4_0.bin" >> "$GITHUB_ENV"
          echo "GPTNEOX_INT4_CKPT_PATH=${INT4_CKPT_DIR}/bigdl_llm_redpajama_7b_q4_0.bin" >> "$GITHUB_ENV"
          echo "BLOOM_INT4_CKPT_PATH=${INT4_CKPT_DIR}/bigdl_llm_bloom_7b_q4_0.bin" >> "$GITHUB_ENV"
          echo "STARCODER_INT4_CKPT_PATH=${INT4_CKPT_DIR}/bigdl_llm_santacoder_1b_q4_0.bin" >> "$GITHUB_ENV"
          echo "CHATGLM_INT4_CKPT_PATH=${INT4_CKPT_DIR}/chatglm2-6b-q4_0.bin" >> "$GITHUB_ENV"
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        shell: bash
        run: |
          python -m pip install --upgrade pip
          python -m pip install --upgrade setuptools==58.0.4
          python -m pip install --upgrade wheel

          # May remove later
          pip uninstall sentence-transformers -y || true

      - name: Download llm binary
        uses: ./.github/actions/llm/download-llm-binary

      - name: Run LLM install (all) test
        uses: ./.github/actions/llm/setup-llm-env

      - name: Download ckpt & original models
        shell: bash
        run: |
          if [ ! -e $LLAMA_INT4_CKPT_PATH ]; then
            echo "Directory $LLAMA_INT4_CKPT_PATH not found. Downloading from FTP server..."
            echo "wget --no-verbose $LLM_FTP_URL/llm/ggml-actions/stable/bigdl_llm_llama_7b_q4_0.bin -P $INT4_CKPT_DIR"
            wget --no-verbose $LLM_FTP_URL/llm/ggml-actions/stable/bigdl_llm_llama_7b_q4_0.bin -P $INT4_CKPT_DIR
          fi
          if [ ! -e $GPTNEOX_INT4_CKPT_PATH ]; then
            echo "Directory $GPTNEOX_INT4_CKPT_PATH not found. Downloading from FTP server..."
            wget --no-verbose $LLM_FTP_URL/llm/ggml-actions/stable/bigdl_llm_redpajama_7b_q4_0.bin -P $INT4_CKPT_DIR
          fi
          if [ ! -e $BLOOM_INT4_CKPT_PATH ]; then
            echo "Directory $BLOOM_INT4_CKPT_PATH not found. Downloading from FTP server..."
            wget --no-verbose $LLM_FTP_URL/llm/ggml-actions/stable/bigdl_llm_bloom_7b_q4_0.bin -P $INT4_CKPT_DIR
          fi
          if [ ! -e $STARCODER_INT4_CKPT_PATH ]; then
            echo "Directory $STARCODER_INT4_CKPT_PATH not found. Downloading from FTP server..."
            wget --no-verbose $LLM_FTP_URL/llm/ggml-actions/stable/bigdl_llm_santacoder_1b_q4_0.bin -P $INT4_CKPT_DIR
          fi
          # if [ ! -e $CHATGLM_INT4_CKPT_PATH ]; then
          #   echo "Directory $CHATGLM_INT4_CKPT_PATH not found. Downloading from FTP server..."
          #   wget --no-verbose $LLM_FTP_URL/llm/ggml-actions/stable/chatglm2-6b-q4_0.bin -P $INT4_CKPT_DIR
          # fi
          if [ ! -d $ORIGINAL_CHATGLM2_6B_PATH ]; then
            echo "Directory $ORIGINAL_CHATGLM2_6B_PATH not found. Downloading from FTP server..."
            echo "wget -r -nH --no-verbose --cut-dirs=2 $LLM_FTP_URL/llm/updated_for_4.36/chatglm2-6b -P $ORIGIN_DIR"
            wget -r -nH --no-verbose --cut-dirs=2 $LLM_FTP_URL/llm/updated_for_4.36/chatglm2-6b -P $ORIGIN_DIR
          fi
          if [ ! -d $ORIGINAL_CODESHELL_7B_PATH ]; then
            echo "Directory $ORIGINAL_CODESHELL_7B_PATH not found. Downloading from FTP server..."
            echo "wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/CodeShell-7B-Chat -P $ORIGIN_DIR"
            wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/CodeShell-7B-Chat -P $ORIGIN_DIR
          fi
          if [ ! -d $ORIGINAL_WHISPER_TINY_PATH ]; then
            echo "Directory $ORIGINAL_WHISPER_TINY_PATH not found. Downloading from FTP server..."
            echo "wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/whisper-tiny -P $ORIGIN_DIR"
            wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/whisper-tiny -P $ORIGIN_DIR
          fi
          if [ ! -d $MISTRAL_ORIGIN_PATH ]; then
            echo "Directory $MISTRAL_ORIGIN_PATH not found. Downloading from FTP server..."
            echo "wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/Mistral-7B-v0.1 -P $ORIGIN_DIR"
            wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/Mistral-7B-v0.1 -P $ORIGIN_DIR
          fi
          if [ ! -d $LLAMA_ORIGIN_PATH ]; then
            echo "Directory $LLAMA_ORIGIN_PATH not found. Downloading from FTP server..."
            echo "wget --no-verbose $LLM_FTP_URL/llm/llama-7b-hf -P $ORIGIN_DIR"
            wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/llama-7b-hf -P $ORIGIN_DIR
          fi
          if [ ! -d $BLOOM_ORIGIN_PATH ]; then
            echo "Directory $BLOOM_ORIGIN_PATH not found. Downloading from FTP server..."
            echo "wget --no-verbose $LLM_FTP_URL/llm/bloom-7b1 -P $ORIGIN_DIR"
            wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/bloom-7b1 -P $ORIGIN_DIR
          fi
          if [ ! -d $SPEECH_DATASET_PATH ]; then
            echo "Directory $SPEECH_DATASET_PATH not found. Downloading from FTP server..."
            echo "wget -r -nH --no-verbose --cut-dirs=2 $LLM_FTP_URL/llm/datasets/librispeech_asr_dummy -P $DATASET_DIR"
            wget -r -nH --no-verbose --cut-dirs=2 $LLM_FTP_URL/llm/datasets/librispeech_asr_dummy -P $DATASET_DIR
          fi
          if [ ! -d $COMMON_VOICE_PATH ]; then
            echo "Directory $COMMON_VOICE_PATH not found. Downloading from FTP server..."
            echo "wget -r -nH --no-verbose --cut-dirs=2 $LLM_FTP_URL/llm/datasets/common_voice -P $DATASET_DIR"
            wget -r -nH --no-verbose --cut-dirs=2 $LLM_FTP_URL/llm/datasets/common_voice -P $DATASET_DIR
          fi
          if [ ! -d $LLAMA2_7B_ORIGIN_PATH ]; then
            echo "Directory $LLAMA2_7B_ORIGIN_PATH not found. Downloading from FTP server..."
            wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/Llama-2-7b-chat-hf -P $ORIGIN_DIR
          fi
          if [ ! -d $VICUNA_7B_1_3_ORIGIN_PATH ]; then
            echo "Directory $VICUNA_7B_1_3_ORIGIN_PATH not found. Downloading from FTP server..."
            wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/vicuna-7b-v1.3 -P $ORIGIN_DIR
          fi

      - name: Run LLM cli test (Linux)
        if: runner.os == 'Linux'
        uses: ./.github/actions/llm/cli-test-linux
      - name: Run LLM cli test (Windows)
        if: runner.os == 'Windows'
        uses: ./.github/actions/llm/cli-test-windows
      - name: Run LLM inference test
        shell: bash
        run: |
          python -m pip install einops datasets librosa openai-whisper
          bash python/llm/test/run-llm-inference-tests.sh
      - name: Run LLM langchain test
        shell: bash
        run: |
          pip install -U langchain==0.0.184
          pip install -U chromadb==0.3.25
          pip install -U pandas==2.0.3
          bash python/llm/test/run-llm-langchain-tests.sh
      - name: Run LLM llamaindex test
        shell: bash
        run: |
          pip install llama-index-readers-file llama-index-vector-stores-postgres llama-index-embeddings-huggingface
          pip install transformers==4.36.2
          pip install "pydantic>=2.0.0"
          bash python/llm/test/run-llm-llamaindex-tests.sh
      - name: Run sentence-transformers uninstallation
        if: ${{ always() }}
        shell: bash
        run: |
          pip uninstall sentence-transformers -y || true

  llm-unit-test-on-arc:
    needs: [setup-python-version, llm-cpp-build]
    strategy:
      fail-fast: false
      matrix:
        runner: ['arc-ut', 'arc-ut-win']
        pytorch-version: ['2.1']
        python-version: ${{ fromJson(needs.setup-python-version.outputs.python-version) }}
    runs-on: [self-hosted, llm, "${{ matrix.runner }}"]
    env:
      # OMP_NUM_THREADS: 16
      # THREAD_NUM: 16
      ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
    steps:
      - name: Set environment variables
        shell: bash
        run: |
          echo "DATASET_DIR=${ORIGIN_DIR}/../datasets" >> "$GITHUB_ENV"
          echo "YAHMA_ALPACA_CLEANED_PATH=${ORIGIN_DIR}/../datasets/yahma_alpaca_cleaned" >> "$GITHUB_ENV"
          echo "SPEECH_DATASET_PATH=${ORIGIN_DIR}/../datasets/librispeech_asr_dummy" >> "$GITHUB_ENV"

          echo "LLAMA2_7B_ORIGIN_PATH=${ORIGIN_DIR}/Llama-2-7b-chat-hf" >> "$GITHUB_ENV"
          echo "CHATGLM2_6B_ORIGIN_PATH=${ORIGIN_DIR}/chatglm2-6b" >> "$GITHUB_ENV"
          echo "FALCON_7B_ORIGIN_PATH=${ORIGIN_DIR}/falcon-7b-instruct-with-patch" >> "$GITHUB_ENV"
          echo "MPT_7B_ORIGIN_PATH=${ORIGIN_DIR}/mpt-7b-chat" >> "$GITHUB_ENV"
          echo "WHISPER_TINY_ORIGIN_PATH=${ORIGIN_DIR}/whisper-tiny" >> "$GITHUB_ENV"
          echo "MISTRAL_7B_INSTRUCT_V0_1_ORIGIN_PATH=${ORIGIN_DIR}/Mistral-7B-Instruct-v0.1" >> "$GITHUB_ENV"
          echo "BAICHUAN2_7B_ORIGIN_PATH=${ORIGIN_DIR}/Baichuan2-7B-Chat" >> "$GITHUB_ENV"
          echo "QWEN_7B_ORIGIN_PATH=${ORIGIN_DIR}/Qwen-7B-Chat" >> "$GITHUB_ENV"
          echo "VICUNA_7B_1_3_ORIGIN_PATH=${ORIGIN_DIR}/vicuna-7b-v1.3" >> "$GITHUB_ENV"
      - name: Checkout repo
        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        shell: bash
        run: |
          python -m pip install --upgrade pip
          python -m pip install --upgrade "setuptools<70.0.0"
          python -m pip install --upgrade wheel
          python -m pip install --upgrade notebook

          # May remove later
          pip uninstall sentence-transformers -y || true

          # On Windows, we need to add "Python3_ROOT_DIR/bin" to path to make libuv work
          if [[ "$RUNNER_OS" == "Windows" ]]; then
            echo $Python3_ROOT_DIR'\bin\'
            echo $Python3_ROOT_DIR'\bin\' >> $GITHUB_PATH
          fi

      - name: Download llm binary
        uses: ./.github/actions/llm/download-llm-binary

      - name: Install IPEX-LLM for xpu
        uses: ./.github/actions/llm/setup-llm-env
        with:
          extra-dependency: "xpu_${{ matrix.pytorch-version }}"

      - name: Test installed xpu version
        shell: bash
        run: |
          # Specific oneapi position on arc ut test machines
          if [[ "$RUNNER_OS" == "Linux" ]]; then
            if [[ '${{ matrix.pytorch-version }}' == '2.1' ]]; then
              source /opt/intel/oneapi/setvars.sh
            elif [[ '${{ matrix.pytorch-version }}' == '2.0' ]]; then
              source /home/arda/intel/oneapi/setvars.sh
            fi
          fi
          bash python/llm/test/run-llm-install-tests.sh

      - name: Download LLMs and datasets
        shell: bash
        run: |
          if [ ! -d $LLAMA2_7B_ORIGIN_PATH ]; then
            echo "Directory $LLAMA2_7B_ORIGIN_PATH not found. Downloading from FTP server..."
            wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/Llama-2-7b-chat-hf -P $ORIGIN_DIR
          fi
          if [ ! -d $CHATGLM2_6B_ORIGIN_PATH ]; then
            echo "Directory $CHATGLM2_6B_ORIGIN_PATH not found. Downloading from FTP server..."
            wget -r -nH --no-verbose --cut-dirs=2 $LLM_FTP_URL/llm/updated_for_4.36/chatglm2-6b -P $ORIGIN_DIR
          fi
          if [ ! -d $FALCON_7B_ORIGIN_PATH ]; then
            echo "Directory $FALCON_7B_ORIGIN_PATH not found. Downloading from FTP server..."
            wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/falcon-7b-instruct-with-patch -P $ORIGIN_DIR
          fi
          if [ ! -d $MPT_7B_ORIGIN_PATH ]; then
            echo "Directory $MPT_7B_ORIGIN_PATH not found. Downloading from FTP server..."
            wget -r -nH --no-verbose --cut-dirs=2 $LLM_FTP_URL/llm/updated_for_4.36/mpt-7b-chat -P $ORIGIN_DIR
          fi
          if [ ! -d $WHISPER_TINY_ORIGIN_PATH ]; then
            echo "Directory $WHISPER_TINY_ORIGIN_PATH not found. Downloading from FTP server..."
            wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/whisper-tiny -P $ORIGIN_DIR
          fi
          if [ ! -d $DATASET_DIR ]; then
            mkdir -p $DATASET_DIR
          fi
          if [ ! -d $YAHMA_ALPACA_CLEANED_PATH ]; then
            echo "Directory $YAHMA_ALPACA_CLEANED_PATH not found. Downloading from FTP server..."
            wget -r -nH --no-verbose --cut-dirs=2 $LLM_FTP_URL/llm/datasets/yahma_alpaca_cleaned -P $DATASET_DIR
          fi
          if [ ! -d $SPEECH_DATASET_PATH ]; then
            echo "Directory $SPEECH_DATASET_PATH not found. Downloading from FTP server..."
            wget -r -nH --no-verbose --cut-dirs=2 $LLM_FTP_URL/llm/datasets/librispeech_asr_dummy -P $DATASET_DIR
          fi
          if [ ! -d $MISTRAL_7B_INSTRUCT_V0_1_ORIGIN_PATH ]; then
            echo "Directory $MISTRAL_7B_INSTRUCT_V0_1_ORIGIN_PATH not found. Downloading from FTP server..."
            wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/Mistral-7B-Instruct-v0.1 -P $ORIGIN_DIR
          fi
          if [ ! -d $QWEN_7B_ORIGIN_PATH ]; then
            echo "Directory $QWEN_7B_ORIGIN_PATH not found. Downloading from FTP server..."
            wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/Qwen-7B-Chat -P $ORIGIN_DIR
          fi
          if [ ! -d $BAICHUAN2_7B_ORIGIN_PATH ]; then
            echo "Directory $BAICHUAN2_7B_ORIGIN_PATH not found. Downloading from FTP server..."
            wget -r -nH --no-verbose --cut-dirs=2 $LLM_FTP_URL/llm/updated_for_4.36/Baichuan2-7B-Chat -P $ORIGIN_DIR
          fi
          if [ ! -d $VICUNA_7B_1_3_ORIGIN_PATH ]; then
            echo "Directory $VICUNA_7B_1_3_ORIGIN_PATH not found. Downloading from FTP server..."
            wget -r -nH --no-verbose --cut-dirs=1 $LLM_FTP_URL/llm/vicuna-7b-v1.3 -P $ORIGIN_DIR
          fi

      - name: Run LLM inference test
        shell: bash
        run: |
          # Specific oneapi position on arc ut test machines
          if [[ "$RUNNER_OS" == "Linux" ]]; then
            if [[ '${{ matrix.pytorch-version }}' == '2.1' ]]; then
              source /opt/intel/oneapi/setvars.sh
            elif [[ '${{ matrix.pytorch-version }}' == '2.0' ]]; then
              source /home/arda/intel/oneapi/setvars.sh
            fi
          fi
          python -m pip install datasets librosa soundfile einops tiktoken transformers_stream_generator

          bash python/llm/test/run-llm-inference-tests-gpu.sh

      - name: Run LLM example tests
        shell: bash
        run: |
          python -m pip uninstall datasets -y
          python -m pip install transformers==4.36.0 datasets peft==0.10.0
          python -m pip install bitsandbytes scipy
          # Specific oneapi position on arc ut test machines
          if [[ "$RUNNER_OS" == "Linux" ]]; then
            if [[ '${{ matrix.pytorch-version }}' == '2.1' ]]; then
              source /opt/intel/oneapi/setvars.sh
            elif [[ '${{ matrix.pytorch-version }}' == '2.0' ]]; then
              source /home/arda/intel/oneapi/setvars.sh
            fi
          fi
          bash python/llm/test/run-llm-example-tests-gpu.sh

      - name: Get Langchain version
        shell: bash
        id: get_langchain_version
        run: |
          pip install langchain
          LANGCHAIN_VERSION=$(pip show langchain | grep Version | cut -d " " -f 2)
          LANGCHAIN_REF="langchain==$LANGCHAIN_VERSION"
          echo "langchain_ver=$LANGCHAIN_REF" >> $GITHUB_OUTPUT

      - name: Checkout Langchain repo
        uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744
        with:
          repository: "langchain-ai/langchain"
          ref: ${{ join(steps.get_langchain_version.outputs.*, '\n') }}
          path: langchain_upstream

      - name: Run LLM langchain GPU test
        shell: bash
        run: |
          pip install -U langchain==0.0.184
          pip install -U chromadb==0.3.25
          pip install -U pandas==2.0.3
          # Specific oneapi position on arc ut test machines
          if [[ "$RUNNER_OS" == "Linux" ]]; then
            if [[ '${{ matrix.pytorch-version }}' == '2.1' ]]; then
              source /opt/intel/oneapi/setvars.sh
            elif [[ '${{ matrix.pytorch-version }}' == '2.0' ]]; then
              source /home/arda/intel/oneapi/setvars.sh
            fi
          fi
          bash python/llm/test/run-llm-langchain-tests-gpu.sh

          pip install -U langchain
          pip install -U langchain-community
          pip install --pre --upgrade bigdl-llm[all]
          bash python/llm/test/run-langchain-upstream-tests.sh

      - name: Run LLM llamaindex GPU test
        shell: bash
        run: |
          pip install llama-index-readers-file llama-index-vector-stores-postgres llama-index-embeddings-huggingface
          # Specific oneapi position on arc ut test machines
          if [[ '${{ matrix.pytorch-version }}' == '2.1' ]]; then
            pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
            if [[ "$RUNNER_OS" == "Linux" ]]; then
              source /opt/intel/oneapi/setvars.sh
            fi
          elif [[ '${{ matrix.pytorch-version }}' == '2.0' ]]; then
            pip install --pre --upgrade ipex-llm[xpu_2.0] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
            if [[ "$RUNNER_OS" == "Linux" ]]; then
              source /home/arda/intel/oneapi/setvars.sh
            fi
          fi
          pip install transformers==4.36.2
          pip install "pydantic>=2.0.0"
          bash python/llm/test/run-llm-llamaindex-tests-gpu.sh
      - name: Run sentence-transformers uninstallation
        if: ${{ always() }}
        shell: bash
        run: |
          pip uninstall sentence-transformers -y || true


LLM Performance Test

name: LLM Performance Test

# Cancel previous runs in the PR when you push new commits
concurrency:
  group: ${{ github.workflow }}-llm-performance-tests-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: true

permissions:
  contents: read

# Controls when the action will run.
on:
  schedule:
    - cron: "30 16 * * *" # GMT time, 16:30 GMT == 00:30 China
  # please uncomment it for PR tests
  # pull_request:
  #   branches: [main]
  #   paths:
  #     - ".github/workflows/llm_performance_tests.yml"
  #     - "python/llm/test/benchmark/**"
  #     - "python/llm/dev/benchmark/all-in-one/**"
  workflow_dispatch:
    inputs:
      arc:
        description: "If trigger performance test on Arc"
        required: false
        type: boolean
        default: true
      spr:
        description: "If trigger performance test on SPR"
        required: false
        type: boolean
        default: true
      core:
        description: "If trigger performance test on Core"
        required: false
        type: boolean
        default: true
      igpu:
        description: "If trigger performance test on iGPU"
        required: false
        type: boolean
        default: true
  workflow_call:

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  # llm-cpp-build: # please uncomment it for PR tests
  #   uses: ./.github/workflows/llm-binary-build.yml

  llm-performance-test-on-arc:
    if: ${{ github.event.schedule || ( github.event_name == 'workflow_dispatch' && inputs.arc ) || github.event.inputs.artifact == 'llm-performance-test-on-arc' || github.event.inputs.artifact == 'all' }} # please comment it for PR tests
    # needs: llm-cpp-build # please uncomment it for PR tests
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11"]
    runs-on: [self-hosted, llm, perf]
    env:
      OMP_NUM_THREADS: 16
      THREAD_NUM: 16
      ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
      CSV_SAVE_PATH: ${{ (github.event.schedule || github.event_name == 'workflow_dispatch') && '/mnt/disk1/nightly_perf_gpu/' || '/mnt/disk1/pr_perf_gpu/' }}

    steps:
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        shell: bash
        # pip install transformers_stream_generator for model internlm-chat-7b-8k
        # pip install tiktoken for model Qwen-7B-Chat-10-12
        run: |
          python -m pip install --upgrade pip
          python -m pip install --upgrade wheel
          python -m pip install --upgrade omegaconf
          python -m pip install --upgrade pandas
          python -m pip install --upgrade einops
          python -m pip install --upgrade transformers_stream_generator
          python -m pip install --upgrade tiktoken

      # please uncomment it and comment the "Install IPEX-LLM from Pypi" part for PR tests
      # - name: Download llm binary
      #   uses: ./.github/actions/llm/download-llm-binary

      # - name: Run LLM install (all) test
      #   uses: ./.github/actions/llm/setup-llm-env
      #   with:
      #     extra-dependency: "xpu_2.1"

      - name: Install IPEX-LLM from Pypi
        shell: bash
        run: |
          pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
          test_version_date=`date -d 'yesterday' '+%Y%m%d'`
          if ! pip show ipex-llm | grep $test_version_date; then
            echo "Did not install ipex-llm with excepted version $test_version_date"
            exit 1
          fi

      - name: Test installed xpu version
        shell: bash
        run: |
          source /opt/intel/oneapi/setvars.sh
          bash python/llm/test/run-llm-install-tests.sh

      - name: Test on xpu(transformers==4.36.2)
        shell: bash
        run: |
          date_for_test_version=$(date -d yesterday +%Y-%m-%d)
          sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py
          source /opt/intel/oneapi/setvars.sh
          export USE_XETLA=OFF
          export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
          cp python/llm/test/benchmark/arc-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
          cd python/llm/dev/benchmark/all-in-one
          mkdir test_batch1
          mkdir test_batch2
          mkdir test_batch4
          # batch_size 1
          # hide time info
          sed -i 's/str(end - st)/"xxxxxx"/g' run.py
          # change csv name
          sed -i 's/{today}/{today}_test1_batch1/g' run.py
          python run.py
          mv *.csv test_batch1
          # batch_size 2
          cd ../../../../../
          cp python/llm/test/benchmark/arc-perf-test-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
          cd python/llm/dev/benchmark/all-in-one
          # change csv name
          sed -i 's/batch1/batch2/g' run.py
          python run.py
          mv *.csv test_batch2
          # batch_size 4
          cd ../../../../../
          cp python/llm/test/benchmark/arc-perf-test-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml
          cd python/llm/dev/benchmark/all-in-one
          # change csv name
          sed -i 's/batch2/batch4/g' run.py
          python run.py
          mv *.csv test_batch4

      - name: Test on xpu(transformers==4.37.0)
        shell: bash
        run: |
          source /opt/intel/oneapi/setvars.sh
          export USE_XETLA=OFF
          export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
          # upgrade transformers for model Qwen/Qwen1.5-7B-Chat
          python -m pip install transformers==4.37.0
          # batch_size 1
          cp python/llm/test/benchmark/arc-perf-transformers-437.yaml python/llm/dev/benchmark/all-in-one/config.yaml
          cd python/llm/dev/benchmark/all-in-one
          # change csv name
          sed -i 's/test1_batch4/test2_batch1/g' run.py
          python run.py
          mv *.csv test_batch1
          # batch_size 2
          cd ../../../../../
          cp python/llm/test/benchmark/arc-perf-transformers-437-batch2.yaml python/llm/dev/benchmark/all-in-one/config.yaml
          cd python/llm/dev/benchmark/all-in-one
          # change csv name
          sed -i 's/batch1/batch2/g' run.py
          python run.py
          mv *.csv test_batch2
          # batch_size 4
          cd ../../../../../
          cp python/llm/test/benchmark/arc-perf-transformers-437-batch4.yaml python/llm/dev/benchmark/all-in-one/config.yaml
          cd python/llm/dev/benchmark/all-in-one
          # change csv name
          sed -i 's/batch2/batch4/g' run.py
          python run.py
          mv *.csv test_batch4

      - name: Test on xpu(transformers==4.40.0)
        shell: bash
        run: |
          source /opt/intel/oneapi/setvars.sh
          export USE_XETLA=OFF
          export SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS=1
          # upgrade transformers for model Qwen/Qwen1.5-MoE-A2.7B-Chat
          python -m pip install transformers==4.40.0
          python -m pip install trl
          # batch_size 1
          cp python/llm/test/benchmark/arc-perf-transformers-440.yaml python/llm/dev/benchmark/all-in-one/config.yaml
          cd python/llm/dev/benchmark/all-in-one
          # change csv name
          sed -i 's/test2_batch4/test3_batch1/g' run.py
          python run.py
          mv *.csv test_batch1

      - name: Concat csv and generate html
        shell: bash
        run: |
          # batch_size 1
          cd python/llm/dev/benchmark/all-in-one/test_batch1
          python ../../../../test/benchmark/concat_csv.py
          for file in *.csv; do
              if [[ $file != *test* ]]; then
                  cp "$file" $CSV_SAVE_PATH/batch_size_1
              fi
          done
          python -m pip install pandas==1.5.3
          cd ../../../../test/benchmark
          python csv_to_html.py -f $CSV_SAVE_PATH/batch_size_1
          # batch_size 2
          cd ../../../../
          cd python/llm/dev/benchmark/all-in-one/test_batch2
          python ../../../../test/benchmark/concat_csv.py
          for file in *.csv; do
              if [[ $file != *test* ]]; then
                  cp "$file" $CSV_SAVE_PATH/batch_size_2
              fi
          done
          cd ../../../../test/benchmark
          python csv_to_html.py -f $CSV_SAVE_PATH/batch_size_2
          # batch_size 4
          cd ../../../../
          cd python/llm/dev/benchmark/all-in-one/test_batch4
          python ../../../../test/benchmark/concat_csv.py
          for file in *.csv; do
              if [[ $file != *test* ]]; then
                  cp "$file" $CSV_SAVE_PATH/batch_size_4
              fi
          done
          cd ../../../../test/benchmark
          python csv_to_html.py -f $CSV_SAVE_PATH/batch_size_4

      - name: Merge and sort csv files of multiple batches and generate html
        shell: bash
        run: |
          cd python/llm/test/benchmark
          mkdir merged_temp
          # go through all the files and go to merged_temp
          cd ../../dev/benchmark/all-in-one/test_batch1
          for file in *.csv; do
              if [[ $file != *test* ]]; then
                  cp "$file" ../../../../test/benchmark/merged_temp
              fi
          done
          cd ../test_batch2
          for file in *.csv; do
              if [[ $file != *test* ]]; then
                  cp "$file" ../../../../test/benchmark/merged_temp
              fi
          done
          cd ../test_batch4
          for file in *.csv; do
              if [[ $file != *test* ]]; then
                  cp "$file" ../../../../test/benchmark/merged_temp
              fi
          done
          cd ../../../../test/benchmark
          python merge_csv_batch.py -f ./merged_temp
          cd merged_temp
          find . -name "*batch*.csv" -delete
          for file in *.csv; do
              cp "$file" $CSV_SAVE_PATH/merged
          done
          cd ..
          python csv_to_html.py -f $CSV_SAVE_PATH/merged
          rm -r merged_temp

      - name: Update html in parent folder
        shell: bash
        run: |
          cd python/llm/test/benchmark
          python update_html_in_parent_folder.py -f $CSV_SAVE_PATH

      - name: Check and upload results to ftp
        shell: bash
        run: |
          # batch_size 1
          cd python/llm/dev/benchmark/all-in-one/test_batch1
          python ../../../../test/benchmark/check_results.py -c test1 -y ../../../../test/benchmark/arc-perf-test.yaml
          python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437.yaml
          python ../../../../test/benchmark/check_results.py -c test3 -y ../../../../test/benchmark/arc-perf-transformers-440.yaml
          find . -name "*test*.csv" -delete
          if [ ${{ github.event_name }} == "schedule" ] || [ ${{ github.event_name }} == "workflow_dispatch" ]; then
            curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
          fi
          cd ../
          rm -r test_batch1
          # batch_size 2
          cd test_batch2
          python ../../../../test/benchmark/check_results.py -c test1 -y ../../../../test/benchmark/arc-perf-test-batch2.yaml
          python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437-batch2.yaml
          find . -name "*test*.csv" -delete
          if [ ${{ github.event_name }} == "schedule" ] || [ ${{ github.event_name }} == "workflow_dispatch" ]; then
            curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
          fi
          cd ../
          rm -r test_batch2
          # batch_size 4
          cd test_batch4
          python ../../../../test/benchmark/check_results.py -c test1 -y ../../../../test/benchmark/arc-perf-test-batch4.yaml
          python ../../../../test/benchmark/check_results.py -c test2 -y ../../../../test/benchmark/arc-perf-transformers-437-batch4.yaml
          find . -name "*test*.csv" -delete
          if [ ${{ github.event_name }} == "schedule" ] || [ ${{ github.event_name }} == "workflow_dispatch" ]; then
            curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/gpu/
          fi
          cd ../
          rm -r test_batch4

  llm-performance-test-on-spr:
    if: ${{ github.event.schedule || ( github.event_name == 'workflow_dispatch' && inputs.spr ) || github.event.inputs.artifact == 'llm-performance-test-on-spr' || github.event.inputs.artifact == 'all' }} # please comment it for PR tests
    # needs: llm-cpp-build # please uncomment it for PR tests
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11"]
    runs-on: [self-hosted, llm, spr-perf]
    env:
      OMP_NUM_THREADS: 16
      THREAD_NUM: 16
      ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
    steps:
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        shell: bash
        run: |
          python -m pip install --upgrade pip
          python -m pip install --upgrade wheel
          python -m pip install --upgrade omegaconf
          python -m pip install --upgrade pandas
          python -m pip install --upgrade einops
          python -m pip install --upgrade tiktoken
          python -m pip install --upgrade transformers_stream_generator

      # please uncomment it and comment the "Install IPEX-LLM from Pypi" part for PR tests
      # - name: Download llm binary
      #   uses: ./.github/actions/llm/download-llm-binary

      # - name: Run LLM install (all) test
      #   uses: ./.github/actions/llm/setup-llm-env

      - name: Install IPEX-LLM from Pypi
        shell: bash
        run: |
          pip install --pre --upgrade ipex-llm[all] --extra-index-url https://download.pytorch.org/whl/cpu
          test_version_date=`date -d 'yesterday' '+%Y%m%d'`
          if ! pip show ipex-llm | grep $test_version_date; then
            echo "Did not install ipex-llm with excepted version $test_version_date"
            exit 1
          fi

      - name: Test on cpu
        shell: bash
        run: |
          date_for_test_version=$(date -d yesterday +%Y-%m-%d)
          sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py

          mv python/llm/test/benchmark/cpu-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
          cd python/llm/dev/benchmark/all-in-one
          export http_proxy=${HTTP_PROXY}
          export https_proxy=${HTTPS_PROXY}
          source ipex-llm-init -t
          export OMP_NUM_THREADS=48
          # hide time info
          sed -i 's/str(end - st)/"xxxxxx"/g' run.py
          python run.py
          cp ./*.csv /mnt/disk1/models/nightly_perf_cpu
          cd ../../../test/benchmark
          python -m pip install pandas==1.5.3
          python csv_to_html.py -f /mnt/disk1/models/nightly_perf_cpu
          cd /mnt/disk1/models/nightly_perf_cpu
          for f in *.html; do
            curl -T "$f" ${LLM_FTP_URL}/llm/nightly_perf/nightly_perf_cpu/
          done

  llm-performance-test-on-core:
    if: ${{ github.event.schedule || ( github.event_name == 'workflow_dispatch' && inputs.core ) || github.event.inputs.artifact == 'llm-performance-test-on-core' || github.event.inputs.artifact == 'all' }} # please comment it for PR tests
    # needs: llm-cpp-build # please uncomment it for PR tests
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: windows
            platform: dp
            python-version: "3.11"
          # - os: windows
          #   platform: lp
          #   python-version: "3.11"
    runs-on: [self-hosted, "${{ matrix.os }}", llm, perf-core, "${{ matrix.platform }}"]
    env:
      ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
      CSV_SAVE_PATH: ${{ (github.event.schedule || github.event_name == 'workflow_dispatch') && 'D:/action-runners/nightly_perf_core_' || 'D:/action-runners/pr_perf_core_' }}${{ matrix.platform }}/
    steps:
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3

      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        shell: bash
        run: |
          python -m pip install --upgrade pip
          python -m pip install --upgrade wheel
          python -m pip install --upgrade omegaconf pandas
          python -m pip install --upgrade tiktoken einops transformers_stream_generator

      # please uncomment it and comment the "Install IPEX-LLM from Pypi" part for PR tests
      # - name: Download llm binary
      #   uses: ./.github/actions/llm/download-llm-binary

      # - name: Run LLM install (all) test
      #   uses: ./.github/actions/llm/setup-llm-env

      - name: Install IPEX-LLM from Pypi
        shell: bash
        run: |
          pip install --pre --upgrade ipex-llm[all]
          test_version_date=`date -d 'yesterday' '+%Y%m%d'`
          if ! pip show ipex-llm | grep $test_version_date; then
            echo "Did not install ipex-llm with excepted version $test_version_date"
            exit 1
          fi

      - name: Test on core ${{ matrix.platform }}
        shell: bash
        run: |
          date_for_test_version=$(date -d yesterday +%Y-%m-%d)
          sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py

          mv python/llm/test/benchmark/core-perf-test.yaml python/llm/dev/benchmark/all-in-one/config.yaml
          cd python/llm/dev/benchmark/all-in-one
          export http_proxy=${HTTP_PROXY}
          export https_proxy=${HTTPS_PROXY}
          # hide time info
          sed -i 's/str(end - st)/"xxxxxx"/g' run.py
          python run.py
          cp ./*.csv $CSV_SAVE_PATH
          cd ../../../test/benchmark
          python -m pip install pandas==1.5.3
          python csv_to_html.py -f $CSV_SAVE_PATH
          cd ../../dev/benchmark/all-in-one/
          if [ ${{ github.event_name }} == "schedule" ] || [ ${{ github.event_name }} == "workflow_dispatch" ]; then
            curl -T ./*.csv ${LLM_FTP_URL}/llm/nightly_perf/core_${{ matrix.platform }}/
          fi

  llm-performance-test-on-igpu:
    if: ${{ github.event.schedule || ( github.event_name == 'workflow_dispatch' && inputs.igpu ) || github.event.inputs.artifact == 'llm-performance-test-on-igpu' || github.event.inputs.artifact == 'all' }} # please comment it for PR tests
    # needs: llm-cpp-build # please uncomment it for PR tests
    strategy:
      fail-fast: false
      matrix:
        include:
          - os: windows
            python-version: "3.11"
    runs-on: [self-hosted, "${{ matrix.os }}", llm, perf-igpu]
    env:
      ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
    steps:
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3

      # TODO: Put the ipex-llm related install process for win gpu into a action function

      # Please uncomment it and commment the install from pypi for PR tests
      # - name: Download llm binary
      #   uses: ./.github/actions/llm/download-llm-binary

      # - name: Prepare for install ipex-llm from source
      #   shell: bash
      #   run: |
      #     sed -i 's/"bigdl-core-xe-21==" + CORE_XE_VERSION/"bigdl-core-xe-21"/g' python/llm/setup.py
      #     sed -i 's/"bigdl-core-xe-batch-21==" + CORE_XE_VERSION/"bigdl-core-xe-batch-21"/g' python/llm/setup.py
      #     sed -i 's/"bigdl-core-xe-addons-21==" + CORE_XE_VERSION/"bigdl-core-xe-addons-21"/g' python/llm/setup.py
      #     sed -i 's/"bigdl-core-xe-esimd-21==" + CORE_XE_VERSION/"bigdl-core-xe-esimd-21"/g' python/llm/setup.py

      # - name: Install ipex-llm and other related packages (install from source)
      #   shell: cmd
      #   run: |
      #     call conda create -n igpu-perf python=${{ matrix.python-version }} libuv -y
      #     call conda activate igpu-perf

      #     pip install --upgrade pip
      #     pip install --upgrade wheel
      #     pip install --upgrade omegaconf pandas
      #     pip install --upgrade tiktoken einops transformers_stream_generator

      #     cd python\llm
      #     python setup.py clean --all bdist_wheel --win
      #     if not exist dist\ipex_llm*.whl (exit /b 1)
      #     for %%i in (dist\ipex_llm*.whl) do set whl_name=%%i

      #     pip install --pre --upgrade %whl_name%[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
      #     if %ERRORLEVEL% neq 0 (exit /b 1)
      #     pip list

      #     call conda deactivate

      - name: Determine desired ipex-llm version
        shell: bash
        run: |
          test_version_date=`date -d 'yesterday' '+%Y%m%d'`
          echo "TEST_VERSION_DATE=${test_version_date}" >> "$GITHUB_ENV"

      - name: Install ipex-llm and other related packages (install from pypi)
        shell: cmd
        run: |
          call conda create -n igpu-perf python=${{ matrix.python-version }} libuv -y
          call conda activate igpu-perf

          pip install --upgrade pip
          pip install --upgrade wheel
          pip install --upgrade omegaconf pandas
          pip install --upgrade tiktoken einops transformers_stream_generator

          pip install --pre --upgrade ipex-llm[xpu] --extra-index-url https://pytorch-extension.intel.com/release-whl/stable/xpu/cn/
          pip show ipex-llm | findstr %TEST_VERSION_DATE%
          if %ERRORLEVEL% neq 0 (
            echo "Did not install ipex-llm with excepted version %TEST_VERSION_DATE%"
            exit /b 1
          )
          pip list

          call conda deactivate

      - name: Create env for html generation
        shell: cmd
        run: |
          call conda create -n html-gen python=3.11 -y
          call conda activate html-gen

          pip install pandas==1.5.3
          pip install Jinja2
          pip install "numpy<2.0.0"

          call conda deactivate

      - name: Set directory envs & and fix generated csv date name
        shell: bash
        run: |
          if [ ${{ github.event_name }} == "schedule" ] || [ ${{ github.event_name }} == "workflow_dispatch" ]; then
            echo "CSV_SAVE_PATH=${CSV_NIGHTLY_PATH}" >> "$GITHUB_ENV"
          else
            echo "CSV_SAVE_PATH=${CSV_PR_PATH}" >> "$GITHUB_ENV"
          fi
          date_for_test_version=$(date -d yesterday +%Y-%m-%d)
          echo "LOG_FILE=${date_for_test_version}_output.txt" >> "$GITHUB_ENV"

          sed -i "s/date.today()/\"$date_for_test_version\"/g" python/llm/dev/benchmark/all-in-one/run.py

      - name: Add extra warmup for chatglm3-6b int4+fp32 for more stable results
        shell: bash
        run: |
          sed -i '/^\s*result = run_transformer_int4_gpu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, cpu_embedding, batch_size, streaming)/ i\
                  if repo_id in ["THUDM/chatglm3-6b"]:\
                      run_transformer_int4_gpu_win(repo_id, local_model_hub, in_out_pairs, warm_up, num_trials, num_beams, low_bit, cpu_embedding, batch_size, streaming)
          ' python/llm/dev/benchmark/all-in-one/run.py

      - name: Prepare igpu perf test (32-32)
        shell: bash
        run: |
          # hide time info
          # sed -i 's/str(end - st)/"xxxxxx"/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i 's/{api}-results-{today}.csv/32-32-{api}-results-{today}_test1.csv/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32.yaml

      - name: Test on igpu (32-32)
        shell: cmd
        run: |
          call conda activate igpu-perf
          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1
          REM for llava
          set TRANSFORMERS_OFFLINE=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\32-32.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\32-32\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Prepare igpu perf test for transformers 4.37 (32-32)
        shell: bash
        run: |
          sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/32-32_437.yaml

      - name: Test on igpu for transformers 4.37 (32-32)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.37.0

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\32-32_437.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\32-32\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Concat csv and generate html (32-32)
        shell: cmd
        run: |
          call conda activate html-gen

          cd python\llm\dev\benchmark\all-in-one
          python ..\..\..\test\benchmark\concat_csv.py
          if %ERRORLEVEL% neq 0 (exit /b 1)
          del /q *test*.csv
          move *.csv %CSV_SAVE_PATH%\32-32\
          cd ..\..\..\test\benchmark
          python csv_to_html.py -f %CSV_SAVE_PATH%\32-32\
          if %ERRORLEVEL% neq 0 (exit /b 1)
          move %CSV_SAVE_PATH%\32-32\*.html %CSV_SAVE_PATH%

          call conda deactivate

      # TODO: create a action function here for different input
      # 1024-128
      - name: Prepare igpu perf test (1024-128)
        shell: bash
        run: |
          sed -i 's/32-32/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128.yaml

      - name: Test on igpu (1024-128)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.36.2

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1
          REM for llava
          set TRANSFORMERS_OFFLINE=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\1024-128.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Prepare igpu perf test for transformers 4.37 (1024-128)
        shell: bash
        run: |
          sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_437.yaml

      - name: Test on igpu for transformers 4.37 (1024-128)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.37.0

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\1024-128_437.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\1024-128\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Concat csv and generate html (1024-128)
        shell: cmd
        run: |
          call conda activate html-gen

          cd python\llm\dev\benchmark\all-in-one
          python ..\..\..\test\benchmark\concat_csv.py
          if %ERRORLEVEL% neq 0 (exit /b 1)
          del /q *test*.csv
          move *.csv %CSV_SAVE_PATH%\1024-128\
          cd ..\..\..\test\benchmark
          python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128\
          if %ERRORLEVEL% neq 0 (exit /b 1)
          move %CSV_SAVE_PATH%\1024-128\*.html %CSV_SAVE_PATH%

          call conda deactivate

      # 2048-256
      - name: Prepare igpu perf test (2048-256)
        shell: bash
        run: |
          sed -i 's/1024-128/2048-256/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256.yaml

      - name: Test on igpu (2048-256)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.36.2

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1
          REM for llava
          set TRANSFORMERS_OFFLINE=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\2048-256.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\2048-256\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Prepare igpu perf test for transformers 4.37 (2048-256)
        shell: bash
        run: |
          sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/2048-256_437.yaml

      - name: Test on igpu for transformers 4.37 (2048-256)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.37.0

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\2048-256_437.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\2048-256\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Concat csv and generate html (2048-256)
        shell: cmd
        run: |
          call conda activate html-gen

          cd python\llm\dev\benchmark\all-in-one
          python ..\..\..\test\benchmark\concat_csv.py
          if %ERRORLEVEL% neq 0 (exit /b 1)
          del /q *test*.csv
          move *.csv %CSV_SAVE_PATH%\2048-256\
          cd ..\..\..\test\benchmark
          python csv_to_html.py -f %CSV_SAVE_PATH%\2048-256\
          if %ERRORLEVEL% neq 0 (exit /b 1)
          move %CSV_SAVE_PATH%\2048-256\*.html %CSV_SAVE_PATH%

          call conda deactivate

      # load_low_bit 1024-128
      - name: Prepare igpu perf test (load_low_bit 1024-128)
        shell: bash
        run: |
          sed -i 's/2048-256/1024-128/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit.yaml

      - name: Test on igpu (load_low_bit 1024-128)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.36.2

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1
          REM for llava
          set TRANSFORMERS_OFFLINE=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\1024-128_loadlowbit.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\1024-128_loadlowbit\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Prepare igpu perf test for transformers 4.37 (load_low_bit 1024-128)
        shell: bash
        run: |
          sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_loadlowbit_437.yaml

      - name: Test on igpu for transformers 4.37 (load_low_bit 1024-128)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.37.0

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\1024-128_loadlowbit_437.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\1024-128_loadlowbit\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Concat csv and generate html (load_low_bit 1024-128)
        shell: cmd
        run: |
          call conda activate html-gen

          cd python\llm\dev\benchmark\all-in-one
          python ..\..\..\test\benchmark\concat_csv.py
          if %ERRORLEVEL% neq 0 (exit /b 1)
          del /q *test*.csv
          move *.csv %CSV_SAVE_PATH%\1024-128_loadlowbit\
          cd ..\..\..\test\benchmark
          python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128_loadlowbit\
          if %ERRORLEVEL% neq 0 (exit /b 1)
          move %CSV_SAVE_PATH%\1024-128_loadlowbit\*.html %CSV_SAVE_PATH%

          call conda deactivate

      - name: Prepare igpu perf test (int4+fp16 1024-128)
        shell: bash
        run: |
          sed -i 's/{today}_test2/{today}_test1/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16.yaml

      - name: Test on igpu (int4+fp16 1024-128)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.36.2

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1
          REM for llava
          set TRANSFORMERS_OFFLINE=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Prepare igpu perf test for transformers 4.37 (int4+fp16 1024-128)
        shell: bash
        run: |
          sed -i 's/{today}_test1/{today}_test2/g' python/llm/dev/benchmark/all-in-one/run.py
          sed -i "s/path to your local model hub/$MODEL_HUB_DIR/g" python/llm/test/benchmark/igpu-perf/1024-128_int4_fp16_437.yaml

      - name: Test on igpu for transformers 4.37 (int4+fp16 1024-128)
        shell: cmd
        run: |
          call conda activate igpu-perf
          pip install transformers==4.37.0

          set SYCL_CACHE_PERSISTENT=1
          set BIGDL_LLM_XMX_DISABLED=1

          cd python\llm\dev\benchmark\all-in-one
          move ..\..\..\test\benchmark\igpu-perf\1024-128_int4_fp16_437.yaml config.yaml
          set PYTHONIOENCODING=utf-8
          python run.py >> %CSV_SAVE_PATH%\1024-128_int4_fp16\log\%LOG_FILE% 2>&1
          if %ERRORLEVEL% neq 0 (exit /b 1)

          call conda deactivate

      - name: Concat csv and generate html (int4+fp16 1024-128)
        shell: cmd
        run: |
          call conda activate html-gen

          cd python\llm\dev\benchmark\all-in-one
          python ..\..\..\test\benchmark\concat_csv.py
          if %ERRORLEVEL% neq 0 (exit /b 1)
          del /q *test*.csv
          move *.csv %CSV_SAVE_PATH%\1024-128_int4_fp16\
          cd ..\..\..\test\benchmark
          python csv_to_html.py -f %CSV_SAVE_PATH%\1024-128_int4_fp16\
          if %ERRORLEVEL% neq 0 (exit /b 1)
          move %CSV_SAVE_PATH%\1024-128_int4_fp16\*.html %CSV_SAVE_PATH%

          call conda deactivate

      # TODO: avoid duplicated code
      - name: Upload results to ftp
        if: ${{ always() }}
        shell: cmd
        run: |
          cd %CSV_SAVE_PATH%
          IF "${{ github.event_name }}"=="schedule" (
            for %%f in (*.html) do (
                curl -T "%%f" %FTP_IGPU_NIGHTLY_PERF_PATH%
            )
          )
          IF "${{ github.event_name }}"=="workflow_dispatch" (
            for %%f in (*.html) do (
                curl -T "%%f" %FTP_IGPU_NIGHTLY_PERF_PATH%
            )
          )
      # for test on machine when encountering error
      # - name: Remove conda env
      #   if: ${{ always() }}
      #   shell: cmd
      #   run: |
      #     call conda env remove -n igpu-perf -y


LLM Harness Evaluation

name: LLM Harness Evaluation

# Cancel previous runs in the PR when you push new commits
concurrency:
  group: ${{ github.workflow }}-llm-nightly-test-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: true

permissions:
  contents: read

# Controls when the action will run.
on:
  schedule:
    - cron: "30 12 * * *" # GMT time, 12:30 GMT == 20:30 China
  pull_request:
    branches: [main]
    paths:
      - ".github/workflows/llm-harness-evaluation.yml"
  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:
    inputs:
      model_name:
        description: 'Model names, separated by comma and must be quoted.'
        required: true
        type: string
      precision:
        description: 'Precisions, separated by comma and must be quoted.'
        required: true
        type: string
      task:
        description: 'Tasks, separated by comma and must be quoted.'
        required: true
        type: string
      runs-on:
        description: 'Labels to filter the runners, separated by comma and must be quoted.'
        default: "accuracy"
        required: false
        type: string

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  llm-cpp-build:
    uses: ./.github/workflows/llm-binary-build.yml
    with:
      platform: 'Linux'
  # Set the testing matrix based on the event (schedule, PR, or manual dispatch)
  set-matrix:
    runs-on: ubuntu-latest
    outputs:
      model_name: ${{ steps.set-matrix.outputs.model_name }}
      precision: ${{ steps.set-matrix.outputs.precision }}
      task: ${{ steps.set-matrix.outputs.task }}
      runner: ${{ steps.set-matrix.outputs.runner }}
    steps:
      - name: set-nightly-env
        if: ${{github.event_name == 'schedule'}}
        env:
          NIGHTLY_MATRIX_MODEL_NAME: '["Llama2-7b-guanaco-dolphin-500", "falcon-7b-instruct-with-patch",
                        "Mistral-7B-v0.1", "mpt-7b-chat", "Baichuan2-7B-Chat-LLaMAfied"]'
          NIGHTLY_MATRIX_TASK: '["arc", "truthfulqa", "winogrande"]'
          NIGHTLY_MATRIX_PRECISION: '["sym_int4", "fp8"]'
          NIGHTLY_LABELS: '["self-hosted", "llm", "accuracy-nightly"]'
        run: |
            echo "model_name=$NIGHTLY_MATRIX_MODEL_NAME" >> $GITHUB_ENV
            echo "precision=$NIGHTLY_MATRIX_PRECISION" >> $GITHUB_ENV
            echo "task=$NIGHTLY_MATRIX_TASK" >> $GITHUB_ENV
            echo "runner=$NIGHTLY_LABELS" >> $GITHUB_ENV

      - name: set-pr-env
        if: ${{github.event_name == 'pull_request'}}
        env:
          PR_MATRIX_MODEL_NAME: '["stablelm-3b-4e1t"]'
          PR_MATRIX_TASK: '["winogrande"]'
          PR_MATRIX_PRECISION: '["sym_int4"]'
          PR_LABELS: '["self-hosted", "llm", "temp-arc01"]'

        run: |
            echo "model_name=$PR_MATRIX_MODEL_NAME" >> $GITHUB_ENV
            echo "precision=$PR_MATRIX_PRECISION" >> $GITHUB_ENV
            echo "task=$PR_MATRIX_TASK" >> $GITHUB_ENV
            echo "runner=$PR_LABELS" >> $GITHUB_ENV
      - name: set-manual-env
        if: ${{github.event_name == 'workflow_dispatch'}}
        env:
          MANUAL_MATRIX_MODEL_NAME: ${{format('[ {0} ]', inputs.model_name)}}
          MANUAL_MATRIX_TASK: ${{format('[ {0} ]', inputs.task)}}
          MANUAL_MATRIX_PRECISION: ${{format('[ {0} ]', inputs.precision)}}
          MANUAL_LABELS: ${{format('["self-hosted", "llm", {0}]', inputs.runs-on)}}
        run: |
            echo "model_name=$MANUAL_MATRIX_MODEL_NAME" >> $GITHUB_ENV
            echo "precision=$MANUAL_MATRIX_PRECISION" >> $GITHUB_ENV
            echo "task=$MANUAL_MATRIX_TASK" >> $GITHUB_ENV
            echo "runner=$MANUAL_LABELS" >> $GITHUB_ENV
      - name: set-matrix
        id: set-matrix
        run: |
            echo "model_name=$model_name" >> $GITHUB_OUTPUT
            echo "precision=$precision" >> $GITHUB_OUTPUT
            echo "task=$task" >> $GITHUB_OUTPUT
            echo "runner=$runner" >> $GITHUB_OUTPUT
  llm-harness-evaluation:
    timeout-minutes: 1000
    needs: [llm-cpp-build, set-matrix]
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11"]
        model_name: ${{ fromJson(needs.set-matrix.outputs.model_name) }}
        task: ${{ fromJson(needs.set-matrix.outputs.task) }}
        precision: ${{ fromJson(needs.set-matrix.outputs.precision) }}
        device: [xpu]

    runs-on: ${{ fromJson(needs.set-matrix.outputs.runner) }}
    env:
      ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
      ORIGIN_DIR: /mnt/disk1/models
      HARNESS_HF_HOME: /mnt/disk1/harness_home
    steps:
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        shell: bash
        run: |
          set -e
          python -m pip install --upgrade pip
          python -m pip install --upgrade setuptools==58.0.4
          python -m pip install --upgrade wheel

      - name: Download llm binary
        uses: ./.github/actions/llm/download-llm-binary
        with:
          platform: 'Linux'

      - name: Run LLM install (all) test
        uses: ./.github/actions/llm/setup-llm-env
        with:
          extra-dependency: "xpu_2.1"

      - name: Install harness
        working-directory: ${{ github.workspace }}/python/llm/dev/benchmark/harness/
        shell: bash
        run: |
          git clone https://github.com/EleutherAI/lm-evaluation-harness.git
          cd lm-evaluation-harness
          git checkout b281b09
          pip install -e .

      - name: Download models and datasets
        shell: bash
        run: |
          echo "MODEL_PATH=${ORIGIN_DIR}/${{ matrix.model_name }}/" >> "$GITHUB_ENV"
          MODEL_PATH=${ORIGIN_DIR}/${{ matrix.model_name }}/
          if [ ! -d $HARNESS_HF_HOME ]; then
            mkdir -p $HARNESS_HF_HOME
          fi
          wget -r -nH -nc -l inf --no-verbose --cut-dirs=2 ${LLM_FTP_URL}/llm/LeaderBoard_Datasets/ -P $HARNESS_HF_HOME/
          wget -r -nH -nc --no-verbose --cut-dirs=1 ${LLM_FTP_URL}/llm/${{ matrix.model_name }} -P ${ORIGIN_DIR}

      - name: Upgrade packages
        shell: bash
        run: |
          pip install --upgrade datasets==2.14.6

      - name: Run harness
        shell: bash
        working-directory: ${{ github.workspace }}/python/llm/dev/benchmark/harness
        env:
          USE_XETLA: OFF
          # SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS: 1
        run: |
          export HF_HOME=${HARNESS_HF_HOME}
          export HF_DATASETS=$HARNESS_HF_HOME/datasets
          export HF_DATASETS_CACHE=$HARNESS_HF_HOME/datasets
          source /opt/intel/oneapi/setvars.sh

          # set --limit if it's pr-triggered to accelerate pr action
          if ${{github.event_name == 'pull_request'}}; then
            export LIMIT="--limit 6"
          fi

          python run_llb.py \
            --model ipex-llm \
            --pretrained ${MODEL_PATH} \
            --precision ${{ matrix.precision }} \
            --device ${{ matrix.device }} \
            --tasks ${{ matrix.task }} \
            --batch_size 1 --no_cache --output_path results \
            $LIMIT

      - uses: actions/upload-artifact@v3
        with:
          name: harness_results
          path:
            ${{ github.workspace }}/python/llm/dev/benchmark/harness/results/**

      - name: echo single result
        shell: bash

        working-directory: ${{ github.workspace }}/python/llm/dev/benchmark/harness/results/
        run: |
          cat ${{ matrix.model_name }}/${{ matrix.device }}/${{ matrix.precision }}/${{ matrix.task }}/result.json

  llm-harness-summary:
    if: ${{ always() }}
    needs: llm-harness-evaluation
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
      - name: Set up Python 3.11
        uses: actions/setup-python@v4
        with:
          python-version: 3.11
      - name: Install dependencies
        shell: bash
        run: |
          pip install --upgrade pip
          pip install jsonlines  pytablewriter regex
      - name: Download all results
        uses: actions/download-artifact@v3
        with:
          name: harness_results
          path: results
      - name: Summarize the results
        shell: bash
        run: |
          ls results
          python ${{ github.workspace }}/python/llm/dev/benchmark/harness/make_table.py results

  # TODO: change machine to store the results later
  llm-harness-html:
    if: ${{github.event_name == 'schedule' || github.event_name == 'pull_request'}}
    needs: [llm-harness-evaluation]
    runs-on: ["self-hosted", "llm", "accuracy1", "accuracy-nightly"]
    steps:
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
      - name: Set up Python 3.11
        uses: actions/setup-python@v4
        with:
          python-version: 3.11
      - name: Install dependencies
        shell: bash
        run: |
          pip install --upgrade pip
          pip install jsonlines  pytablewriter regex
          pip install pandas==1.5.3

      - name: Set output path
        shell: bash
        run: |
          echo "DATE=$(date +%Y-%m-%d)" >> $GITHUB_ENV
          if ${{github.event_name == 'pull_request'}}; then
            echo 'ACC_FOLDER=/home/arda/action-runners/harness/pr-accuracy-data' >> $GITHUB_ENV
          fi
          if ${{github.event_name == 'schedule'}}; then
            echo 'ACC_FOLDER=/home/arda/action-runners/harness/nightly-accuracy-data' >> $GITHUB_ENV
          fi

      - name: Download harness results
        uses: actions/download-artifact@v3
        with:
          name: harness_results
          path: ${{ env.ACC_FOLDER}}/${{ env.DATE }}

      # Save fp16.csv in the parent folder of env.nightly_folder
      - name: Download FP16 results
        shell: bash
        run: |
          wget https://raw.githubusercontent.com/intel-analytics/ipex-llm/main/python/llm/test/benchmark/harness/fp16.csv -O $ACC_FOLDER/../fp16.csv
          ls $ACC_FOLDER/..

      - name: Write to CSV
        working-directory: ${{ github.workspace }}/python/llm/dev/benchmark/harness
        shell: bash
        run: |
          ls $ACC_FOLDER/$DATE
          python make_csv.py $ACC_FOLDER/$DATE $ACC_FOLDER

      - name: Update HTML
        working-directory: ${{ github.workspace }}/python/llm/test/benchmark/harness
        shell: bash
        run: |
          python harness_csv_to_html.py -f $ACC_FOLDER
          if ${{github.event_name == 'schedule'}}; then
            python update_html_in_parent_folder.py -f $ACC_FOLDER
          fi


LLM Perplexity Evaluation

name: LLM Perplexity Evaluation

# Cancel previous runs in the PR when you push new commits
concurrency:
  group: ${{ github.workflow }}-llm-nightly-test-${{ github.event.pull_request.number || github.run_id }}
  cancel-in-progress: true

permissions:
  contents: read

# Controls when the action will run.
on:
  schedule:
    - cron: "00 12 * * *" # GMT time, 12:00 GMT == 20:00 China
  pull_request:
    branches: [main]
    paths:
      - ".github/workflows/llm-ppl-evaluation.yml"
  # Allows you to run this workflow manually from the Actions tab
  workflow_dispatch:
    inputs:
      seq_len:
        description: 'sequence length, input one number and must be quoted.'
        required: true
        type: string
      model_name:
        description: 'Model names, separated by comma and must be quoted.'
        required: true
        type: string
      precision:
        description: 'Precisions, separated by comma and must be quoted.'
        required: true
        type: string
      runs-on:
        description: 'Labels to filter the runners, separated by comma and must be quoted.'
        default: "accuracy"
        required: false
        type: string

# A workflow run is made up of one or more jobs that can run sequentially or in parallel
jobs:
  llm-cpp-build:
    uses: ./.github/workflows/llm-binary-build.yml
    with:
      platform: 'Linux'
  set-matrix:
    runs-on: ubuntu-latest
    outputs:
      seq_len: ${{ steps.set-matrix.outputs.seq_len }}
      model_name: ${{ steps.set-matrix.outputs.model_name }}
      precision: ${{ steps.set-matrix.outputs.precision }}
      runner: ${{ steps.set-matrix.outputs.runner }}
    steps:
      - name: set-nightly-env
        if: ${{github.event_name == 'schedule'}}
        env:
          NIGHTLY_MATRIX_SEQ_LEN: '["512"]'
          NIGHTLY_MATRIX_MODEL_NAME: '["Llama-2-7b-chat-hf", "mpt-7b-chat",
                        "falcon-7b-instruct-with-patch", "Mistral-7B-v0.1",
                        "chatglm2-6b", "chatglm3-6b", "Baichuan2-7B-Chat"]'
          NIGHTLY_MATRIX_PRECISION: '["sym_int4", "fp8"]'
          NIGHTLY_LABELS: '["self-hosted", "llm", "accuracy-nightly"]'
        run: |
            echo "seq_len=$NIGHTLY_MATRIX_SEQ_LEN" >> $GITHUB_ENV
            echo "model_name=$NIGHTLY_MATRIX_MODEL_NAME" >> $GITHUB_ENV
            echo "precision=$NIGHTLY_MATRIX_PRECISION" >> $GITHUB_ENV
            echo "runner=$NIGHTLY_LABELS" >> $GITHUB_ENV

      - name: set-pr-env
        if: ${{github.event_name == 'pull_request'}}
        env:
          PR_MATRIX_SEQ_LEN: '["512"]'
          PR_MATRIX_MODEL_NAME: '["Llama-2-7b-chat-hf", "chatglm3-6b"]'
          PR_MATRIX_PRECISION: '["sym_int4"]'
          PR_LABELS: '["self-hosted", "llm", "temp-arc01"]'
        run: |
            echo "seq_len=$PR_MATRIX_SEQ_LEN" >> $GITHUB_ENV
            echo "model_name=$PR_MATRIX_MODEL_NAME" >> $GITHUB_ENV
            echo "precision=$PR_MATRIX_PRECISION" >> $GITHUB_ENV
            echo "runner=$PR_LABELS" >> $GITHUB_ENV
      - name: set-manual-env
        if: ${{github.event_name == 'workflow_dispatch'}}
        env:
          MANUAL_MATRIX_SEQ_LEN: ${{format('[ {0} ]', inputs.seq_len)}}
          MANUAL_MATRIX_MODEL_NAME: ${{format('[ {0} ]', inputs.model_name)}}
          MANUAL_MATRIX_PRECISION: ${{format('[ {0} ]', inputs.precision)}}
          MANUAL_LABELS: ${{format('["self-hosted", "llm", {0}]', inputs.runs-on)}}
        run: |
            echo "seq_len=$MANUAL_MATRIX_SEQ_LEN" >> $GITHUB_ENV
            echo "model_name=$MANUAL_MATRIX_MODEL_NAME" >> $GITHUB_ENV
            echo "precision=$MANUAL_MATRIX_PRECISION" >> $GITHUB_ENV
            echo "runner=$MANUAL_LABELS" >> $GITHUB_ENV
      - name: set-matrix
        id: set-matrix
        run: |
            echo "seq_len=$seq_len" >> $GITHUB_OUTPUT
            echo "model_name=$model_name" >> $GITHUB_OUTPUT
            echo "precision=$precision" >> $GITHUB_OUTPUT
            echo "runner=$runner" >> $GITHUB_OUTPUT
  llm-ppl-evaluation:
    timeout-minutes: 1000
    needs: [llm-cpp-build, set-matrix]
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11"]
        model_name: ${{ fromJson(needs.set-matrix.outputs.model_name) }}
        precision: ${{ fromJson(needs.set-matrix.outputs.precision) }}
        seq_len: ${{ fromJson(needs.set-matrix.outputs.seq_len) }}
        device: [xpu]

    runs-on: ${{ fromJson(needs.set-matrix.outputs.runner) }}
    env:
      ANALYTICS_ZOO_ROOT: ${{ github.workspace }}
      ORIGIN_DIR: /mnt/disk1/models
      DATASET_DIR: /mnt/disk1/datasets/THUDM___long_bench/
    steps:
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
      - name: Set up Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}
      - name: Install dependencies
        shell: bash
        run: |
          python -m pip install --upgrade pip
          python -m pip install --upgrade setuptools==58.0.4
          python -m pip install --upgrade wheel

      - name: Download llm binary
        uses: ./.github/actions/llm/download-llm-binary
        with:
          platform: 'Linux'

      - name: Run LLM install (all) test
        uses: ./.github/actions/llm/setup-llm-env
        with:
          extra-dependency: "xpu_2.1"

      - name: Download models
        shell: bash
        run: |
          echo "MODEL_PATH=${ORIGIN_DIR}/${{ matrix.model_name }}/" >> "$GITHUB_ENV"
          MODEL_PATH=${ORIGIN_DIR}/${{ matrix.model_name }}/
          wget -r -nH -nc --no-verbose --cut-dirs=1 ${LLM_FTP_URL}/llm/${{ matrix.model_name }} -P ${ORIGIN_DIR}

      - name: Upgrade packages
        shell: bash
        run: |
          pip install --upgrade datasets==2.14.6

      - name: Run perplexity
        shell: bash
        working-directory: ${{ github.workspace }}/python/llm/dev/benchmark/perplexity
        env:
          USE_XETLA: OFF
          SYCL_PI_LEVEL_ZERO_USE_IMMEDIATE_COMMANDLISTS: 1
        run: |
          source /opt/intel/oneapi/setvars.sh
          if [[ "${{ matrix.model_name }}" == *"chatglm"* || "${{ matrix.model_name }}" == *"Baichuan"* ]]; then
            LANGUAGE="zh"
          else
            LANGUAGE="en"
          fi
          python run.py \
          --seq_len ${{ matrix.seq_len }} \
          --model_path ${MODEL_PATH} \
          --precisions ${{ matrix.precision }} \
          --device ${{ matrix.device }} \
          --dataset_path ${DATASET_DIR} \
          --language ${LANGUAGE} \
          --output_path results

      - uses: actions/upload-artifact@v3
        with:
          name: ppl_results
          path:
            ${{ github.workspace }}/python/llm/dev/benchmark/perplexity/results/**

      - name: echo single result
        shell: bash

        working-directory: ${{ github.workspace }}/python/llm/dev/benchmark/perplexity/results/
        run: |
          if [[ "${{ matrix.model_name }}" == *"chatglm"* || "${{ matrix.model_name }}" == *"Baichuan"* ]]; then
            LANGUAGE="zh"
          else
            LANGUAGE="en"
          fi
          cat ${{ matrix.model_name }}/${{ matrix.device }}/${{ matrix.precision }}/${LANGUAGE}/result.json

  llm-ppl-summary:
    if: ${{ always() }}
    needs: llm-ppl-evaluation
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
      - name: Set up Python 3.11
        uses: actions/setup-python@v4
        with:
          python-version: 3.11
      - name: Install dependencies
        shell: bash
        run: |
          pip install --upgrade pip
          pip install jsonlines  pytablewriter regex
      - name: Download all results
        uses: actions/download-artifact@v3
        with:
          name: ppl_results
          path: results
      - name: Summarize the results
        shell: bash
        run: |
          ls results
          python ${{ github.workspace }}/python/llm/dev/benchmark/perplexity/make_table.py results

  llm-ppl-html:
    if: ${{github.event_name == 'schedule' || github.event_name == 'pull_request'}}
    needs: [llm-ppl-evaluation]
    runs-on: ["self-hosted", "llm", "accuracy1", "accuracy-nightly"]
    steps:
      - uses: actions/checkout@f43a0e5ff2bd294095638e18286ca9a3d1956744 # actions/checkout@v3
      - name: Set up Python 3.11
        uses: actions/setup-python@v4
        with:
          python-version: 3.11
      - name: Install dependencies
        shell: bash
        run: |
          pip install --upgrade pip
          pip install jsonlines  pytablewriter regex
          pip install pandas==1.5.3

      - name: Set output path
        shell: bash
        run: |
          echo "DATE=$(date +%Y-%m-%d)" >> $GITHUB_ENV
          if ${{github.event_name == 'pull_request'}}; then
            echo 'ACC_FOLDER=/home/arda/action-runners/ppl/pr-accuracy-data' >> $GITHUB_ENV
          fi
          if ${{github.event_name == 'schedule'}}; then
            echo 'ACC_FOLDER=/home/arda/action-runners/ppl/nightly-accuracy-data' >> $GITHUB_ENV
          fi

      - name: Download ppl results
        uses: actions/download-artifact@v3
        with:
          name: ppl_results
          path: ${{ env.ACC_FOLDER}}/${{ env.DATE }}

      # Save fp16.csv in the parent folder of env.nightly_folder
      - name: Download fp16.results
        shell: bash
        run: |
          wget https://raw.githubusercontent.com/intel-analytics/ipex-llm/main/python/llm/test/benchmark/perplexity/fp16.csv -O $ACC_FOLDER/../fp16.csv
          ls $ACC_FOLDER/..

      - name: Write to CSV
        working-directory: ${{ github.workspace }}/python/llm/dev/benchmark/perplexity
        shell: bash
        run: |
          ls $ACC_FOLDER/$DATE
          python make_csv.py $ACC_FOLDER/$DATE $ACC_FOLDER

      - name: Update HTML
        working-directory: ${{ github.workspace }}/python/llm/test/benchmark/perplexity
        shell: bash
        run: |
          python ppl_csv_to_html.py -f $ACC_FOLDER
          if ${{github.event_name == 'schedule'}}; then
            python ../harness/update_html_in_parent_folder.py -f $ACC_FOLDER
          fi