on:
  push:
permissions: read-all
jobs:
  get_commit:
    if: github.event_name == 'push' && !contains(github.ref, 'refs/tags/')
    runs-on: ubuntu-20.04
    outputs:
      long: ${{ steps.commit.outputs.long}}
      short: ${{ steps.commit.outputs.short}}
      hash: ${{ steps.commit.outputs.hash}}

    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true

      - name: commit
        uses: pr-mpt/actions-commit-hash@v2
        id: commit
  create_version:
    needs: [get_commit]
    if: github.event_name == 'push' && !contains(github.ref, 'refs/tags/')
    runs-on: ubuntu-20.04
    name: create version doc
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true
      - run: git checkout ${{needs.get_commit.outputs.long}}
      - name: commit
        uses: pr-mpt/actions-commit-hash@v2
        id: commit
      - name: Set up Python 3.11.5
        uses: actions/setup-python@v5
        with:
          python-version: 3.11.5
      - name: install dunamai
        run: pip install dunamai
      - name: print version
        run: dunamai from git --format "{base}+{branch}.{commit}" --pattern "(?P<base>\d+\.\d+(\.((\d+\.\w+)|\w+)|))"
      - name: replace_version
        run: |
          sed -i -e s/'[^ ]*hardcoded__.*None'/__hardcoded__="\"$(dunamai from git --format "{base}+{branch}.{commit}" --pattern "(?P<base>\d+\.\d+(\.((\d+\.\w+)|\w+)|))")\""/ ofscraper/__version__.py
      - name: show file
        run: cat ofscraper/__version__.py
      - name: upload release file
        uses: Wandalen/wretry.action@master
        with:
          attempt_limit: 3
          action: actions/upload-artifact@v4
          with: |
            name: ofscraper_version_${{needs.get_commit.outputs.short}}
            path: /home/runner/work/OF-Scraper/OF-Scraper/ofscraper/__version__.py
            overwrite: true
  linux_file:
    if: github.event_name == 'push' && !contains(github.ref, 'refs/tags/')
    needs: [create_version, get_commit]
    runs-on: ubuntu-20.04
    name: create linux release file
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true
      - run: git checkout ${{needs.get_commit.outputs.long}}
      - name: list files
        run: ls -d $PWD/*
      - name: list files2
        run: ls -d $PWD/ofscraper/*
      - name: replace version.py
        uses: Wandalen/wretry.action@master
        with:
          attempt_limit: 3
          action: actions/download-artifact@v4
          with: |
            name: ofscraper_version_${{needs.get_commit.outputs.short}}
            path: ofscraper
      - name: create one file
        uses: addnab/docker-run-action@v3
        with:
          image: datawhores/ubuntu:latest
          options: -v ${{ github.workspace }}:/root/work
          shell: bash
          run: |
            cd work
            rm -rf ./ofscraper/dist
            rm -rf ./ofscraper/build
            apt-get install zip libxml2-dev libxslt-dev -y
            python3.11 -m venv .venv
            source .venv/bin/activate
            python3.11 -m pip install pyinstaller==6.4.0
            /root/.local/bin/poetry install
            echo "ls files"
            ls .
            echo "test ofscraper"
            ofscraper -h
            echo "install run pyinstaller dir"
            pyinstaller ./ofscraper/__main__.py --name "ofscraper_linux_${{needs.get_commit.outputs.short}}" --onefile
            echo "list files dir"
            ls dist

      - name: upload file
        uses: Wandalen/wretry.action@master
        with:
          attempt_limit: 3
          action: actions/upload-artifact@v4
          with: |
            name: ofscraper_linux_${{needs.get_commit.outputs.short}}
            path: "dist/ofscraper_linux_${{needs.get_commit.outputs.short}}"
            overwrite: true

  linux_dir:
    if: github.event_name == 'push' && !contains(github.ref, 'refs/tags/')
    needs: [create_version, get_commit]
    runs-on: ubuntu-20.04
    name: create linux release zip
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true
      - run: git checkout ${{needs.get_commit.outputs.long}}
      - name: list files
        run: ls -d $PWD/*
      - name: list files2
        run: ls -d $PWD/ofscraper/*
      - name: replace version.py
        uses: Wandalen/wretry.action@master
        with:
          attempt_limit: 3
          action: actions/download-artifact@v4
          with: |
            name: ofscraper_version_${{needs.get_commit.outputs.short}}
            path: ofscraper

      - name: create zip
        uses: addnab/docker-run-action@v3
        with:
          image: datawhores/ubuntu:latest
          options: -v ${{ github.workspace }}:/root/work
          shell: bash
          run: |
            cd work
            rm -rf ./ofscraper/dist
            rm -rf ./ofscraper/build
            apt-get install zip libxml2-dev libxslt-dev -y
            python3.11 -m venv .venv
            source .venv/bin/activate
            python3.11 -m pip install pyinstaller==6.4.0
            /root/.local/bin/poetry install
            echo "ls files"
            ls .
            echo "test ofscraper"
            ofscraper -h
            echo "run pyinstaller dir"
            pyinstaller ./ofscraper/__main__.py --name "ofscraper_linux_${{needs.get_commit.outputs.short}}"
            echo "list one dir"
            ls dist
            zip -r "dist/ofscraper_linux_${{needs.get_commit.outputs.short}}.zip" "dist/ofscraper_linux_${{needs.get_commit.outputs.short}}"
      - name: upload one file
        uses: Wandalen/wretry.action@master
        with:
          attempt_limit: 3
          action: actions/upload-artifact@v4
          with: |
            name: "ofscraper_linux_zip_${{needs.get_commit.outputs.short}}"
            path: "dist/ofscraper_linux_${{needs.get_commit.outputs.short}}.zip"
            overwrite: true

  windows_dir:
    if: github.event_name == 'push' && !contains(github.ref, 'refs/tags/')
    needs: [create_version, get_commit]
    runs-on: windows-latest
    name: create windows release zip
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true
      - run: git checkout ${{needs.get_commit.outputs.long}}
      - name: list files
        run: ls .
      - name: list files2
        run: ls ofscraper
      - name: commit
        uses: pr-mpt/actions-commit-hash@v2
        id: commit
      - name: replace version.py
        uses: Wandalen/wretry.action@master
        with:
          attempt_limit: 3
          action: actions/download-artifact@v4
          with: |
            name: ofscraper_version_${{needs.get_commit.outputs.short}}
            path: ofscraper
      - name: Set up Python 3.11.5
        uses: actions/setup-python@v5
        with:
          python-version: 3.11.5
      - name: install poetry
        uses: abatilo/actions-poetry@v2
        with:
          poetry-version: 1.4.1
      - name: create virtualenv
        run: python3 -m venv .venv
      - name: install ofscraper windows
        run: |
          .venv\Scripts\activate.bat
          pip install .
          .venv\Scripts\deactivate.bat
      - name: Install pyinstaller
        run: |
          .venv\Scripts\activate.bat
          pip install pyinstaller==6.4.0
          .venv\Scripts\deactivate.bat
      - name: test ofscraper
        run: |
          .venv\Scripts\activate.bat
          ofscraper -h
      - name: create-dir
        id: create-dir
        run: |
          .venv\Scripts\activate.bat
          pyinstaller ofscraper\__main__.py --collect-submodules ofscraper --name "ofscraper_windows_${{needs.get_commit.outputs.short}}" --paths . --clean --log DEBUG
        continue-on-error: true
      - name: create-dir2
        if: steps.create-dir.outcome != 'success'
        run: |
          .venv\Scripts\activate.bat
          pyinstaller .venv/Scripts/ofscraper --collect-submodules ofscrape --name "ofscraper_windows_${{needs.get_commit.outputs.short}}" --paths ofscraper --clean --log DEBUG
      - uses: vimtor/action-zip@v1
        with:
          files: "dist/ofscraper_windows_${{needs.get_commit.outputs.short}}"
          dest: "dist/ofscraper_windows_${{needs.get_commit.outputs.short}}.zip"
      - name: upload one file
        uses: Wandalen/wretry.action@master

        with:
          attempt_limit: 3
          action: actions/upload-artifact@v4
          with: |
            name: ofscraper_windows_zip_${{needs.get_commit.outputs.short}}
            path: "dist/ofscraper_windows_${{needs.get_commit.outputs.short}}.zip"
            overwrite: true

  windows_file:
    if: github.event_name == 'push' && !contains(github.ref, 'refs/tags/')
    needs: [create_version, get_commit]
    runs-on: windows-latest
    name: create windows release file
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true
      - run: git checkout ${{needs.get_commit.outputs.long}}
      - name: list files
        run: ls .
      - name: list files2
        run: ls ofscraper

      - name: replace version.py
        uses: Wandalen/wretry.action@master
        with:
          attempt_limit: 3
          action: actions/download-artifact@v4
          with: |
            name: ofscraper_version_${{needs.get_commit.outputs.short}}
            path: ofscraper
      - name: Set up Python 3.11.5
        uses: actions/setup-python@v5
        with:
          python-version: 3.11.5
      - name: install poetry
        uses: abatilo/actions-poetry@v2
        with:
          poetry-version: 1.4.1
      - name: create virtualenv
        run: python3 -m venv .venv
      - name: install ofscraper windows
        run: |
          .venv\Scripts\activate.bat
          pip install .
          .venv\Scripts\deactivate.bat
      - name: Install pyinstaller
        run: |
          .venv\Scripts\activate.bat
           pip install pyinstaller==6.4.0
          .venv\Scripts\deactivate.bat
      - name: test ofscraper
        run: ofscraper -h
      - name: create-one-file
        id: create-one-file
        run: |
          .venv\Scripts\activate.bat
          pyinstaller ofscraper\__main__.py --collect-submodules ofscraper --onefile --name "ofscraper_windows_${{needs.get_commit.outputs.short}}" -p ofscraper --clean --log DEBUG
          .venv\Scripts\deactivate.bat
        continue-on-error: true
      - name: create-one-file2
        if: steps.create-one-file.outcome != 'success'
        run: |
          .venv\Scripts\activate.bat
          pyinstaller .venv/Scripts/ofscraper --onefile --collect-submodules ofscraper --name "ofscraper_windows_${{needs.get_commit.outputs.short}}" -p ofscraper --clean --log DEBUG
          .venv\Scripts\deactivate.bat
      - name: upload one file
        uses: Wandalen/wretry.action@master
        with:
          attempt_limit: 3
          action: actions/upload-artifact@v4
          with: |
            path: "dist/ofscraper_windows_${{needs.get_commit.outputs.short}}.exe"
            name: ofscraper_windows_${{needs.get_commit.outputs.short}}
            overwrite: true

  mac_file:
    if: github.event_name == 'push' && !contains(github.ref, 'refs/tags/')
    needs: [create_version, get_commit]
    runs-on: macos-latest
    name: create mac release file
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true
      - run: git checkout ${{needs.get_commit.outputs.long}}
      - name: list files
        run: ls -d $PWD/*
      - name: list files2
        run: ls -d $PWD/ofscraper/*

      - name: replace version.py
        uses: Wandalen/wretry.action@master

        with:
          action: actions/download-artifact@v4
          attempt_limit: 3
          with: |
            name: ofscraper_version_${{needs.get_commit.outputs.short}}
            path: ofscraper
      - name: Set up Python 3.11.5
        uses: actions/setup-python@v5
        with:
          python-version: 3.11.5
      - name: create virtualenv
        run: python3 -m venv .venv
      - name: install poetry
        uses: abatilo/actions-poetry@v2
        with:
          poetry-version: 1.4.1
      - name: Install package
        run: |
          source .venv/bin/activate
          pip install . -vvv
          deactivate
      - name: Install pyinstaller
        run: |
          source .venv/bin/activate
          pip install pyinstaller==6.4.0
          deactivate
      - name: test ofscraper
        run: |
          export PATH=$PATH:/usr/local/bin
          /Users/runner/work/OF-Scraper/OF-Scraper/.venv/bin/ofscraper -h
      - name: create-one-file
        id: create-one-file
        run: |
          rm -rf dist
          rm -rf build
          source .venv/bin/activate
          export PATH=$PATH:/usr/local/bin
          pyinstaller /Users/runner/work/OF-Scraper/OF-Scraper/ofscraper/__main__.py --collect-submodules /Users/runner/work/OF-Scraper/OF-Scraper/.venv/bin/ofscraper --onefile --name "ofscraper_macos_${{needs.get_commit.outputs.short}}"
          deactivate
        continue-on-error: true
      - name: create-one-file2
        id: create-one-file2
        if: steps.create-one-file.outcome != 'success'
        run: |
          rm -rf dist
          rm -rf build
          source .venv/bin/activate
          poetry run pyinstaller .venv/bin/ofscraper --onefile --name "ofscraper_macos_${{needs.get_commit.outputs.short}}"
          deactivate

      - name: upload file
        uses: Wandalen/wretry.action@master

        with:
          action: actions/upload-artifact@v4
          attempt_limit: 3
          with: |
            name: ofscraper_macos_${{needs.get_commit.outputs.short}}
            path: "dist/ofscraper_macos_${{needs.get_commit.outputs.short}}"
            overwrite: true

  mac_dir:
    if: github.event_name == 'push' && !contains(github.ref, 'refs/tags/')
    needs: [create_version, get_commit]
    runs-on: macos-latest
    name: create mac release zip
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true
      - run: git checkout ${{needs.get_commit.outputs.long}}
      - name: list files
        run: ls -d $PWD/*
      - name: list files2
        run: ls -d $PWD/ofscraper/*
      - name: replace version.py
        uses: Wandalen/wretry.action@master

        with:
          action: actions/download-artifact@v4
          attempt_limit: 3
          with: |
            name: ofscraper_version_${{needs.get_commit.outputs.short}}
            path: ofscraper
      - name: Set up Python 3.11.5
        uses: actions/setup-python@v5
        with:
          python-version: 3.11.5
      - name: create virtualenv
        run: python3 -m venv .venv
      - name: install poetry
        uses: abatilo/actions-poetry@v2
        with:
          poetry-version: 1.4.1
      - name: Install package
        run: |
          source .venv/bin/activate
          pip install . -vvv
          deactivate
      - name: Install pyinstaller
        run: |
          source .venv/bin/activate
          pip install pyinstaller==6.4.0
          deactivate
      - name: test ofscraper
        run: |
          export PATH=$PATH:/usr/local/bin
          /Users/runner/work/OF-Scraper/OF-Scraper/.venv/bin/ofscraper -h
      - name: create-dir
        id: create-dir
        run: |
          rm -rf dist
          rm -rf build
          source .venv/bin/activate
          pyinstaller /Users/runner/work/OF-Scraper/OF-Scraper/ofscraper/__main__.py --name "ofscraper_macos_${{needs.get_commit.outputs.short}}"
          deactivate
        continue-on-error: true
      - name: create-dir2
        id: create-dir2
        if: steps.create-dir.outcome != 'success'
        run: |
          rm -rf dist
          rm -rf build
          source .venv/bin/activate
          poetry run pyinstaller .venv/bin/ofscraper --name "ofscraper_macos_${{needs.get_commit.outputs.short}}"
          deactivate
      - uses: vimtor/action-zip@v1
        with:
          files: "dist/ofscraper_macos_${{needs.get_commit.outputs.short}}"
          dest: "dist/ofscraper_macos_${{needs.get_commit.outputs.short}}.zip"

      - name: upload zip
        uses: Wandalen/wretry.action@master

        with:
          action: actions/upload-artifact@v4
          with: |
            name: ofscraper_macos_zip_${{needs.get_commit.outputs.short}}
            path: "dist/ofscraper_macos_${{needs.get_commit.outputs.short}}.zip"
            overwrite: true

  publish_release:
    if: github.event_name == 'push' && !contains(github.ref, 'refs/tags/')
    runs-on: ubuntu-latest
    name: publish
    needs:
      [
        mac_file,
        mac_dir,
        windows_dir,
        windows_file,
        linux_file,
        linux_dir,
        get_commit,
      ]
    permissions:
      contents: write
    steps:
      - name: download linux
        uses: Wandalen/wretry.action@master

        with:
          attempt_limit: 3
          action: actions/download-artifact@v4
          with: |
            name:  ofscraper_linux_${{needs.get_commit.outputs.short}}
      - name: download linux zip
        uses: Wandalen/wretry.action@master

        with:
          attempt_limit: 3
          action: actions/download-artifact@v4
          with: |
            name:  ofscraper_linux_zip_${{needs.get_commit.outputs.short}}
      - name: download mac
        uses: Wandalen/wretry.action@master

        with:
          attempt_limit: 3
          action: actions/download-artifact@v4
          with: |
            name:  ofscraper_macos_${{needs.get_commit.outputs.short}}
      - name: download mac zip
        uses: Wandalen/wretry.action@master

        with:
          attempt_limit: 3
          action: actions/download-artifact@v4
          with: |
            name:  ofscraper_macos_zip_${{needs.get_commit.outputs.short}}
      - name: download windows
        uses: Wandalen/wretry.action@master

        with:
          attempt_limit: 3
          action: actions/download-artifact@v4
          with: |
            name:  ofscraper_windows_${{needs.get_commit.outputs.short}}
      - name: download windows zip
        uses: Wandalen/wretry.action@master

        with:
          attempt_limit: 3
          action: actions/download-artifact@v4
          with: |
            name:  ofscraper_windows_zip_${{needs.get_commit.outputs.short}}

      - name: Get current date
        id: date
        run: echo "date=$(date +'%Y.%m.%d')_$(date +%s| tail -c 7)" >> $GITHUB_OUTPUT

      - name: create release
        uses: ncipollo/release-action@v1
        with:
          artifacts: /home/runner/work/OF-Scraper/OF-Scraper/ofscraper_*
          tag: "${{needs.get_commit.outputs.short}}"
          name: ${{env.name}}
          body: ${{env.body}}
          commit: "${{needs.get_commit.outputs.long}}"
        env:
          name: "${{steps.date.outputs.date}}_${{ github.head_ref || github.ref_name }}_${{needs.get_commit.outputs.short}}_package"
          body: "This is a auto generated release created from commit ${{ needs.get_commit.outputs.hash}}"


package builder release

name: package builder release
permissions: read-all
on:
  workflow_dispatch:
    inputs:
      version:
        description: version
        required: true
      files:
        required: true
        type: boolean
      pip:
        required: true
        type: boolean
      docker:
        required: true
        type: boolean
      docker_old:
        required: true
        type: boolean
jobs:
  get_date:
    runs-on: ubuntu-20.04
    if: ${{ inputs.files}}
    name: get_date
    steps:
      - name: Get current date
        id: date
        run: echo "date=$(date +'%Y.%m.%d')_$(date +%s| tail -c 7)" >> $GITHUB_OUTPUT
    outputs:
      date: ${{ steps.date.outputs.date}}

  get_commit:
    runs-on: ubuntu-20.04
    outputs:
      long: ${{ steps.commit.outputs.long}}
      short: ${{ steps.commit.outputs.short}}
      hash: ${{ steps.commit.outputs.hash}}

    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true
          ref: ${{inputs.version}}
      - name: commit
        uses: pr-mpt/actions-commit-hash@v2
        id: commit
  create_version:
    needs: [get_date, get_commit]
    runs-on: ubuntu-20.04
    if: ${{ inputs.files}}
    name: create version doc
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true
          ref: ${{inputs.version}}
      - run: git checkout ${{needs.get_commit.outputs.long}}
      - name: update line
        run: sed -i -e s/'>=3.7.0,<4'/'>=3.7.0,<3.13'/ pyproject.toml
      - name: Set up Python 3.11.5
        uses: actions/setup-python@v4
        with:
          python-version: 3.11.5
      - name: install dunamai
        run: pip install dunamai
      - name: replace_version
        run: |
          sed -i -e s/'[^ ]*hardcoded.*None'/__hardcoded__="\"$(dunamai from git --format "{base}" --pattern "(?P<base>\d+\.\d+(\.((\d+\.\w+)|\w+)|))")\""/ ofscraper/__version__.py
      - name: show file
        run: cat ofscraper/__version__.py
      - name: upload release file
        uses: Wandalen/wretry.action@master
        with:
          attempt_limit: 3
          action: actions/upload-artifact@v4
          with: |
            name: ofscraper_version_${{needs.get_date.outputs.date}}
            path: /home/runner/work/OF-Scraper/OF-Scraper/ofscraper/__version__.py

  linux_file:
    needs: [create_version, get_date, get_commit]
    runs-on: ubuntu-20.04
    name: create linux release file
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true
          ref: ${{inputs.version}}
      - run: git checkout ${{needs.get_commit.outputs.long}}
      - name: list files
        run: ls -d $PWD/*
      - name: list files2
        run: ls -d $PWD/ofscraper/*
      - name: replace version.py
        uses: Wandalen/wretry.action@master
        with:
          attempt_limit: 3
          action: actions/download-artifact@v4
          with: |
            name: ofscraper_version_${{needs.get_date.outputs.date}}
            path: ofscraper
      - name: create one file
        uses: addnab/docker-run-action@v3
        with:
          image: datawhores/ubuntu:latest
          options: -v ${{ github.workspace }}:/root/work
          shell: bash
          run: |
            cd work
            rm -rf ./ofscraper/dist
            rm -rf ./ofscraper/build
            apt-get install zip libxml2-dev libxslt-dev -y
            python3.11 -m venv .venv
            source .venv/bin/activate
            python3.11 -m pip install pyinstaller==6.4.0
            /root/.local/bin/poetry install
            echo "ls files"
            ls .
            echo "test ofscraper"
            ofscraper -h
            echo "install run pyinstaller dir"
            pyinstaller ./ofscraper/__main__.py --name "ofscraper_linux_${{inputs.version}}" --onefile

            echo "list files dir"
            ls dist

      - name: upload one file
        uses: Wandalen/wretry.action@master
        with:
          attempt_limit: 3
          action: actions/upload-artifact@v4
          with: |
            name: ofscraper_linux_${{needs.get_date.outputs.date}}
            path: "dist/ofscraper_linux_${{inputs.version}}"
            overwrite: true

  linux_dir:
    needs: [create_version, get_date, get_commit]
    runs-on: ubuntu-20.04
    name: create linux release zip
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true
          ref: ${{inputs.version}}
      - run: git checkout ${{needs.get_commit.outputs.long}}
      - name: list files
        run: ls -d $PWD/*
      - name: list files2
        run: ls -d $PWD/ofscraper/*
      - name: replace version.py
        uses: Wandalen/wretry.action@master
        with:
          attempt_limit: 3
          action: actions/download-artifact@v4
          with: |
            name: ofscraper_version_${{needs.get_date.outputs.date}}
            path: ofscraper

      - name: create zip
        uses: addnab/docker-run-action@v3
        with:
          image: datawhores/ubuntu:latest
          options: -v ${{ github.workspace }}:/root/work
          shell: bash
          run: |
            cd work
            rm -rf ./ofscraper/dist
            rm -rf ./ofscraper/build
            apt-get install zip libxml2-dev libxslt-dev -y
            python3.11 -m venv .venv
            source .venv/bin/activate
            python3.11 -m pip install pyinstaller==6.4.0
            /root/.local/bin/poetry install
            echo "ls files"
            ls .
            echo "test ofscraper"
            ofscraper -h
            echo "run pyinstaller dir"
            pyinstaller ./ofscraper/__main__.py --name "ofscraper_linux_${{inputs.version}}"
            echo "list one dir"
            ls dist
            zip -r "dist/ofscraper_linux_${{inputs.version}}.zip" "dist/ofscraper_linux_${{inputs.version}}"
      - name: upload one file
        uses: Wandalen/wretry.action@master
        with:
          attempt_limit: 3
          action: actions/upload-artifact@v4
          with: |
            name: "ofscraper_linux_zip_${{needs.get_date.outputs.date}}"
            path: "dist/ofscraper_linux_${{inputs.version}}.zip"
            overwrite: true

  windows_dir:
    needs: [create_version, get_date, get_commit]
    runs-on: windows-latest
    name: create windows release zip
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true
          ref: ${{inputs.version}}
      - run: git checkout ${{needs.get_commit.outputs.long}}
      - name: list files
        run: ls .
      - name: list files2
        run: ls ofscraper
      - name: commit
        uses: pr-mpt/actions-commit-hash@v2
        id: commit
      - name: replace version.py
        uses: Wandalen/wretry.action@master
        with:
          attempt_limit: 3
          action: actions/download-artifact@v4
          with: |
            name: ofscraper_version_${{needs.get_date.outputs.date}}
            path: ofscraper
      - name: Set up Python 3.11.5
        uses: actions/setup-python@v5
        with:
          python-version: 3.11.5
      - name: install poetry
        uses: abatilo/actions-poetry@v2
        with:
          poetry-version: 1.4.1
      - name: create virtualenv
        run: python3 -m venv .venv
      - name: install ofscraper windows
        run: |
          .venv\Scripts\activate.bat
          pip install .
          .venv\Scripts\deactivate.bat
      - name: Install pyinstaller
        run: |
          .venv\Scripts\activate.bat
          pip install pyinstaller==6.4.0
          .venv\Scripts\deactivate.bat
      - name: test ofscraper
        run: |
          .venv\Scripts\activate.bat
          ofscraper -h
      - name: create-dir
        id: create-dir
        run: |
          .venv\Scripts\activate.bat
          pyinstaller ofscraper\__main__.py --collect-submodules ofscraper --name "ofscraper_windows_${{inputs.version}}" --paths . --clean --log DEBUG
        continue-on-error: true
      - name: create-dir2
        if: steps.create-dir.outcome != 'success'
        run: |
          .venv\Scripts\activate.bat
          pyinstaller .venv/Scripts/ofscraper --collect-submodules ofscrape --name "ofscraper_windows_${{inputs.version}}" --paths ofscraper --clean --log DEBUG
      - uses: vimtor/action-zip@v1
        with:
          files: "dist/ofscraper_windows_${{inputs.version}}"
          dest: "dist/ofscraper_windows_${{inputs.version}}.zip"
      - name: upload one file
        uses: Wandalen/wretry.action@master

        with:
          attempt_limit: 3
          action: actions/upload-artifact@v4
          with: |
            name: ofscraper_windows_zip_${{needs.get_date.outputs.date}}
            path: "dist/ofscraper_windows_${{inputs.version}}.zip"
            overwrite: true

  windows_file:
    needs: [create_version, get_date, get_commit]
    runs-on: windows-latest
    name: create windows release file
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true
          ref: ${{inputs.version}}
      - run: git checkout ${{needs.get_commit.outputs.long}}
      - name: list files
        run: ls .
      - name: list files2
        run: ls ofscraper

      - name: replace version.py
        uses: Wandalen/wretry.action@master
        with:
          attempt_limit: 3
          action: actions/download-artifact@v4
          with: |
            name: ofscraper_version_${{needs.get_date.outputs.date}}
            path: ofscraper
      - name: Set up Python 3.11.5
        uses: actions/setup-python@v5
        with:
          python-version: 3.11.5
      - name: install poetry
        uses: abatilo/actions-poetry@v2
        with:
          poetry-version: 1.4.1
      - name: create virtualenv
        run: python3 -m venv .venv
      - name: install ofscraper windows
        run: |
          .venv\Scripts\activate.bat
          pip install .
          .venv\Scripts\deactivate.bat
      - name: Install pyinstaller
        run: |
          .venv\Scripts\activate.bat
            pip install pyinstaller==6.4.0
          .venv\Scripts\deactivate.bat
      - name: test ofscraper
        run: ofscraper -h
      - name: create-one-file
        id: create-one-file
        run: |
          .venv\Scripts\activate.bat
          pyinstaller ofscraper\__main__.py --collect-submodules ofscraper --onefile --name "ofscraper_windows_${{inputs.version}}" -p ofscraper --clean --log DEBUG
          .venv\Scripts\deactivate.bat
        continue-on-error: true
      - name: create-one-file2
        if: steps.create-one-file.outcome != 'success'
        run: |
          .venv\Scripts\activate.bat
          pyinstaller .venv/Scripts/ofscraper --onefile --collect-submodules ofscraper --name "ofscraper_windows_${{inputs.version}}" -p ofscraper --clean --log DEBUG
          .venv\Scripts\deactivate.bat
      - name: upload one file
        uses: Wandalen/wretry.action@master
        with:
          attempt_limit: 3
          action: actions/upload-artifact@v4
          with: |
            path: "dist/ofscraper_windows_${{inputs.version}}.exe"
            name: ofscraper_windows_${{needs.get_date.outputs.date}}
            overwrite: true

  mac_file:
    needs: [create_version, get_date, get_commit]
    runs-on: macos-latest
    name: create mac release file
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true
          ref: ${{inputs.version}}
      - run: git checkout ${{needs.get_commit.outputs.long}}
      - name: list files
        run: ls -d $PWD/*
      - name: list files2
        run: ls -d $PWD/ofscraper/*

      - name: replace version.py
        uses: Wandalen/wretry.action@master

        with:
          action: actions/download-artifact@v4
          attempt_limit: 3
          with: |
            name: ofscraper_version_${{needs.get_date.outputs.date}}
            path: ofscraper
      - name: Set up Python 3.11.5
        uses: actions/setup-python@v5
        with:
          python-version: 3.11.5
      - name: create virtualenv
        run: python3 -m venv .venv
      - name: install poetry
        uses: abatilo/actions-poetry@v2
        with:
          poetry-version: 1.4.1
      - name: Install package
        run: |
          source .venv/bin/activate
          pip install . -vvv
          deactivate
      - name: Install pyinstaller
        run: |
          source .venv/bin/activate
          pip install pyinstaller==6.4.0
          deactivate
      - name: test ofscraper
        run: |
          export PATH=$PATH:/usr/local/bin
          /Users/runner/work/OF-Scraper/OF-Scraper/.venv/bin/ofscraper -h
      - name: create-one-file
        id: create-one-file
        run: |
          rm -rf dist
          rm -rf build
          source .venv/bin/activate
          export PATH=$PATH:/usr/local/bin
          pyinstaller /Users/runner/work/OF-Scraper/OF-Scraper/ofscraper/__main__.py --collect-submodules /Users/runner/work/OF-Scraper/OF-Scraper/.venv/bin/ofscraper --onefile --name "ofscraper_macos_${{inputs.version}}"
          deactivate
        continue-on-error: true
      - name: create-one-file2
        id: create-one-file2
        if: steps.create-one-file.outcome != 'success'
        run: |
          rm -rf dist
          rm -rf build
          source .venv/bin/activate
          poetry run pyinstaller .venv/bin/ofscraper --onefile --name "ofscraper_macos_${{inputs.version}}"
          deactivate

      - name: upload one file
        uses: Wandalen/wretry.action@master

        with:
          action: actions/upload-artifact@v4
          attempt_limit: 3
          with: |
            name: ofscraper_macos_${{needs.get_date.outputs.date}}
            path: "dist/ofscraper_macos_${{inputs.version}}"
            overwrite: true

  mac_dir:
    needs: [create_version, get_date, get_commit]
    runs-on: macos-latest
    name: create mac release zip
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true
          ref: ${{inputs.version}}
      - run: git checkout ${{needs.get_commit.outputs.long}}
      - name: list files
        run: ls -d $PWD/*
      - name: list files2
        run: ls -d $PWD/ofscraper/*

      - name: replace version.py
        uses: Wandalen/wretry.action@master

        with:
          action: actions/download-artifact@v4
          attempt_limit: 3
          with: |
            name: ofscraper_version_${{needs.get_date.outputs.date}}
            path: ofscraper
      - name: Set up Python 3.11.5
        uses: actions/setup-python@v5
        with:
          python-version: 3.11.5
      - name: create virtualenv
        run: python3 -m venv .venv
      - name: install poetry
        uses: abatilo/actions-poetry@v2
        with:
          poetry-version: 1.4.1
      - name: Install package
        run: |
          source .venv/bin/activate
          pip install . -vvv
          deactivate
      - name: Install pyinstaller
        run: |
          source .venv/bin/activate
          pip install pyinstaller==6.4.0
          deactivate
      - name: test ofscraper
        run: |
          export PATH=$PATH:/usr/local/bin
          /Users/runner/work/OF-Scraper/OF-Scraper/.venv/bin/ofscraper -h
      - name: create-dir
        id: create-dir
        run: |
          rm -rf dist
          rm -rf build
          source .venv/bin/activate
          pyinstaller /Users/runner/work/OF-Scraper/OF-Scraper/ofscraper/__main__.py --name "ofscraper_macos_${{inputs.version}}"
          deactivate
        continue-on-error: true
      - name: create-dir2
        id: create-dir2
        if: steps.create-dir.outcome != 'success'
        run: |
          rm -rf dist
          rm -rf build
          source .venv/bin/activate
          poetry run pyinstaller .venv/bin/ofscraper --name "ofscraper_macos_${{inputs.version}}"
          deactivate
      - uses: vimtor/action-zip@v1
        with:
          files: "dist/ofscraper_macos_${{inputs.version}}"
          dest: "dist/ofscraper_macos_${{inputs.version}}.zip"

      - name: upload one file
        uses: Wandalen/wretry.action@master

        with:
          action: actions/upload-artifact@v4
          with: |
            name: ofscraper_macos_zip_${{needs.get_date.outputs.date}}
            path: "dist/ofscraper_macos_${{inputs.version}}.zip"
            overwrite: true

  publish_release:
    runs-on: ubuntu-latest
    name: publish
    needs:
      [
        get_commit,
        get_date,
        linux_dir,
        linux_file,
        windows_dir,
        windows_file,
        mac_file,
        mac_dir,
      ]
    permissions:
      contents: write
    steps:
      - name: download linux
        uses: Wandalen/wretry.action@master

        with:
          attempt_limit: 3
          action: actions/download-artifact@v4
          with: |
            name:  ofscraper_linux_${{needs.get_date.outputs.date}}
      - name: download linux zip
        uses: Wandalen/wretry.action@master

        with:
          attempt_limit: 3
          action: actions/download-artifact@v4
          with: |
            name:  ofscraper_linux_zip_${{needs.get_date.outputs.date}}
      - name: download mac
        uses: Wandalen/wretry.action@master

        with:
          attempt_limit: 3
          action: actions/download-artifact@v4
          with: |
            name:  ofscraper_macos_${{needs.get_date.outputs.date}}
      - name: download mac zip
        uses: Wandalen/wretry.action@master

        with:
          attempt_limit: 3
          action: actions/download-artifact@v4
          with: |
            name:  ofscraper_macos_zip_${{needs.get_date.outputs.date}}
      - name: download windows
        uses: Wandalen/wretry.action@master

        with:
          attempt_limit: 3
          action: actions/download-artifact@v4
          with: |
            name:  ofscraper_windows_${{needs.get_date.outputs.date}}
      - name: download windows zip
        uses: Wandalen/wretry.action@master

        with:
          attempt_limit: 3
          action: actions/download-artifact@v4
          with: |
            name:  ofscraper_windows_zip_${{needs.get_date.outputs.date}}

      - name: show files
        run: ls /home/runner/work/OF-Scraper/OF-Scraper

      - name: upload package
        uses: ncipollo/release-action@v1
        with:
          artifacts: /home/runner/work/OF-Scraper/OF-Scraper/ofscraper_*
          tag: "${{inputs.version}}"
          allowUpdates: True
          omitBodyDuringUpdate: True
          omitDraftDuringUpdate: True
          commit: "${{needs.get_commit.outputs.long}}"

  pypi:
    needs: [get_commit]
    if: ${{ inputs.pip}}
    runs-on: ubuntu-latest
    name: create pypi
    steps:
      - name: Checkout
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          fetch-tags: true
          ref: ${{inputs.version}}
      - run: git checkout ${{needs.get_commit.outputs.long}}
      - name: Set up Python 3.11.5
        uses: actions/setup-python@v4
        with:
          python-version: 3.11.5
      - name: 🐍 Install dependencies
        run: |
          pip install poetry-dynamic-versioning
          pip install dunamai
      - name: Install Poetry
        uses: snok/install-poetry@v1
      - name: Build
        run: |
          poetry version $(dunamai from git --format "{base}" --pattern="(?P<base>\d+\.\d+(\.((\d+\.\w+)|\w+)|))")
          poetry build
      - name: Push to pypi
        env:
          PYPI_TOKEN: ${{ secrets.pypi }}
        run: |
          poetry config pypi-token.pypi $PYPI_TOKEN
          poetry publish
  docker:
    needs: [get_commit]
    if: ${{ inputs.docker}}
    runs-on: ubuntu-latest
    name: DockerHub Release
    permissions:
      contents: read
      packages: write
      # This is used to complete the identity challenge
      # with sigstore/fulcio when running outside of PRs.
      id-token: write

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{inputs.version}}
      - run: git checkout ${{needs.get_commit.outputs.long}}
      - name: get tag
        run: echo release=$(git describe --tags `git rev-list --tags --max-count=1`) >> $GITHUB_ENV
      - name: Setup Docker buildx
        uses: docker/setup-buildx-action@v2
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2
      #Install the cosign tool except on PR
      #https://github.com/sigstore/cosign-installer
      - name: Install cosign
        uses: sigstore/cosign-installer@main #v2.6.0
        with:
          cosign-release: v1.13.1
      #Login against a Docker registry except on PR
      #https://github.com/docker/login-action

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Login to Docker Hub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: push image to Docker Hub
        id: build-and-push
        uses: docker/build-push-action@v4
        with:
          context: .
          tags: |
            datawhores/of-scraper:${{ inputs.version }}
            datawhores/of-scraper:latest

          cache-from: type=gha
          cache-to: type=gha,mode=max
          push: true

      - name: cosign-installer
        uses: sigstore/cosign-installer@v3.0.5

      - name: Write signing key to disk
        run: echo "$KEY" > cosign.key
        shell: bash
        env:
          KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}

      - name: Sign the published Docker image
        env:
          COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
          COSIGN_EXPERIMENTAL: "true"
        run: echo "datawhores/of-scraper:${{ env.release }}" | xargs -I {} cosign sign --yes --key cosign.key {}@${{ steps.build-and-push.outputs.digest }}

      #       # Sign the resulting Docker image digest except on PRs.
      #       # This will only write to the public Rekor transparency log when the Docker
      #       # repository is public to avoid leaking data.  If you would like to publish
      #       # transparency data even for private images, pass --force to cosign below.
      #       # https://github.com/sigstore/cosign
      #       - name: Sign the published Docker image
      #         env:
      #           COSIGN_EXPERIMENTAL: "true"
      #         # This step uses the identity token to provision an ephemeral certificate
      #         # against the sigstore community Fulcio instance.
      #         run: echo "datawhores/of-scraper:${{ env.release }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }}
      #Login against a Docker registry except on PR
      #https://github.com/docker/login-action
  github_docker:
    needs: [get_commit]
    if: ${{ inputs.docker}}
    runs-on: ubuntu-latest
    name: Docker GHCR Release
    permissions:
      contents: read
      packages: write
      # This is used to complete the identity challenge
      # with sigstore/fulcio when running outside of PRs.
      id-token: write

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{inputs.version}}
      - run: git checkout ${{needs.get_commit.outputs.long}}
      - name: get tag
        run: echo release=$(git describe --tags `git rev-list --tags --max-count=1`) >> $GITHUB_ENV
      - name: Setup Docker buildx
        uses: docker/setup-buildx-action@v2
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2
      #Install the cosign tool except on PR
      #https://github.com/sigstore/cosign-installer
      - name: Install cosign
        uses: sigstore/cosign-installer@main #v2.6.0
        with:
          cosign-release: v1.13.1
      #Login against a Docker registry except on PR
      #https://github.com/docker/login-action

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: login to ghcr
        uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN}}
          # Extract metadata (tags, labels) for Docker
      - name: push image to ghcr
        id: build-and-push2
        uses: docker/build-push-action@v4
        with:
          context: .
          tags: |
            ghcr.io/datawhores/of-scraper:${{ inputs.version }}
            ghcr.io/datawhores/of-scraper:latest
          cache-from: type=gha
          cache-to: type=gha,mode=max
          push: true

  docker_old:
    needs: [get_commit]
    if: ${{ inputs.docker_old}}
    runs-on: ubuntu-latest
    name: DockerHub Release Old
    permissions:
      contents: read
      packages: write
      # This is used to complete the identity challenge
      # with sigstore/fulcio when running outside of PRs.
      id-token: write

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{inputs.version}}
      - run: git checkout ${{needs.get_commit.outputs.long}}
      - name: get tag
        run: echo release=$(git describe --tags `git rev-list --tags --max-count=1`) >> $GITHUB_ENV
      - name: Setup Docker buildx
        uses: docker/setup-buildx-action@v2
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2
      #Install the cosign tool except on PR
      #https://github.com/sigstore/cosign-installer
      - name: Install cosign
        uses: sigstore/cosign-installer@main #v2.6.0
        with:
          cosign-release: v1.13.1
      #Login against a Docker registry except on PR
      #https://github.com/docker/login-action

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2

      - name: Login to Docker Hub
        uses: docker/login-action@v2
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}

      - name: push image to Docker Hub
        id: build-and-push
        uses: docker/build-push-action@v4
        with:
          context: .
          tags: |
            datawhores/of-scraper:${{ inputs.version }}

          cache-from: type=gha
          cache-to: type=gha,mode=max
          push: true

      - name: cosign-installer
        uses: sigstore/cosign-installer@v3.0.5

      - name: Write signing key to disk
        run: echo "$KEY" > cosign.key
        shell: bash
        env:
          KEY: ${{ secrets.COSIGN_PRIVATE_KEY }}

      - name: Sign the published Docker image
        env:
          COSIGN_PASSWORD: ${{ secrets.COSIGN_PASSWORD }}
          COSIGN_EXPERIMENTAL: "true"
        run: echo "datawhores/of-scraper:${{ env.release }}" | xargs -I {} cosign sign --yes --key cosign.key {}@${{ steps.build-and-push.outputs.digest }}

      #       # Sign the resulting Docker image digest except on PRs.
      #       # This will only write to the public Rekor transparency log when the Docker
      #       # repository is public to avoid leaking data.  If you would like to publish
      #       # transparency data even for private images, pass --force to cosign below.
      #       # https://github.com/sigstore/cosign
      #       - name: Sign the published Docker image
      #         env:
      #           COSIGN_EXPERIMENTAL: "true"
      #         # This step uses the identity token to provision an ephemeral certificate
      #         # against the sigstore community Fulcio instance.
      #         run: echo "datawhores/of-scraper:${{ env.release }}" | xargs -I {} cosign sign {}@${{ steps.build-and-push.outputs.digest }}
      #Login against a Docker registry except on PR
      #https://github.com/docker/login-action

  docker_old_github:
    needs: [get_commit]
    if: ${{ inputs.docker_old}}
    runs-on: ubuntu-latest
    name: GHCR Release Old
    permissions:
      contents: read
      packages: write
      # This is used to complete the identity challenge
      # with sigstore/fulcio when running outside of PRs.
      id-token: write

    steps:
      - name: Checkout repository
        uses: actions/checkout@v4
        with:
          fetch-depth: 0
          ref: ${{inputs.version}}
      - run: git checkout ${{needs.get_commit.outputs.long}}
      - name: get tag
        run: echo release=$(git describe --tags `git rev-list --tags --max-count=1`) >> $GITHUB_ENV
      - name: Setup Docker buildx
        uses: docker/setup-buildx-action@v2
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v2
      #Install the cosign tool except on PR
      #https://github.com/sigstore/cosign-installer
      - name: Install cosign
        uses: sigstore/cosign-installer@main #v2.6.0
        with:
          cosign-release: v1.13.1
      #Login against a Docker registry except on PR
      #https://github.com/docker/login-action

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v2
      - name: login to ghcr
        uses: docker/login-action@v2
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN}}
          # Extract metadata (tags, labels) for Docker
      - name: push image to ghcr
        id: build-and-push2
        uses: docker/build-push-action@v4
        with:
          context: .
          tags: |
            ghcr.io/datawhores/of-scraper:${{ inputs.version }}
          cache-from: type=gha
          cache-to: type=gha,mode=max
          push: true


Build docs

name: Build docs

on:
  push:
    branches:
      - main
  workflow_dispatch:

jobs:
  build-docs:
    runs-on: ubuntu-latest
    name: Sphinx docs to gh-pages
    steps:
      - uses: actions/checkout@v4
      - uses: actions/cache@v4
        with:
          path: |
            ~/.gdsfactory/
          key: 0.0.1
          restore-keys: 0.0.1
      - uses: mamba-org/setup-micromamba@v1
        with:
          micromamba-version: '1.3.1-0'
          init-shell: bash
      - name: Install dependencies and build docs
        env:
          SIMCLOUD_APIKEY: ${{ secrets.SIMCLOUD_APIKEY }}
          GDSFACTORY_DISPLAY_TYPE: klayout
        run: |
          eval "$(micromamba shell hook --shell=bash)"
          micromamba activate
          micromamba install -c conda-forge pymeep=*=mpi_mpich_* nlopt -y
          make dev test-data
          make docs
      - name: Upload artifact
        uses: actions/upload-pages-artifact@v3
        with:
          path: "./docs/_build/html/"
  deploy-docs:
    needs: build-docs
    permissions:
      pages: write
      id-token: write

    environment:
      name: github-pages
      url: ${{ steps.deployment.outputs.page_url }}

    runs-on: ubuntu-latest
    steps:
      - name: Deploy to GitHub Pages
        id: deployment
        uses: actions/deploy-pages@v4


Release package

name: Release package

on:
  push:
    tags: "v*"

jobs:
  release_pypi:
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: 3.x
          cache-dependency-path: pyproject.toml
      - name: Install dependencies
        run: |
          python -m pip install --upgrade pip
          pip install setuptools wheel twine
      - name: Build and publish
        env:
          TWINE_USERNAME: ${{ secrets.PYPI_USERNAME }}
          TWINE_PASSWORD: ${{ secrets.PYPI_PASSWORD }}
        run: |
          make build
          twine upload dist/*

  release_environment:
    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags')
    runs-on: ubuntu-latest
    permissions:
      contents: write
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: '3.11'
          cache: "pip"
          cache-dependency-path: pyproject.toml
      - name: Install dependencies
        run: |
          make install
          pip freeze > requirements.txt
      - name: Publish Latest Draft
        run: |
          gh release edit ${{ github.ref_name }} --draft=false
          gh release upload ${{ github.ref_name }} requirements.txt --clobber
        env:
          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}


Test code

name: Test code

on:
  pull_request:
  push:
    branches:
      - main

jobs:
  pre-commit:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v5
        with:
          python-version: "3.11"
          cache: "pip"
          cache-dependency-path: pyproject.toml
      - name: Test pre-commit hooks
        run: |
          python -m pip install --upgrade pip
          pip install pre-commit
          pre-commit run -a
  test_code_pip:
    runs-on: ubuntu-latest
    container: ghcr.io/gdsfactory/gdsfactory:main
    strategy:
      fail-fast: false
      max-parallel: 12
      matrix:
        plugin: [femwell, gmsh, meow, sax, tidy3d, klayout, vlsir]

    name: Test ${{ matrix.plugin }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cache/pip
          key: ${{ hashFiles('pyproject.toml') }}
      - name: Install dependencies
        run: |
          pip install -e .[${{ matrix.plugin }},dev]
          pip install git+https://github.com/gdsfactory/gdsfactory --force-reinstall
      - name: Test with pytest
        env:
          SIMCLOUD_APIKEY: ${{ secrets.SIMCLOUD_APIKEY }}
          GDSFACTORY_DISPLAY_TYPE: klayout
        run: pytest gplugins/${{ matrix.plugin }}
  test_code_conda:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: mamba-org/setup-micromamba@v1
        with:
          micromamba-version: "1.3.1-0"
          init-shell: bash
      - name: Install dependencies
        run: |
          eval "$(micromamba shell hook --shell=bash)"
          micromamba shell init --shell=bash --prefix=~/micromamba
          micromamba activate
          micromamba install -c conda-forge pymeep=*=mpi_mpich_* nlopt -y
          pip install -e .[dev]
          pytest gplugins/gmeep gplugins/modes
  test_non_pip:
    runs-on: ${{ matrix.os }}
    strategy:
      fail-fast: false
      max-parallel: 2
      matrix:
        os: [ubuntu-latest]
        plugin: [elmer]
    name: Test ${{ matrix.plugin }} on ${{ matrix.os }}
    steps:
      - uses: actions/checkout@v4
      - uses: actions/cache@v4
        with:
          path: |
            ~/.cache/pip
          key: ${{ hashFiles('pyproject.toml') }}
      - uses: actions/setup-python@v5
        with:
          python-version: 3.11
      - name: Install dependencies
        run: |
          make gmsh
          make ${{ matrix.plugin }}
          pip install -e .[dev,gmsh]
      - name: Test with pytest
        env:
          GDSFACTORY_DISPLAY_TYPE: klayout
        run: pytest gplugins/${{ matrix.plugin }}
  test_code_coverage:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: mamba-org/setup-micromamba@v1
        with:
          micromamba-version: "1.3.1-0"
          init-shell: bash
      - name: Install dependencies
        env:
          SIMCLOUD_APIKEY: ${{ secrets.SIMCLOUD_APIKEY }}
          GDSFACTORY_DISPLAY_TYPE: klayout
        run: |
          eval "$(micromamba shell hook --shell=bash)"
          micromamba activate
          micromamba install -c conda-forge pymeep=*=mpi_mpich_* nlopt -y
          make dev
          pip freeze > requirements.txt
          pytest --cov=gplugins
      - name: Upload coverage to Codecov
        uses: codecov/codecov-action@v4
        with:
          token: ${{ secrets.CODECOV_TOKEN }}
          fail_ci_if_error: false
  test_docs:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - uses: actions/cache@v4
        with:
          path: |
            ~/.gdsfactory/
          key: 0.0.1
          restore-keys: 0.0.1
      - uses: mamba-org/setup-micromamba@v1
        with:
          micromamba-version: "1.3.1-0"
          init-shell: bash
      - name: Install dependencies and build docs
        env:
          SIMCLOUD_APIKEY: ${{ secrets.SIMCLOUD_APIKEY }}
          GDSFACTORY_DISPLAY_TYPE: klayout
        run: |
          eval "$(micromamba shell hook --shell=bash)"
          micromamba activate
          micromamba install -c conda-forge pymeep=*=mpi_mpich_* nlopt -y
          make test-data
          make dev
          make docs
      - name: Expose docs artifact
        uses: actions/upload-artifact@v4
        if: always()
        with:
          name: docs
          path: docs/_build/html/


Release Drafter and Labels

name: Release Drafter and Labels

on:
  push:
    branches:
      - main
  pull_request:
    types: [edited, opened, reopened, synchronize, unlabeled, labeled]

permissions:
  contents: read

jobs:
  update_release_draft:
    permissions:
      contents: write
      pull-requests: write
    runs-on: ubuntu-latest
    steps:
      # Drafts your next Release notes as Pull Requests are merged into "master"
      - uses: release-drafter/release-drafter@v6
        env:
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
  require_label:
    if: github.event.pull_request
    needs: update_release_draft
    runs-on: ubuntu-latest
    permissions:
      issues: write
      pull-requests: write
    steps:
      - uses: mheap/github-action-required-labels@v5
        with:
          mode: minimum
          count: 1
          labels: "breaking, bug, github_actions, documentation, dependencies, enhancement, feature, maintenance, security"
          add_comment: true


Build and Push Container

name: Build and Push Container

on:
  push:
    branches:
      - main
  workflow_dispatch:

jobs:
  build:
    runs-on: ubuntu-latest

    steps:
      - uses: actions/checkout@v4

      - name: Login to github container registry
        uses: docker/login-action@v3
        with:
          registry: ghcr.io
          username: ${{ github.actor }}
          password: ${{ secrets.GITHUB_TOKEN }}

      - name: Build and tag latest docker image
        run: |
          docker build -f Dockerfile -t ghcr.io/${{ github.repository_owner }}/nuts:$GITHUB_SHA .

      - name: Tag versioned image
        run: docker tag ghcr.io/${{ github.repository_owner }}/nuts:$GITHUB_SHA ghcr.io/${{ github.repository_owner }}/nuts:latest

      - name: Push the images
        run: |
          docker push ghcr.io/${{ github.repository_owner }}/nuts:latest
          docker push ghcr.io/${{ github.repository_owner }}/nuts:$GITHUB_SHA


# https://packaging.python.org/guides/publishing-package-distribution-releases-using-github-actions-ci-cd-workflows/
# https://www.ianwootten.co.uk/2020/10/23/publishing-to-pypi-using-github-actions/
name: Publish nuts to PyPI and release on GitHub
on:
  push:
    tags:
      - "*"

jobs:
  build-and-publish-to-pypi:
    name: Build and publish nuts to PyPI
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python 3.8
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
      - name: Install poetry
        run: pip install poetry
      - name: Configure API token
        run: poetry config pypi-token.pypi "${{ secrets.PYPI_API_TOKEN }}"
      - name: Build and then publish nuts to PyPI
        run: poetry publish --build
  relase-on-github:
    name: Release on GitHub
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Release on Github
        uses: softprops/action-gh-release@v2
        with:
          body_path: ${{ github.workspace }}/release_notes/${{ github.ref_name }}.md


Run tests against pytest-master

name: Run tests against pytest-master
on:
  schedule:
    - cron:  '* 22 * * 0'
  workflow_dispatch:

jobs:
  self-tests:
    name: test against pytest-main
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: 3.8
      - name: install tox
        run: pip install tox
      - name: run tox against pytest-main
        run: tox -e pytest-main


Run tests with tox

name: Run tests with tox
on: [push, workflow_dispatch]
jobs:
  self-tests:
    name: pytest tests with tox
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python: [3.8, 3.9, "3.10"]
        include:
          - python: 3.8
            testenv: py
          - python: 3.9
            testenv: py
          - python: "3.10"
            testenv: py
    steps:
      - uses: actions/checkout@v4
      - uses: actions/cache@v2
        with:
          path: |
            .mypy_cache
            .tox
            ~/.cache/pip
          key: "${{ matrix.testenv }}-${{ hashFiles('poetry.lock') }}-${{ hashFiles('pyproject.toml') }}"
      - name: "set up Python ${{ matrix.python }}"
        uses: actions/setup-python@v2
        with:
          python-version: "${{ matrix.python }}"
      - name: install tox
        run: pip install tox
      - name: run tox
        run: "tox -e ${{ matrix.testenv }}"

  quality:
    name: quality tests with tox
    runs-on: ubuntu-latest
    strategy:
      matrix:
        testenv: [mypy, black, flake8, docs]
    steps:
      - uses: actions/checkout@v4
      - uses: actions/cache@v2
        with:
          path: |
            .mypy_cache
            .tox
            ~/.cache/pip
          key: "${{ matrix.testenv }}-${{ hashFiles('poetry.lock') }}-${{ hashFiles('pyproject.toml') }}"
      - name: Set up Python
        uses: actions/setup-python@v2
        with:
          python-version: "${{ matrix.python }}"
      - name: install tox
        run: pip install tox
      - name: "Run tox with ${{ matrix.testenv }}"
        run: "tox -e ${{ matrix.testenv}}"


CI / cd . / make spell_check

---
name: CI / cd . / make spell_check

on:
  push:
    branches: [master, v0.1]
  pull_request:
    branches: [master, v0.1]

permissions:
  contents: read

jobs:
  codespell:
    name: (Check for spelling errors)
    runs-on: ubuntu-latest

    steps:
      - name: Checkout
        uses: actions/checkout@v4

      - name: Install Dependencies
        run: |
          pip install toml

      - name: Extract Ignore Words List
        run: |
          # Use a Python script to extract the ignore words list from pyproject.toml
          python .github/workflows/extract_ignored_words_list.py
        id: extract_ignore_words

#      - name: Codespell
#        uses: codespell-project/actions-codespell@v2
#        with:
#          skip: guide_imports.json,*.ambr,./cookbook/data/imdb_top_1000.csv,*.lock
#          ignore_words_list: ${{ steps.extract_ignore_words.outputs.ignore_words_list }}
#          exclude_file: ./.github/workflows/codespell-exclude


test

name: test

on:
  workflow_call:
    inputs:
      working-directory:
        required: true
        type: string
        description: "From which folder this pipeline executes"
      langchain-location:
        required: false
        type: string
        description: "Relative path to the langchain library folder"

env:
  POETRY_VERSION: "1.7.1"

jobs:
  build:
    defaults:
      run:
        working-directory: ${{ inputs.working-directory }}
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version:
          - "3.8"
          - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"
    name: "make test #${{ matrix.python-version }}"
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
        uses: "./.github/actions/poetry_setup"
        with:
          python-version: ${{ matrix.python-version }}
          poetry-version: ${{ env.POETRY_VERSION }}
          working-directory: ${{ inputs.working-directory }}
          cache-key: core

      - name: Install dependencies
        shell: bash
        run: poetry install --with test

      - name: Install langchain editable
        working-directory: ${{ inputs.working-directory }}
        if: ${{ inputs.langchain-location }}
        env:
          LANGCHAIN_LOCATION: ${{ inputs.langchain-location }}
        run: |
          poetry run pip install -e "$LANGCHAIN_LOCATION"

      - name: Run core tests
        shell: bash
        run: |
          make test

      - name: Ensure the tests did not create any additional files
        shell: bash
        run: |
          set -eu

          STATUS="$(git status)"
          echo "$STATUS"

          # grep will exit non-zero if the target message isn't found,
          # and `set -e` above will cause the step to fail.
          echo "$STATUS" | grep 'nothing to commit, working tree clean'


lint

name: lint

on:
  workflow_call:
    inputs:
      working-directory:
        required: true
        type: string
        description: "From which folder this pipeline executes"
      langchain-location:
        required: false
        type: string
        description: "Relative path to the langchain library folder"

env:
  POETRY_VERSION: "1.7.1"
  WORKDIR: ${{ inputs.working-directory == '' && '.' || inputs.working-directory }}

  # This env var allows us to get inline annotations when ruff has complaints.
  RUFF_OUTPUT_FORMAT: github

jobs:
  build:
    name: "make lint #${{ matrix.python-version }}"
    runs-on: ubuntu-latest
    strategy:
      matrix:
        # Only lint on the min and max supported Python versions.
        # It's extremely unlikely that there's a lint issue on any version in between
        # that doesn't show up on the min or max versions.
        #
        # GitHub rate-limits how many jobs can be running at any one time.
        # Starting new jobs is also relatively slow,
        # so linting on fewer versions makes CI faster.
        python-version:
          - "3.8"
          - "3.12"
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
        uses: "./.github/actions/poetry_setup"
        with:
          python-version: ${{ matrix.python-version }}
          poetry-version: ${{ env.POETRY_VERSION }}
          working-directory: ${{ inputs.working-directory }}
          cache-key: lint-with-extras

      - name: Check Poetry File
        shell: bash
        working-directory: ${{ inputs.working-directory }}
        run: |
          poetry check

      - name: Check lock file
        shell: bash
        working-directory: ${{ inputs.working-directory }}
        run: |
          poetry lock --check

      - name: Install dependencies
        # Also installs dev/lint/test/typing dependencies, to ensure we have
        # type hints for as many of our libraries as possible.
        # This helps catch errors that require dependencies to be spotted, for example:
        # https://github.com/langchain-ai/langchain/pull/10249/files#diff-935185cd488d015f026dcd9e19616ff62863e8cde8c0bee70318d3ccbca98341
        #
        # If you change this configuration, make sure to change the `cache-key`
        # in the `poetry_setup` action above to stop using the old cache.
        # It doesn't matter how you change it, any change will cause a cache-bust.
        working-directory: ${{ inputs.working-directory }}
        run: |
          poetry install --with lint,typing

      - name: Install langchain editable
        working-directory: ${{ inputs.working-directory }}
        if: ${{ inputs.langchain-location }}
        env:
          LANGCHAIN_LOCATION: ${{ inputs.langchain-location }}
        run: |
          poetry run pip install -e "$LANGCHAIN_LOCATION"

      - name: Get .mypy_cache to speed up mypy
        uses: actions/cache@v4
        env:
          SEGMENT_DOWNLOAD_TIMEOUT_MIN: "2"
        with:
          path: |
            ${{ env.WORKDIR }}/.mypy_cache
          key: mypy-lint-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ inputs.working-directory }}-${{ hashFiles(format('{0}/poetry.lock', inputs.working-directory)) }}

      - name: Analysing the code with our lint
        working-directory: ${{ inputs.working-directory }}
        run: |
          make lint_package

      - name: Install unit test dependencies
        # Also installs dev/lint/test/typing dependencies, to ensure we have
        # type hints for as many of our libraries as possible.
        # This helps catch errors that require dependencies to be spotted, for example:
        # https://github.com/langchain-ai/langchain/pull/10249/files#diff-935185cd488d015f026dcd9e19616ff62863e8cde8c0bee70318d3ccbca98341
        #
        # If you change this configuration, make sure to change the `cache-key`
        # in the `poetry_setup` action above to stop using the old cache.
        # It doesn't matter how you change it, any change will cause a cache-bust.
        if: ${{ ! startsWith(inputs.working-directory, 'libs/partners/') }}
        working-directory: ${{ inputs.working-directory }}
        run: |
          poetry install --with test
      - name: Install unit+integration test dependencies
        if: ${{ startsWith(inputs.working-directory, 'libs/partners/') }}
        working-directory: ${{ inputs.working-directory }}
        run: |
          poetry install --with test,test_integration

      - name: Get .mypy_cache_test to speed up mypy
        uses: actions/cache@v4
        env:
          SEGMENT_DOWNLOAD_TIMEOUT_MIN: "2"
        with:
          path: |
            ${{ env.WORKDIR }}/.mypy_cache_test
          key: mypy-test-${{ runner.os }}-${{ runner.arch }}-py${{ matrix.python-version }}-${{ inputs.working-directory }}-${{ hashFiles(format('{0}/poetry.lock', inputs.working-directory)) }}

      - name: Analysing the code with our lint
        working-directory: ${{ inputs.working-directory }}
        run: |
          make lint_tests


release

name: release
run-name: Release ${{ inputs.working-directory }} by @${{ github.actor }}
on:
  workflow_call:
    inputs:
      working-directory:
        required: true
        type: string
        description: "From which folder this pipeline executes"
  workflow_dispatch:
    inputs:
      working-directory:
        required: true
        type: string
        default: 'libs/langchain'
      dangerous-nonmaster-release:
        required: false
        type: boolean
        default: false
        description: "Release from a non-master branch (danger!)"

env:
  PYTHON_VERSION: "3.11"
  POETRY_VERSION: "1.7.1"

jobs:
  build:
    if: github.ref == 'refs/heads/master' || inputs.dangerous-nonmaster-release
    environment: Scheduled testing
    runs-on: ubuntu-latest

    outputs:
      pkg-name: ${{ steps.check-version.outputs.pkg-name }}
      version: ${{ steps.check-version.outputs.version }}

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python + Poetry ${{ env.POETRY_VERSION }}
        uses: "./.github/actions/poetry_setup"
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          poetry-version: ${{ env.POETRY_VERSION }}
          working-directory: ${{ inputs.working-directory }}
          cache-key: release

      # We want to keep this build stage *separate* from the release stage,
      # so that there's no sharing of permissions between them.
      # The release stage has trusted publishing and GitHub repo contents write access,
      # and we want to keep the scope of that access limited just to the release job.
      # Otherwise, a malicious `build` step (e.g. via a compromised dependency)
      # could get access to our GitHub or PyPI credentials.
      #
      # Per the trusted publishing GitHub Action:
      # > It is strongly advised to separate jobs for building [...]
      # > from the publish job.
      # https://github.com/pypa/gh-action-pypi-publish#non-goals
      - name: Build project for distribution
        run: poetry build
        working-directory: ${{ inputs.working-directory }}

      - name: Upload build
        uses: actions/upload-artifact@v4
        with:
          name: dist
          path: ${{ inputs.working-directory }}/dist/

      - name: Check Version
        id: check-version
        shell: bash
        working-directory: ${{ inputs.working-directory }}
        run: |
          echo pkg-name="$(poetry version | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT
          echo version="$(poetry version --short)" >> $GITHUB_OUTPUT
  release-notes:
    needs:
      - build
    runs-on: ubuntu-latest
    outputs:
      release-body: ${{ steps.generate-release-body.outputs.release-body }}
    steps:
      - uses: actions/checkout@v4
        with:
          repository: langchain-ai/langchain
          path: langchain
          sparse-checkout: | # this only grabs files for relevant dir
            ${{ inputs.working-directory }}
          ref: master # this scopes to just master branch
          fetch-depth: 0 # this fetches entire commit history
      - name: Check Tags
        id: check-tags
        shell: bash
        working-directory: langchain/${{ inputs.working-directory }}
        env:
          PKG_NAME: ${{ needs.build.outputs.pkg-name }}
          VERSION: ${{ needs.build.outputs.version }}
        run: |
          REGEX="^$PKG_NAME==\\d+\\.\\d+\\.\\d+\$"
          echo $REGEX
          PREV_TAG=$(git tag --sort=-creatordate | grep -P $REGEX || true | head -1)
          TAG="${PKG_NAME}==${VERSION}"
          if [ "$TAG" == "$PREV_TAG" ]; then
            echo "No new version to release"
            exit 1
          fi
          echo tag="$TAG" >> $GITHUB_OUTPUT
          echo prev-tag="$PREV_TAG" >> $GITHUB_OUTPUT
      - name: Generate release body
        id: generate-release-body
        working-directory: langchain
        env:
          WORKING_DIR: ${{ inputs.working-directory }}
          PKG_NAME: ${{ needs.build.outputs.pkg-name }}
          TAG: ${{ steps.check-tags.outputs.tag }}
          PREV_TAG: ${{ steps.check-tags.outputs.prev-tag }}
        run: |
          PREAMBLE="Changes since $PREV_TAG"
          # if PREV_TAG is empty, then we are releasing the first version
          if [ -z "$PREV_TAG" ]; then
            PREAMBLE="Initial release"
            PREV_TAG=$(git rev-list --max-parents=0 HEAD)
          fi
          {
            echo 'release-body<<EOF'
            echo "# Release $TAG"
            echo $PREAMBLE
            echo
            git log --format="%s" "$PREV_TAG"..HEAD -- $WORKING_DIR
            echo EOF
          } >> "$GITHUB_OUTPUT"

  test-pypi-publish:
    needs:
      - build
      - release-notes
    uses:
      ./.github/workflows/_test_release.yml
    with:
      working-directory: ${{ inputs.working-directory }}
      dangerous-nonmaster-release: ${{ inputs.dangerous-nonmaster-release }}
    secrets: inherit

  pre-release-checks:
    needs:
      - build
      - release-notes
      - test-pypi-publish
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@v4

      # We explicitly *don't* set up caching here. This ensures our tests are
      # maximally sensitive to catching breakage.
      #
      # For example, here's a way that caching can cause a falsely-passing test:
      # - Make the langchain package manifest no longer list a dependency package
      #   as a requirement. This means it won't be installed by `pip install`,
      #   and attempting to use it would cause a crash.
      # - That dependency used to be required, so it may have been cached.
      #   When restoring the venv packages from cache, that dependency gets included.
      # - Tests pass, because the dependency is present even though it wasn't specified.
      # - The package is published, and it breaks on the missing dependency when
      #   used in the real world.

      - name: Set up Python + Poetry ${{ env.POETRY_VERSION }}
        uses: "./.github/actions/poetry_setup"
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          poetry-version: ${{ env.POETRY_VERSION }}
          working-directory: ${{ inputs.working-directory }}

      - name: Import published package
        shell: bash
        working-directory: ${{ inputs.working-directory }}
        env:
          PKG_NAME: ${{ needs.build.outputs.pkg-name }}
          VERSION: ${{ needs.build.outputs.version }}
        # Here we use:
        # - The default regular PyPI index as the *primary* index, meaning
        #   that it takes priority (https://pypi.org/simple)
        # - The test PyPI index as an extra index, so that any dependencies that
        #   are not found on test PyPI can be resolved and installed anyway.
        #   (https://test.pypi.org/simple). This will include the PKG_NAME==VERSION
        #   package because VERSION will not have been uploaded to regular PyPI yet.
        # - attempt install again after 5 seconds if it fails because there is
        #   sometimes a delay in availability on test pypi
        run: |
          poetry run pip install \
            --extra-index-url https://test.pypi.org/simple/ \
            "$PKG_NAME==$VERSION" || \
          ( \
            sleep 5 && \
            poetry run pip install \
              --extra-index-url https://test.pypi.org/simple/ \
              "$PKG_NAME==$VERSION" \
          )

          # Replace all dashes in the package name with underscores,
          # since that's how Python imports packages with dashes in the name.
          IMPORT_NAME="$(echo "$PKG_NAME" | sed s/-/_/g)"

          poetry run python -c "import $IMPORT_NAME; print(dir($IMPORT_NAME))"

      - name: Import test dependencies
        run: poetry install --with test,test_integration
        working-directory: ${{ inputs.working-directory }}

      # Overwrite the local version of the package with the test PyPI version.
      - name: Import published package (again)
        working-directory: ${{ inputs.working-directory }}
        shell: bash
        env:
          PKG_NAME: ${{ needs.build.outputs.pkg-name }}
          VERSION: ${{ needs.build.outputs.version }}
        run: |
          poetry run pip install \
            --extra-index-url https://test.pypi.org/simple/ \
            "$PKG_NAME==$VERSION"

      - name: Run unit tests
        run: make tests
        working-directory: ${{ inputs.working-directory }}

      - name: Get minimum versions
        working-directory: ${{ inputs.working-directory }}
        id: min-version
        run: |
          poetry run pip install packaging
          min_versions="$(poetry run python $GITHUB_WORKSPACE/.github/scripts/get_min_versions.py pyproject.toml)"
          echo "min-versions=$min_versions" >> "$GITHUB_OUTPUT"
          echo "min-versions=$min_versions"

      - name: Run unit tests with minimum dependency versions
        if: ${{ steps.min-version.outputs.min-versions != '' }}
        env:
          MIN_VERSIONS: ${{ steps.min-version.outputs.min-versions }}
        run: |
          poetry run pip install --force-reinstall $MIN_VERSIONS --editable .
          make tests
        working-directory: ${{ inputs.working-directory }}

      - name: 'Authenticate to Google Cloud'
        id: 'auth'
        uses: google-github-actions/auth@v2
        with:
          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'

      - name: Run integration tests
        if: ${{ startsWith(inputs.working-directory, 'libs/partners/') }}
        env:
          AI21_API_KEY: ${{ secrets.AI21_API_KEY }}
          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
          TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }}
          AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
          AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }}
          AZURE_OPENAI_LLM_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LLM_DEPLOYMENT_NAME }}
          AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME }}
          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
          GOOGLE_SEARCH_API_KEY: ${{ secrets.GOOGLE_SEARCH_API_KEY }}
          GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }}
          GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
          EXA_API_KEY: ${{ secrets.EXA_API_KEY }}
          NOMIC_API_KEY: ${{ secrets.NOMIC_API_KEY }}
          WATSONX_APIKEY: ${{ secrets.WATSONX_APIKEY }}
          WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }}
          PINECONE_API_KEY: ${{ secrets.PINECONE_API_KEY }}
          PINECONE_ENVIRONMENT: ${{ secrets.PINECONE_ENVIRONMENT }}
          ASTRA_DB_API_ENDPOINT: ${{ secrets.ASTRA_DB_API_ENDPOINT }}
          ASTRA_DB_APPLICATION_TOKEN: ${{ secrets.ASTRA_DB_APPLICATION_TOKEN }}
          ASTRA_DB_KEYSPACE: ${{ secrets.ASTRA_DB_KEYSPACE }}
          ES_URL: ${{ secrets.ES_URL }}
          ES_CLOUD_ID: ${{ secrets.ES_CLOUD_ID }}
          ES_API_KEY: ${{ secrets.ES_API_KEY }}
          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # for airbyte
          MONGODB_ATLAS_URI: ${{ secrets.MONGODB_ATLAS_URI }}
          VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }}
          UPSTAGE_API_KEY: ${{ secrets.UPSTAGE_API_KEY }}
          FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
        run: make integration_tests
        working-directory: ${{ inputs.working-directory }}

  publish:
    needs:
      - build
      - release-notes
      - test-pypi-publish
      - pre-release-checks
    runs-on: ubuntu-latest
    permissions:
      # This permission is used for trusted publishing:
      # https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/
      #
      # Trusted publishing has to also be configured on PyPI for each package:
      # https://docs.pypi.org/trusted-publishers/adding-a-publisher/
      id-token: write

    defaults:
      run:
        working-directory: ${{ inputs.working-directory }}

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python + Poetry ${{ env.POETRY_VERSION }}
        uses: "./.github/actions/poetry_setup"
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          poetry-version: ${{ env.POETRY_VERSION }}
          working-directory: ${{ inputs.working-directory }}
          cache-key: release

      - uses: actions/download-artifact@v4
        with:
          name: dist
          path: ${{ inputs.working-directory }}/dist/

      - name: Publish package distributions to PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          packages-dir: ${{ inputs.working-directory }}/dist/
          verbose: true
          print-hash: true

  mark-release:
    needs:
      - build
      - release-notes
      - test-pypi-publish
      - pre-release-checks
      - publish
    runs-on: ubuntu-latest
    permissions:
      # This permission is needed by `ncipollo/release-action` to
      # create the GitHub release.
      contents: write

    defaults:
      run:
        working-directory: ${{ inputs.working-directory }}

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python + Poetry ${{ env.POETRY_VERSION }}
        uses: "./.github/actions/poetry_setup"
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          poetry-version: ${{ env.POETRY_VERSION }}
          working-directory: ${{ inputs.working-directory }}
          cache-key: release

      - uses: actions/download-artifact@v4
        with:
          name: dist
          path: ${{ inputs.working-directory }}/dist/

      - name: Create Tag
        uses: ncipollo/release-action@v1
        with:
          artifacts: "dist/*"
          token: ${{ secrets.GITHUB_TOKEN }}
          generateReleaseNotes: false
          tag: ${{needs.build.outputs.pkg-name}}==${{ needs.build.outputs.version }}
          body: ${{ needs.release-notes.outputs.release-body }}
          commit: ${{ github.sha }}
          makeLatest: ${{ needs.build.outputs.pkg-name == 'langchain-core'}}


Scheduled tests

name: Scheduled tests

on:
  workflow_dispatch:  # Allows to trigger the workflow manually in GitHub UI
  schedule:
    - cron:  '0 13 * * *'

env:
  POETRY_VERSION: "1.7.1"

jobs:
  build:
    if: github.repository_owner == 'langchain-ai'
    name: Python ${{ matrix.python-version }} - ${{ matrix.working-directory }}
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version:
          - "3.8"
          - "3.11"
        working-directory:
          - "libs/partners/openai"
          - "libs/partners/anthropic"
          - "libs/partners/ai21"
          - "libs/partners/fireworks"
          - "libs/partners/groq"
          - "libs/partners/mistralai"
          - "libs/partners/together"
          - "libs/partners/cohere"
          - "libs/partners/google-vertexai"
          - "libs/partners/google-genai"
          - "libs/partners/aws"

    steps:
      - uses: actions/checkout@v4
        with:
          path: langchain
      - uses: actions/checkout@v4
        with:
          repository: langchain-ai/langchain-google
          path: langchain-google
      - uses: actions/checkout@v4
        with:
          repository: langchain-ai/langchain-cohere
          path: langchain-cohere
      - uses: actions/checkout@v4
        with:
          repository: langchain-ai/langchain-aws
          path: langchain-aws

      - name: Move libs
        run: |
          rm -rf \
            langchain/libs/partners/google-genai \
            langchain/libs/partners/google-vertexai \
            langchain/libs/partners/cohere
          mv langchain-google/libs/genai langchain/libs/partners/google-genai
          mv langchain-google/libs/vertexai langchain/libs/partners/google-vertexai
          mv langchain-cohere/libs/cohere langchain/libs/partners/cohere
          mv langchain-aws/libs/aws langchain/libs/partners/aws

      - name: Set up Python ${{ matrix.python-version }}
        uses: "./langchain/.github/actions/poetry_setup"
        with:
          python-version: ${{ matrix.python-version }}
          poetry-version: ${{ env.POETRY_VERSION }}
          working-directory: langchain/${{ matrix.working-directory }}
          cache-key: scheduled

      - name: 'Authenticate to Google Cloud'
        id: 'auth'
        uses: google-github-actions/auth@v2
        with:
          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'

      - name: Configure AWS Credentials
        uses: aws-actions/configure-aws-credentials@v4
        with:
          aws-access-key-id: ${{ secrets.AWS_ACCESS_KEY_ID }}
          aws-secret-access-key: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
          aws-region: ${{ secrets.AWS_REGION }}

      - name: Install dependencies
        run: |
          echo "Running scheduled tests, installing dependencies with poetry..."
          cd langchain/${{ matrix.working-directory }}
          poetry install --with=test_integration,test

      - name: Run integration tests
        env:
          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
          ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
          AZURE_OPENAI_API_VERSION: ${{ secrets.AZURE_OPENAI_API_VERSION }}
          AZURE_OPENAI_API_BASE: ${{ secrets.AZURE_OPENAI_API_BASE }}
          AZURE_OPENAI_API_KEY: ${{ secrets.AZURE_OPENAI_API_KEY }}
          AZURE_OPENAI_CHAT_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_CHAT_DEPLOYMENT_NAME }}
          AZURE_OPENAI_LLM_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_LLM_DEPLOYMENT_NAME }}
          AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME: ${{ secrets.AZURE_OPENAI_EMBEDDINGS_DEPLOYMENT_NAME }}
          AI21_API_KEY: ${{ secrets.AI21_API_KEY }}
          FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
          GROQ_API_KEY: ${{ secrets.GROQ_API_KEY }}
          MISTRAL_API_KEY: ${{ secrets.MISTRAL_API_KEY }}
          TOGETHER_API_KEY: ${{ secrets.TOGETHER_API_KEY }}
          COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
          NVIDIA_API_KEY: ${{ secrets.NVIDIA_API_KEY }}
          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
          GOOGLE_SEARCH_API_KEY: ${{ secrets.GOOGLE_SEARCH_API_KEY }}
          GOOGLE_CSE_ID: ${{ secrets.GOOGLE_CSE_ID }}
        run: |
          cd langchain/${{ matrix.working-directory }}
          make integration_tests

      - name: Remove external libraries
        run: |
          rm -rf \
            langchain/libs/partners/google-genai \
            langchain/libs/partners/google-vertexai \
            langchain/libs/partners/cohere \
            langchain/libs/partners/aws

      - name: Ensure the tests did not create any additional files
        working-directory: langchain
        run: |
          set -eu

          STATUS="$(git status)"
          echo "$STATUS"

          # grep will exit non-zero if the target message isn't found,
          # and `set -e` above will cause the step to fail.
          echo "$STATUS" | grep 'nothing to commit, working tree clean'


docker/langchain/langchain Release

---
name: docker/langchain/langchain Release

on:
  workflow_dispatch: # Allows to trigger the workflow manually in GitHub UI
  workflow_call: # Allows triggering from another workflow

jobs:
  release:
    uses: ./.github/workflows/_release_docker.yml
    with:
      dockerfile: docker/Dockerfile.base
      image: langchain/langchain
    secrets: inherit


name: release_docker

on:
  workflow_call:
    inputs:
      dockerfile:
        required: true
        type: string
        description: "Path to the Dockerfile to build"
      image:
        required: true
        type: string
        description: "Name of the image to build"

env:
  TEST_TAG: ${{ inputs.image }}:test
  LATEST_TAG: ${{ inputs.image }}:latest

jobs:
  docker:
    runs-on: ubuntu-latest
    steps:
      - name: Checkout
        uses: actions/checkout@v4
      - name: Get git tag
        uses: actions-ecosystem/action-get-latest-tag@v1
        id: get-latest-tag
      - name: Set docker tag
        env:
          VERSION: ${{ steps.get-latest-tag.outputs.tag }}
        run: |
          echo "VERSION_TAG=${{ inputs.image }}:${VERSION#v}" >> $GITHUB_ENV
      - name: Set up QEMU
        uses: docker/setup-qemu-action@v3
      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v3
      - name: Login to Docker Hub
        uses: docker/login-action@v3
        with:
          username: ${{ secrets.DOCKERHUB_USERNAME }}
          password: ${{ secrets.DOCKERHUB_TOKEN }}
      - name: Build for Test
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ${{ inputs.dockerfile }}
          load: true
          tags: ${{ env.TEST_TAG }}
      - name: Test
        run: |
          docker run --rm ${{ env.TEST_TAG }} python -c "import langchain"
      - name: Build and Push to Docker Hub
        uses: docker/build-push-action@v5
        with:
          context: .
          file: ${{ inputs.dockerfile }}
          # We can only build for the intersection of platforms supported by
          # QEMU and base python image, for now build only for
          # linux/amd64 and linux/arm64
          platforms: linux/amd64,linux/arm64
          tags: ${{ env.LATEST_TAG }},${{ env.VERSION_TAG }}
          push: true


test-release

name: test-release

on:
  workflow_call:
    inputs:
      working-directory:
        required: true
        type: string
        description: "From which folder this pipeline executes"
      dangerous-nonmaster-release:
        required: false
        type: boolean
        default: false
        description: "Release from a non-master branch (danger!)"

env:
  POETRY_VERSION: "1.7.1"
  PYTHON_VERSION: "3.10"

jobs:
  build:
    if: github.ref == 'refs/heads/master' || inputs.dangerous-nonmaster-release
    runs-on: ubuntu-latest

    outputs:
      pkg-name: ${{ steps.check-version.outputs.pkg-name }}
      version: ${{ steps.check-version.outputs.version }}

    steps:
      - uses: actions/checkout@v4

      - name: Set up Python + Poetry ${{ env.POETRY_VERSION }}
        uses: "./.github/actions/poetry_setup"
        with:
          python-version: ${{ env.PYTHON_VERSION }}
          poetry-version: ${{ env.POETRY_VERSION }}
          working-directory: ${{ inputs.working-directory }}
          cache-key: release

      # We want to keep this build stage *separate* from the release stage,
      # so that there's no sharing of permissions between them.
      # The release stage has trusted publishing and GitHub repo contents write access,
      # and we want to keep the scope of that access limited just to the release job.
      # Otherwise, a malicious `build` step (e.g. via a compromised dependency)
      # could get access to our GitHub or PyPI credentials.
      #
      # Per the trusted publishing GitHub Action:
      # > It is strongly advised to separate jobs for building [...]
      # > from the publish job.
      # https://github.com/pypa/gh-action-pypi-publish#non-goals
      - name: Build project for distribution
        run: poetry build
        working-directory: ${{ inputs.working-directory }}

      - name: Upload build
        uses: actions/upload-artifact@v4
        with:
          name: test-dist
          path: ${{ inputs.working-directory }}/dist/

      - name: Check Version
        id: check-version
        shell: bash
        working-directory: ${{ inputs.working-directory }}
        run: |
          echo pkg-name="$(poetry version | cut -d ' ' -f 1)" >> $GITHUB_OUTPUT
          echo version="$(poetry version --short)" >> $GITHUB_OUTPUT

  publish:
    needs:
      - build
    runs-on: ubuntu-latest
    permissions:
      # This permission is used for trusted publishing:
      # https://blog.pypi.org/posts/2023-04-20-introducing-trusted-publishers/
      #
      # Trusted publishing has to also be configured on PyPI for each package:
      # https://docs.pypi.org/trusted-publishers/adding-a-publisher/
      id-token: write

    steps:
      - uses: actions/checkout@v4

      - uses: actions/download-artifact@v4
        with:
          name: test-dist
          path: ${{ inputs.working-directory }}/dist/

      - name: Publish to test PyPI
        uses: pypa/gh-action-pypi-publish@release/v1
        with:
          packages-dir: ${{ inputs.working-directory }}/dist/
          verbose: true
          print-hash: true
          repository-url: https://test.pypi.org/legacy/

          # We overwrite any existing distributions with the same name and version.
          # This is *only for CI use* and is *extremely dangerous* otherwise!
          # https://github.com/pypa/gh-action-pypi-publish#tolerating-release-package-file-duplicates
          skip-existing: true


compile-integration-test

name: compile-integration-test

on:
  workflow_call:
    inputs:
      working-directory:
        required: true
        type: string
        description: "From which folder this pipeline executes"

env:
  POETRY_VERSION: "1.7.1"

jobs:
  build:
    defaults:
      run:
        working-directory: ${{ inputs.working-directory }}
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python-version:
          - "3.8"
          - "3.9"
          - "3.10"
          - "3.11"
          - "3.12"
    name: "poetry run pytest -m compile tests/integration_tests #${{ matrix.python-version }}"
    steps:
      - uses: actions/checkout@v4

      - name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
        uses: "./.github/actions/poetry_setup"
        with:
          python-version: ${{ matrix.python-version }}
          poetry-version: ${{ env.POETRY_VERSION }}
          working-directory: ${{ inputs.working-directory }}
          cache-key: compile-integration

      - name: Install integration dependencies
        shell: bash
        run: poetry install --with=test_integration,test

      - name: Check integration tests compile
        shell: bash
        run: poetry run pytest -m compile tests/integration_tests

      - name: Ensure the tests did not create any additional files
        shell: bash
        run: |
          set -eu

          STATUS="$(git status)"
          echo "$STATUS"

          # grep will exit non-zero if the target message isn't found,
          # and `set -e` above will cause the step to fail.
          echo "$STATUS" | grep 'nothing to commit, working tree clean'