langgraph/.github/workflows/bench.yml

name: bench

on:
  pull_request:
    paths:
      - "libs/**"

env:
  POETRY_VERSION: "1.7.1"

jobs:
  baseline:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: libs/langgraph
    steps:
      - uses: actions/checkout@v4
        with:
          ref: main
      - name: Set up Python 3.11 + Poetry ${{ env.POETRY_VERSION }}
        uses: "./.github/actions/poetry_setup"
        with:
          python-version: "3.11"
          poetry-version: ${{ env.POETRY_VERSION }}
          cache-key: bench
      - name: Install dependencies
        run: poetry install --with dev
      - name: Run benchmarks
        run: OUTPUT=out/benchmark-baseline.json make -s benchmark
      - name: Upload benchmark baseline
        uses: actions/upload-artifact@v4
        with:
          name: benchmark-baseline
          path: libs/langgraph/out/benchmark-baseline.json
  benchmark:
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: libs/langgraph
    needs: [baseline]
    steps:
      - uses: actions/checkout@v4
      - id: files
        name: Get changed files
        uses: Ana06/get-changed-files@v2.3.0
        with:
          format: json
      - name: Set up Python 3.11 + Poetry ${{ env.POETRY_VERSION }}
        uses: "./.github/actions/poetry_setup"
        with:
          python-version: "3.11"
          poetry-version: ${{ env.POETRY_VERSION }}
          cache-key: bench
      - name: Install dependencies
        run: poetry install --with dev
      - name: Run benchmarks
        id: benchmark
        run: |
          {
            echo 'OUTPUT<<EOF'
            make -s benchmark
            echo EOF
          } >> "$GITHUB_OUTPUT"
      - name: Download benchmark baseline
        uses: actions/download-artifact@v4
        with:
          path: libs/langgraph/out
          merge-multiple: true
      - name: Compare benchmarks
        id: compare
        run: |
          {
            echo 'OUTPUT<<EOF'
            poetry run pyperf compare_to out/benchmark-baseline.json out/benchmark.json --table --group-by-speed
            echo EOF
          } >> "$GITHUB_OUTPUT"
      - name: Annotation
        uses: actions/github-script@v7
        with:
          script: |
            const file = JSON.parse(`${{ steps.files.outputs.added_modified_renamed }}`)[0]
            core.notice(`${{ steps.benchmark.outputs.OUTPUT }}`, {
              title: 'Benchmark results',
              file,
            })
            core.notice(`${{ steps.compare.outputs.OUTPUT }}`, {
              title: 'Comparison against main',
              file,
            })