ci(benchmark): Add nightly benchmark workflow

Added autogpts-benchmark-nightly.yml, which will run every night at 02:00 UTC with a selection of challenges.

1 files changed, 71 insertions, 0 deletions

diff --git a/.github/workflows/autogpts-benchmark-nightly.yml b/.github/workflows/autogpts-benchmark-nightly.yml
new file mode 100644
index 000000000..6c4d7461c
--- /dev/null
+++ b/.github/workflows/autogpts-benchmark-nightly.yml
@@ -0,0 +1,71 @@
+name: AutoGPTs Nightly Benchmark
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: '0 2 * * *'
+
+env:
+  TELEMETRY_ENVIRONMENT: autogpt-benchmark-ci
+  TELEMETRY_OPT_IN: ${{ github.event_name == 'push' && github.ref_name == 'master' }}
+
+jobs:
+  benchmark:
+    permissions:
+      contents: write
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        agent-name: [ autogpt ]
+      fail-fast: false
+    timeout-minutes: 120
+    env:
+      min-python-version: '3.10'
+      REPORTS_BRANCH: data/benchmark-reports
+      REPORTS_FOLDER: ${{ format('benchmark/reports/{0}', matrix.agent-name) }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+          submodules: true
+
+      - name: Set up Python ${{ env.min-python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.min-python-version }}
+
+      - name: Install Poetry
+        run: curl -sSL https://install.python-poetry.org | python -
+
+      - name: Prepare reports folder
+        run: mkdir -p ${{ env.REPORTS_FOLDER }}
+
+      - name: Benchmark ${{ matrix.agent-name }}
+        run: |
+          ./run agent start ${{ matrix.agent-name }}
+          cd autogpts/${{ matrix.agent-name }}
+          poetry run agbenchmark run -N 3 \
+            --test=ReadFile \
+            --test=BasicRetrieval --test=RevenueRetrieval2 \
+            --test=CombineCsv --test=LabelCsv --test=AnswerQuestionCombineCsv \
+            --test=UrlShortener --test=TicTacToe --test=Battleship \
+            --test=WebArenaTask_0 --test=WebArenaTask_21 --test=WebArenaTask_124 \
+            --test=WebArenaTask_134 --test=WebArenaTask_163
+        env:
+          AGENT_NAME: ${{ matrix.agent-name }}
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          REQUESTS_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt
+          REPORTS_FOLDER: ${{ format('../../{0}', env.REPORTS_FOLDER) }}  # account for changed workdir
+
+      - name: Push reports to data branch
+        run: |
+          git config --global user.name 'GitHub Actions'
+          git config --global user.email 'github-actions@agpt.co'
+          git fetch origin ${{ env.REPORTS_BRANCH }}:${{ env.REPORTS_BRANCH }} \
+            && git checkout ${{ env.REPORTS_BRANCH }} \
+            || git checkout --orphan ${{ env.REPORTS_BRANCH }}
+          git reset --hard
+          git add ${{ env.REPORTS_FOLDER }}
+          git commit -m "Benchmark report for ${{ matrix.agent-name }} @ $(date +'%Y-%m-%d')" \
+            && git push origin ${{ env.REPORTS_BRANCH }}