diff options
author | Reinier van der Leer <pwuts@agpt.co> | 2024-02-15 18:07:45 +0100 |
---|---|---|
committer | Reinier van der Leer <pwuts@agpt.co> | 2024-02-15 18:07:45 +0100 |
commit | 679339d00c20a4494b6c3ac9f399b2c470a16c17 (patch) | |
tree | e241bdfaab7b28db9618aa7ea18abd76212602dc | |
parent | feat(agent/telemetry): Distinguish between `production` and `dev` environment... (diff) | |
download | Auto-GPT-679339d00c20a4494b6c3ac9f399b2c470a16c17.tar.gz Auto-GPT-679339d00c20a4494b6c3ac9f399b2c470a16c17.tar.bz2 Auto-GPT-679339d00c20a4494b6c3ac9f399b2c470a16c17.zip |
feat(benchmark): Make report output folder configurable
- Make `AgentBenchmarkConfig.reports_folder` directly configurable (through `REPORTS_FOLDER` env variable). The default is still `./agbenchmark_config/reports`.
- Change all mentions of `REPORT_LOCATION` (which fulfilled the same function at some point in the past) to `REPORTS_FOLDER`.
-rw-r--r-- | .github/workflows/autogpts-ci.yml | 2 | ||||
-rw-r--r-- | benchmark/.env.example | 2 | ||||
-rw-r--r-- | benchmark/agbenchmark/README.md | 2 | ||||
-rw-r--r-- | benchmark/agbenchmark/config.py | 16 | ||||
-rw-r--r-- | benchmark/agbenchmark/reports/ReportManager.py | 2 | ||||
-rw-r--r-- | benchmark/agbenchmark/utils/utils.py | 1 |
6 files changed, 16 insertions, 9 deletions
diff --git a/.github/workflows/autogpts-ci.yml b/.github/workflows/autogpts-ci.yml index aab7a73f2..5268755c2 100644 --- a/.github/workflows/autogpts-ci.yml +++ b/.github/workflows/autogpts-ci.yml @@ -68,4 +68,4 @@ jobs: REQUESTS_CA_BUNDLE: /etc/ssl/certs/ca-certificates.crt HELICONE_CACHE_ENABLED: false HELICONE_PROPERTY_AGENT: ${{ matrix.agent-name }} - REPORT_LOCATION: ${{ format('../../reports/{0}', matrix.agent-name) }} + REPORTS_FOLDER: ${{ format('../../reports/{0}', matrix.agent-name) }} diff --git a/benchmark/.env.example b/benchmark/.env.example index 2b08b7906..e4fb03486 100644 --- a/benchmark/.env.example +++ b/benchmark/.env.example @@ -1,4 +1,4 @@ AGENT_NAME=mini-agi -REPORT_LOCATION="reports/mini-agi" +REPORTS_FOLDER="reports/mini-agi" OPENAI_API_KEY="sk-" # for LLM eval BUILD_SKILL_TREE=false # set to true to build the skill tree. diff --git a/benchmark/agbenchmark/README.md b/benchmark/agbenchmark/README.md index 7bc493a63..ee35e0d1d 100644 --- a/benchmark/agbenchmark/README.md +++ b/benchmark/agbenchmark/README.md @@ -30,7 +30,7 @@ 1. Navigate to `auto-gpt-benchmarks/agent/mini-agi` 2. `pip install -r requirements.txt` 3. `cp .env_example .env`, set `PROMPT_USER=false` and add your `OPENAI_API_KEY=`. Sset `MODEL="gpt-3.5-turbo"` if you don't have access to `gpt-4` yet. Also make sure you have Python 3.10^ installed -4. set `AGENT_NAME=mini-agi` in `.env` file and where you want your `REPORT_LOCATION` to be +4. set `AGENT_NAME=mini-agi` in `.env` file and where you want your `REPORTS_FOLDER` to be 5. Make sure to follow the commands above, and remove mock flag `agbenchmark` - To add requirements `poetry add requirement`. diff --git a/benchmark/agbenchmark/config.py b/benchmark/agbenchmark/config.py index 7605b86b5..ec8974bea 100644 --- a/benchmark/agbenchmark/config.py +++ b/benchmark/agbenchmark/config.py @@ -4,7 +4,7 @@ from datetime import datetime from pathlib import Path from typing import Optional -from pydantic import BaseSettings, Field +from pydantic import BaseSettings, Field, validator def _calculate_info_test_path(base_path: Path, benchmark_start_time: datetime) -> Path: @@ -66,6 +66,12 @@ class AgentBenchmarkConfig(BaseSettings, extra="allow"): host: str """Host (scheme://address:port) of the subject agent application.""" + reports_folder: Path = Field(None) + """ + Path to the folder where new reports should be stored. + Defaults to {agbenchmark_config_dir}/reports. + """ + @classmethod def load(cls, config_dir: Optional[Path] = None) -> "AgentBenchmarkConfig": config_dir = config_dir or cls.find_config_folder() @@ -95,9 +101,11 @@ class AgentBenchmarkConfig(BaseSettings, extra="allow"): def config_file(self) -> Path: return self.agbenchmark_config_dir / "config.json" - @property - def reports_folder(self) -> Path: - return self.agbenchmark_config_dir / "reports" + @validator("reports_folder", pre=True, always=True) + def set_reports_folder(cls, v, values): + if not v: + return values["agbenchmark_config_dir"] / "reports" + return v def get_report_dir(self, benchmark_start_time: datetime) -> Path: return _calculate_info_test_path(self.reports_folder, benchmark_start_time) diff --git a/benchmark/agbenchmark/reports/ReportManager.py b/benchmark/agbenchmark/reports/ReportManager.py index d04beee43..24a9893d4 100644 --- a/benchmark/agbenchmark/reports/ReportManager.py +++ b/benchmark/agbenchmark/reports/ReportManager.py @@ -153,7 +153,7 @@ class SessionReportManager(BaseReportManager): total_cost=self.get_total_costs(), ), tests=copy.copy(self.tests), - config=config.dict(exclude_none=True), + config=config.dict(exclude={"reports_folder"}, exclude_none=True), ) agent_categories = get_highest_achieved_difficulty_per_category(self.tests) diff --git a/benchmark/agbenchmark/utils/utils.py b/benchmark/agbenchmark/utils/utils.py index eaa713730..93724de85 100644 --- a/benchmark/agbenchmark/utils/utils.py +++ b/benchmark/agbenchmark/utils/utils.py @@ -14,7 +14,6 @@ from agbenchmark.utils.data_types import DIFFICULTY_MAP, DifficultyLevel load_dotenv() AGENT_NAME = os.getenv("AGENT_NAME") -REPORT_LOCATION = os.getenv("REPORT_LOCATION", None) logger = logging.getLogger(__name__) |