1 files changed, 119 insertions, 0 deletions
diff --git a/benchmark/agbenchmark/config.py b/benchmark/agbenchmark/config.py
new file mode 100644
index 000000000..7605b86b5
--- /dev/null
+++ b/benchmark/agbenchmark/config.py
@@ -0,0 +1,119 @@
+import json
+import sys
+from datetime import datetime
+from pathlib import Path
+from typing import Optional
+
+from pydantic import BaseSettings, Field
+
+
+def _calculate_info_test_path(base_path: Path, benchmark_start_time: datetime) -> Path:
+    """
+    Calculates the path to the directory where the test report will be saved.
+    """
+    # Ensure the reports path exists
+    base_path.mkdir(parents=True, exist_ok=True)
+
+    # Get current UTC date-time stamp
+    date_stamp = benchmark_start_time.strftime("%Y%m%dT%H%M%S")
+
+    # Default run name
+    run_name = "full_run"
+
+    # Map command-line arguments to their respective labels
+    arg_labels = {
+        "--test": None,
+        "--category": None,
+        "--maintain": "maintain",
+        "--improve": "improve",
+        "--explore": "explore",
+    }
+
+    # Identify the relevant command-line argument
+    for arg, label in arg_labels.items():
+        if arg in sys.argv:
+            test_arg = sys.argv[sys.argv.index(arg) + 1] if label is None else None
+            run_name = arg.strip("--")
+            if test_arg:
+                run_name = f"{run_name}_{test_arg}"
+            break
+
+    # Create the full new directory path with ISO standard UTC date-time stamp
+    report_path = base_path / f"{date_stamp}_{run_name}"
+
+    # Ensure the new directory is created
+    # FIXME: this is not a desirable side-effect of loading the config
+    report_path.mkdir(exist_ok=True)
+
+    return report_path
+
+
+class AgentBenchmarkConfig(BaseSettings, extra="allow"):
+    """
+    Configuration model and loader for the AGBenchmark.
+
+    Projects that want to use AGBenchmark should contain an agbenchmark_config folder
+    with a config.json file that - at minimum - specifies the `host` at which the
+    subject application exposes an Agent Protocol compliant API.
+    """
+
+    agbenchmark_config_dir: Path = Field(..., exclude=True)
+    """Path to the agbenchmark_config folder of the subject agent application."""
+
+    categories: list[str] | None = None
+    """Categories to benchmark the agent for. If omitted, all categories are assumed."""
+
+    host: str
+    """Host (scheme://address:port) of the subject agent application."""
+
+    @classmethod
+    def load(cls, config_dir: Optional[Path] = None) -> "AgentBenchmarkConfig":
+        config_dir = config_dir or cls.find_config_folder()
+        with (config_dir / "config.json").open("r") as f:
+            return cls(
+                agbenchmark_config_dir=config_dir,
+                **json.load(f),
+            )
+
+    @staticmethod
+    def find_config_folder(for_dir: Path = Path.cwd()) -> Path:
+        """
+        Find the closest ancestor folder containing an agbenchmark_config folder,
+        and returns the path of that agbenchmark_config folder.
+        """
+        current_directory = for_dir
+        while current_directory != Path("/"):
+            if (path := current_directory / "agbenchmark_config").exists():
+                if (path / "config.json").is_file():
+                    return path
+            current_directory = current_directory.parent
+        raise FileNotFoundError(
+            "No 'agbenchmark_config' directory found in the path hierarchy."
+        )
+
+    @property
+    def config_file(self) -> Path:
+        return self.agbenchmark_config_dir / "config.json"
+
+    @property
+    def reports_folder(self) -> Path:
+        return self.agbenchmark_config_dir / "reports"
+
+    def get_report_dir(self, benchmark_start_time: datetime) -> Path:
+        return _calculate_info_test_path(self.reports_folder, benchmark_start_time)
+
+    @property
+    def regression_tests_file(self) -> Path:
+        return self.reports_folder / "regression_tests.json"
+
+    @property
+    def success_rate_file(self) -> Path:
+        return self.reports_folder / "success_rate.json"
+
+    @property
+    def challenges_already_beaten_file(self) -> Path:
+        return self.agbenchmark_config_dir / "challenges_already_beaten.json"
+
+    @property
+    def temp_folder(self) -> Path:
+        return self.agbenchmark_config_dir / "temp_folder"