aboutsummaryrefslogtreecommitdiff
path: root/benchmark/agbenchmark/reports/processing/report_types_v2.py
diff options
context:
space:
mode:
Diffstat (limited to 'benchmark/agbenchmark/reports/processing/report_types_v2.py')
-rw-r--r--benchmark/agbenchmark/reports/processing/report_types_v2.py46
1 files changed, 21 insertions, 25 deletions
diff --git a/benchmark/agbenchmark/reports/processing/report_types_v2.py b/benchmark/agbenchmark/reports/processing/report_types_v2.py
index 94b53ca97..b26adaa6d 100644
--- a/benchmark/agbenchmark/reports/processing/report_types_v2.py
+++ b/benchmark/agbenchmark/reports/processing/report_types_v2.py
@@ -1,52 +1,48 @@
-from typing import Dict, List
+"""Model definitions for use in the API"""
-datetime_format = r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+00:00$"
from pydantic import BaseModel, constr
-
-class BaseModelBenchmark(BaseModel):
- class Config:
- extra = "forbid"
+datetime_format = r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+00:00$"
-class TaskInfo(BaseModelBenchmark):
+class TaskInfo(BaseModel):
data_path: str
is_regression: bool | None
answer: str
description: str
- category: List[str]
+ category: list[str]
task: str
-class RepositoryInfo(BaseModelBenchmark):
- repo_url: str | None
- team_name: str | None
- benchmark_git_commit_sha: str | None
- agent_git_commit_sha: str | None
+class RepositoryInfo(BaseModel):
+ repo_url: str | None = None
+ team_name: str | None = None
+ agent_git_commit_sha: str | None = None
+ benchmark_git_commit_sha: str | None = None
-class Metrics(BaseModelBenchmark):
- difficulty: str | None
+class Metrics(BaseModel):
+ cost: float | None = None
success: bool
- success_percentage: float | None
- run_time: str | None
- fail_reason: str | None
attempted: bool
- cost: float | None
+ difficulty: str | None = None
+ run_time: str | None = None
+ fail_reason: str | None = None
+ success_percentage: float | None = None
-class RunDetails(BaseModelBenchmark):
+class RunDetails(BaseModel):
test_name: str
- run_id: str | None
+ run_id: str | None = None
command: str
- completion_time: str | None
+ completion_time: str | None = None
benchmark_start_time: constr(regex=datetime_format)
-class BenchmarkRun(BaseModelBenchmark):
+class BenchmarkRun(BaseModel):
repository_info: RepositoryInfo
run_details: RunDetails
task_info: TaskInfo
metrics: Metrics
- reached_cutoff: bool | None
- config: Dict[str, str | dict[str, str]]
+ reached_cutoff: bool | None = None
+ config: dict[str, str | dict[str, str]]