blob: b26adaa6d47de97fc5df997c564fb98073249b9b (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
|
"""Model definitions for use in the API"""
from pydantic import BaseModel, constr
datetime_format = r"^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}\+00:00$"
class TaskInfo(BaseModel):
data_path: str
is_regression: bool | None
answer: str
description: str
category: list[str]
task: str
class RepositoryInfo(BaseModel):
repo_url: str | None = None
team_name: str | None = None
agent_git_commit_sha: str | None = None
benchmark_git_commit_sha: str | None = None
class Metrics(BaseModel):
cost: float | None = None
success: bool
attempted: bool
difficulty: str | None = None
run_time: str | None = None
fail_reason: str | None = None
success_percentage: float | None = None
class RunDetails(BaseModel):
test_name: str
run_id: str | None = None
command: str
completion_time: str | None = None
benchmark_start_time: constr(regex=datetime_format)
class BenchmarkRun(BaseModel):
repository_info: RepositoryInfo
run_details: RunDetails
task_info: TaskInfo
metrics: Metrics
reached_cutoff: bool | None = None
config: dict[str, str | dict[str, str]]
|