blob: ac7444921890f655a72ce5100ca00f06b97b4226 (
plain)
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
|
from enum import Enum
from typing import Literal
from pydantic import BaseModel
class DifficultyLevel(Enum):
interface = "interface"
basic = "basic"
novice = "novice"
intermediate = "intermediate"
advanced = "advanced"
expert = "expert"
human = "human"
# map from enum to difficulty level (numeric)
DIFFICULTY_MAP = {
DifficultyLevel.interface: 1,
DifficultyLevel.basic: 2,
DifficultyLevel.novice: 3,
DifficultyLevel.intermediate: 4,
DifficultyLevel.advanced: 5,
DifficultyLevel.expert: 6,
DifficultyLevel.human: 7,
}
STRING_DIFFICULTY_MAP = {e.value: DIFFICULTY_MAP[e] for e in DifficultyLevel}
class Category(str, Enum):
GENERALIST = "general"
DATA = "data"
CODING = "coding"
SCRAPE_SYNTHESIZE = "scrape_synthesize"
WEB = "web"
GAIA_1 = "GAIA_1"
GAIA_2 = "GAIA_2"
GAIA_3 = "GAIA_3"
class EvalResult(BaseModel):
result: str
result_source: Literal["step_output"] | str
score: float
passed: bool
|