diff options
Diffstat (limited to 'benchmark/agbenchmark/challenges/deprecated/retrieval/3_formatting/data.json')
-rw-r--r-- | benchmark/agbenchmark/challenges/deprecated/retrieval/3_formatting/data.json | 30 |
1 files changed, 30 insertions, 0 deletions
diff --git a/benchmark/agbenchmark/challenges/deprecated/retrieval/3_formatting/data.json b/benchmark/agbenchmark/challenges/deprecated/retrieval/3_formatting/data.json new file mode 100644 index 000000000..e3720860b --- /dev/null +++ b/benchmark/agbenchmark/challenges/deprecated/retrieval/3_formatting/data.json @@ -0,0 +1,30 @@ +{ + "category": [ + "scrape_synthesize" + ], + "cutoff": 60, + "dependencies": [ + "TestRevenueRetrieval1.1" + ], + "eval_id": "e92b86ba-63f3-4322-8f98-4970190d1e69", + "ground": { + "answer": "It was $81.462 billion in 2022. In millions the answer is 81,462.", + "eval": { + "type": "file" + }, + "files": [ + ".txt" + ], + "should_contain": [ + "81,462" + ], + "should_not_contain": [] + }, + "info": { + "description": "Advanced version of the r2.1 challenge that also asks for specific formatting.", + "difficulty": "intermediate", + "side_effects": [] + }, + "name": "RevenueRetrieval1.2", + "task": "Write tesla's exact revenue in 2022 into a .txt file. Use the US notation, with a precision rounded to the nearest million dollars (for instance, $31,578 billion)." +} |