diff options
author | Reinier van der Leer <pwuts@agpt.co> | 2024-02-16 17:18:49 +0100 |
---|---|---|
committer | Reinier van der Leer <pwuts@agpt.co> | 2024-02-16 17:20:44 +0100 |
commit | 2a55efb3221600ec68496275022a94ade48ef79c (patch) | |
tree | 43a68ac800113c13913b532e471600d239c70a93 /benchmark | |
parent | feat(benchmark/cli): Add `challenge list`, `challenge info` subcommands (diff) | |
download | Auto-GPT-2a55efb3221600ec68496275022a94ade48ef79c.tar.gz Auto-GPT-2a55efb3221600ec68496275022a94ade48ef79c.tar.bz2 Auto-GPT-2a55efb3221600ec68496275022a94ade48ef79c.zip |
fix(benchmark): Include `WebArenaSiteInfo.additional_info` (e.g. credentials) in task input
Without the `additional_info`, it is impossible to get past the login page on challenges where that is necessary.
Diffstat (limited to 'benchmark')
-rw-r--r-- | benchmark/agbenchmark/challenges/webarena.py | 26 |
1 files changed, 19 insertions, 7 deletions
diff --git a/benchmark/agbenchmark/challenges/webarena.py b/benchmark/agbenchmark/challenges/webarena.py index 09f801089..395b5a6ee 100644 --- a/benchmark/agbenchmark/challenges/webarena.py +++ b/benchmark/agbenchmark/challenges/webarena.py @@ -37,7 +37,8 @@ site_info_map: dict[WebArenaSite, WebArenaSiteInfo] = { base_url="http://git.junglegym.ai", available=bool(_git_user and _git_password), additional_info=( - f"To log in, use the username '{_git_user}' and password '{_git_password}'." + f"To log in to {{url}}, use the username '{_git_user}' " + f"and password '{_git_password}'." ), unavailable_reason=( "WEBARENA_GIT_CREDENTIALS not set (correctly): " @@ -52,16 +53,22 @@ site_info_map: dict[WebArenaSite, WebArenaSiteInfo] = { "shopping": WebArenaSiteInfo(base_url="http://shop.junglegym.ai"), "shopping_admin": WebArenaSiteInfo( base_url="http://cms.junglegym.ai/admin", - additional_info="To log in, use the username 'admin' and password 'admin1234'.", + additional_info=( + "To log in to {url}, use the username 'admin' and password 'admin1234'." + ), ), "wikipedia": WebArenaSiteInfo(base_url="http://wiki.junglegym.ai"), } -def get_site_url(site: WebArenaSite) -> str: +def get_site_info(site: WebArenaSite) -> WebArenaSiteInfo: if site not in site_info_map: raise ValueError(f"JungleGym site '{site}' unknown, cannot resolve URL") - return site_info_map[site].base_url + return site_info_map[site] + + +def get_site_url(site: WebArenaSite) -> str: + return get_site_info(site).base_url def resolve_uri(uri: str) -> str: @@ -245,10 +252,15 @@ class WebArenaChallengeSpec(BaseModel): @property def assignment_for_agent(self): - sites = [get_site_url(s) for s in self.sites] + sites = [get_site_info(s) for s in self.sites] nav_constraint = ( - f"You are ONLY allowed to access URLs in {' and '.join(sites)}." - ) + "You are ONLY allowed to access URLs in " + f"{' and '.join(s.base_url for s in sites)}.\n\n" + + "\n".join( + s.additional_info.format(url=s.base_url) + for s in sites if s.additional_info + ) + ).strip() return ( f"First of all, go to {self.start_url}. " |