path: root/benchmark/notebooks/LLM Score Experimentation.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 28,
   "id": "15386da8",
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "import os\n",
    "from dotenv import load_dotenv\n",
    "\n",
    "load_dotenv()\n",
    "\n",
    "OPENAI_API_KEY = os.getenv(\"OPENAI_API_KEY\")\n",
    "\n",
    "def llm_eval(evaluation: str) -> float:\n",
    "    openai.api_key = OPENAI_API_KEY\n",
    "    answer = openai.ChatCompletion.create(\n",
    "        model=\"gpt-4\",\n",
    "        messages=[\n",
    "            {\"role\": \"system\", \"content\": evaluation},\n",
    "        ],\n",
    "    )\n",
    "    return answer[\"choices\"][0][\"message\"][\"content\"]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "id": "e9d1c3a6",
   "metadata": {},
   "outputs": [],
   "source": [
    "REFERENCE_PROMPT = \"\"\"Ignore previous directions. You are now an expert at evaluating how close machine generated responses are to human answers. You essentially act as a hyper advanced BLEU score.\n",
    "In order to score the machine generated response you will {scoring}. Make sure to factor in the distance to the ideal response into your thinking, deliberation, and final result regarding scoring. Return nothing but a float score.\n",
    "\n",
    "Here is the given task for you to evaluate:\n",
    "{task}\n",
    "\n",
    "Here is the ideal response you're comparing to based on the task:\n",
    "{answer}\n",
    "\n",
    "Here are some examples of how to score a machine generated response compared to the above ideal response:\n",
    "{examples}\n",
    "\n",
    "Here is the current machine generated response to the task that you need to evaluate:\n",
    "{response}\n",
    "\n",
    "Remember to always end your response with nothing but a float score.\n",
    "Float score:\n",
    "\"\"\"\n",
    "\n",
    "RUBRIC_PROMPT = \"\"\"Ignore previous directions. You are now an expert at evaluating machine generated responses to given tasks.\n",
    "In order to score the generated texts you will {scoring}. Make sure to factor in rubric into your thinking, deliberation, and final result regarding scoring. Return nothing but a float score.\n",
    "\n",
    "Here is the given task for you to evaluate:\n",
    "{task}\n",
    "\n",
    "Use the below rubric to guide your thinking about scoring:\n",
    "{answer}\n",
    "\n",
    "Here are some examples of how to score a machine generated response based on the above rubric.\n",
    "{examples}\n",
    "\n",
    "Here is the current machine generated response to the task that you need to evaluate:\n",
    "{response}\n",
    "\n",
    "Remember to always end your response with nothing but a float score.\n",
    "Float score:\n",
    "\"\"\"\n",
    "\n",
    "QUESTION_PROMPT = \"\"\"Ignore previous directions. You are now an expert at evaluating machine generated responses to given tasks.\n",
    "In order to score the generated texts you will {scoring}. Make sure to think about whether the generated response answers the question well in order to score accurately. Return nothing but a float score.\n",
    "\n",
    "Here is the given task:\n",
    "{task}\n",
    "\n",
    "Here is a question that checks if the task was completed correctly:\n",
    "{answer}\n",
    "\n",
    "Here are some examples of how to score a machine generated response based on task-response pairs.\n",
    "{examples}\n",
    "\n",
    "Here is the current machine generated response to the task that you need to evaluate:\n",
    "{response}\n",
    "\n",
    "Remember to always end your response with nothing but a float score.\n",
    "Float score:\n",
    "\"\"\"\n",
    "\n",
    "CUSTOM_PROMPT = \"\"\"{custom}\n",
    "{scoring}\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "id": "8ff9cd33",
   "metadata": {},
   "outputs": [],
   "source": [
    "task = \"As an agent powered by a large language model, you can execute any digital action. Your task is to find the cheapest flight to San Francisco from Toronto. Create a step by step plan for yourself to execute this task and output this plan into a .txt file.\"\n",
    "\n",
    "percentage = \"assign a float score that will represent a percentage out of 100. Use decimal points to be even more accurate. 0 represents the worst possible generation, while 100 represents the ideal generation\"\n",
    "scale = \"assign an integer score from a scale of 1-10. 1 represents a really bad generation, while 10 represents an ideal generation\"\n",
    "binary = \"assign a binary score of either 0 or 1. 0 represents a failure, while 1 represents a success\"\n",
    "\n",
    "reference_answer = \"\"\"1. Search 'Toronto to San Francisco flights' on the internet\n",
    "2. Click on the first link that is a flight aggregator such as SkyScanner or Google Flights\n",
    "3. Select the dates that you want to travel\n",
    "3. Click on the 'one way' option and click search\n",
    "4. Search through all of the given options and select the cheapest flight\"\"\"\n",
    "\n",
    "rubric_answer = \"\"\"- Reasonable: The plan's steps should be arranged in a logical sequence. Each step should naturally follow from the one before it, leading coherently towards the desired outcome.\n",
    "- Atomic: Each step of the plan should be broken down into the smallest possible units of action. This level of detail allows for better tracking and control, as well as easier troubleshooting and adjustment of the plan if necessary.\n",
    "- Relevant: Each step should directly contribute to achieving the end goal of the plan. Irrelevant steps can waste resources and detract from the main objectives.\n",
    "- Timely: The plan should include a realistic timeline for each step, and not include any redundant steps.\"\"\"\n",
    "\n",
    "question_answer = \"Will this plan find the cheapest flight to San Francisco from Toronto consistently?\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "id": "d0bfdce5",
   "metadata": {},
   "outputs": [],
   "source": [
    "FEW_SHOT_REFERENCE_EXAMPLES = \"\"\"EXAMPLE 1\n",
    "Plan: 'Define the Parameters:\n",
    "\n",
    "Identify the potential range of dates for the journey. Ensure flexibility to capitalize on the best deals.\n",
    "Research Airlines:\n",
    "\n",
    "Compile a list of airlines that operate flights between Toronto and San Francisco. This can be accomplished by checking each airline's website or using an airline directory.\n",
    "Use Comparison Tools:\n",
    "\n",
    "Use flight comparison websites (such as Skyscanner, Expedia, Google Flights, etc.) to compare flight prices across different airlines. These tools aggregate flight data and can present the cheapest options available.\n",
    "Set Price Alerts:\n",
    "\n",
    "If the trip is not immediate, set price alerts on these comparison websites. They will notify you when there is a significant drop in flight prices on the selected route.\n",
    "Check Airlines' Official Websites:\n",
    "\n",
    "After identifying potential flights via comparison tools, visit the airlines' official websites. Sometimes airlines offer deals or discounts on their own websites that are not reflected on comparison sites.\n",
    "Consider Nearby Airports:\n",
    "\n",
    "Check flight prices to and from airports in the vicinity of both Toronto and San Francisco. Sometimes less busy airports can offer cheaper flights.\n",
    "Evaluate Cost Effectiveness:\n",
    "\n",
    "Consider factors such as baggage fees, meal costs, and transportation to and from the airport when evaluating the total cost of the flight. The cheapest ticket price does not necessarily mean the least expensive journey overall.\n",
    "Book the Flight:\n",
    "\n",
    "Once the cheapest and most convenient flight has been identified, proceed to booking. Double-check the flight details before finalizing the booking.\n",
    "Monitor Flight Details:\n",
    "\n",
    "After booking, keep an eye on flight status, gate information, and any potential changes to the flight schedule.'\n",
    "Returned score: 82.7\n",
    "Internal rationale: The plan is solid, however the plan goes to extreme lengths to make things cheap, sacrificing time and simplicity. The task just asks for a vague definition of booking a flight. There are some redundant steps.\n",
    "\n",
    "EXAMPLE 2\n",
    "Plan: 'Determine the Travel Dates and Flexibility: Decide on the dates you want to travel to San Francisco and check if you have any flexibility in your travel schedule. Being flexible with your travel dates can often lead to finding cheaper flights.\n",
    "\n",
    "Use Flight Search Engines: Start by using popular flight search engines like Google Flights, Skyscanner, Kayak, or Expedia. These platforms allow you to compare prices from various airlines and find the most affordable options.\n",
    "\n",
    "Set Up Fare Alerts: If your travel dates are flexible, consider setting up fare alerts on the flight search engines. These alerts will notify you when the prices drop for the specified route.\n",
    "\n",
    "Check Nearby Airports: In both Toronto and San Francisco, there might be multiple airports. Check flights departing from nearby airports as they may offer better deals.\n",
    "\n",
    "Consider Layovers: Non-stop flights are usually more convenient but can be more expensive. Look for flights with one or more layovers as they may offer cost savings.\n",
    "\n",
    "Check Airlines' Official Websites: Once you find a potentially cheap flight on a search engine, verify the price directly on the airline's official website. Sometimes, booking directly with the airline can be cheaper due to exclusive deals and promotions.\n",
    "\n",
    "Use Incognito/Private Browsing Mode: Flight prices can sometimes increase if the website detects repeated searches for the same route. To avoid this, use the incognito or private browsing mode in your web browser.\n",
    "\n",
    "Consider Budget Airlines: Check if there are any budget airlines flying between Toronto and San Francisco. They often offer lower fares, but be mindful of additional fees for baggage and other services.\n",
    "\n",
    "Check for Deals and Promo Codes: Look for any ongoing deals or promo codes that can help you save on your flight booking. Airlines and travel websites occasionally offer special discounts.\n",
    "\n",
    "Be Flexible with Departure and Arrival Times: If possible, consider flying during off-peak hours or mid-week, as flights during these times can be less expensive.\n",
    "\n",
    "Factor in Total Costs: While searching for cheap flights, don't forget to consider other expenses like baggage fees, seat selection, and additional amenities. Some budget airlines might have hidden costs that could make the overall trip more expensive.\n",
    "\n",
    "Book Early: Flight prices tend to rise as the departure date approaches. Once you find a good deal that suits your preferences, don't wait too long to book your flight.'\n",
    "Returned score: 74.9\n",
    "Internal rationale: The individual components of this plan are better than the one previous. But this plan doesn't follow logical steps to completion, and is just more general advice.\n",
    "\n",
    "EXAMPLE 3\n",
    "Plan: 'Search online for cheap flights.\n",
    "Check different dates.\n",
    "Look at nearby airports.\n",
    "Consider layovers.\n",
    "Try budget airlines.\n",
    "Book early if you find a good deal.'\n",
    "Returned score: 42.0\n",
    "Internal rationale: This plan is too vague and does not provide enough detail to be useful.\n",
    "\n",
    "\"\"\"\n",
    "FEW_SHOT_RUBRIC_EXAMPLES = \"\"\n",
    "FEW_SHOT_QUESTION_EXAMPLES = \"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 75,
   "id": "3de1d6d4",
   "metadata": {},
   "outputs": [],
   "source": [
    "response1=\"\"\"Define the Parameters:\n",
    "\n",
    "Identify the potential range of dates for the journey. Ensure flexibility to capitalize on the best deals.\n",
    "Research Airlines:\n",
    "\n",
    "Compile a list of airlines that operate flights between Toronto and San Francisco. This can be accomplished by checking each airline's website or using an airline directory.\n",
    "Use Comparison Tools:\n",
    "\n",
    "Use flight comparison websites (such as Skyscanner, Expedia, Google Flights, etc.) to compare flight prices across different airlines. These tools aggregate flight data and can present the cheapest options available.\n",
    "Set Price Alerts:\n",
    "\n",
    "If the trip is not immediate, set price alerts on these comparison websites. They will notify you when there is a significant drop in flight prices on the selected route.\n",
    "Check Airlines' Official Websites:\n",
    "\n",
    "After identifying potential flights via comparison tools, visit the airlines' official websites. Sometimes airlines offer deals or discounts on their own websites that are not reflected on comparison sites.\n",
    "Consider Nearby Airports:\n",
    "\n",
    "Check flight prices to and from airports in the vicinity of both Toronto and San Francisco. Sometimes less busy airports can offer cheaper flights.\n",
    "Evaluate Cost Effectiveness:\n",
    "\n",
    "Consider factors such as baggage fees, meal costs, and transportation to and from the airport when evaluating the total cost of the flight. The cheapest ticket price does not necessarily mean the least expensive journey overall.\n",
    "Book the Flight:\n",
    "\n",
    "Once the cheapest and most convenient flight has been identified, proceed to booking. Double-check the flight details before finalizing the booking.\n",
    "Monitor Flight Details:\n",
    "\n",
    "After booking, keep an eye on flight status, gate information, and any potential changes to the flight schedule.\"\"\"\n",
    "\n",
    "response2=\"\"\"Determine the Travel Dates and Flexibility: Decide on the dates you want to travel to San Francisco and check if you have any flexibility in your travel schedule. Being flexible with your travel dates can often lead to finding cheaper flights.\n",
    "\n",
    "Use Flight Search Engines: Start by using popular flight search engines like Google Flights, Skyscanner, Kayak, or Expedia. These platforms allow you to compare prices from various airlines and find the most affordable options.\n",
    "\n",
    "Set Up Fare Alerts: If your travel dates are flexible, consider setting up fare alerts on the flight search engines. These alerts will notify you when the prices drop for the specified route.\n",
    "\n",
    "Check Nearby Airports: In both Toronto and San Francisco, there might be multiple airports. Check flights departing from nearby airports as they may offer better deals.\n",
    "\n",
    "Consider Layovers: Non-stop flights are usually more convenient but can be more expensive. Look for flights with one or more layovers as they may offer cost savings.\n",
    "\n",
    "Check Airlines' Official Websites: Once you find a potentially cheap flight on a search engine, verify the price directly on the airline's official website. Sometimes, booking directly with the airline can be cheaper due to exclusive deals and promotions.\n",
    "\n",
    "Use Incognito/Private Browsing Mode: Flight prices can sometimes increase if the website detects repeated searches for the same route. To avoid this, use the incognito or private browsing mode in your web browser.\n",
    "\n",
    "Consider Budget Airlines: Check if there are any budget airlines flying between Toronto and San Francisco. They often offer lower fares, but be mindful of additional fees for baggage and other services.\n",
    "\n",
    "Check for Deals and Promo Codes: Look for any ongoing deals or promo codes that can help you save on your flight booking. Airlines and travel websites occasionally offer special discounts.\n",
    "\n",
    "Be Flexible with Departure and Arrival Times: If possible, consider flying during off-peak hours or mid-week, as flights during these times can be less expensive.\n",
    "\n",
    "Factor in Total Costs: While searching for cheap flights, don't forget to consider other expenses like baggage fees, seat selection, and additional amenities. Some budget airlines might have hidden costs that could make the overall trip more expensive.\n",
    "\n",
    "Book Early: Flight prices tend to rise as the departure date approaches. Once you find a good deal that suits your preferences, don't wait too long to book your flight.\"\"\"\n",
    "\n",
    "response3 = \"\"\"Search online for cheap flights.\n",
    "Check different dates.\n",
    "Look at nearby airports.\n",
    "Consider layovers.\n",
    "Try budget airlines.\n",
    "Book early if you find a good deal.\"\"\"\n",
    "\n",
    "ideal_response = \"\"\"1. Search 'Toronto to San Francisco flights' on the internet\n",
    "2. Click on the first link that is a flight aggregator such as SkyScanner or Google Flights\n",
    "3. Select the dates that you want to travel\n",
    "3. Click on the 'one way' option and click search\n",
    "4. Search through all of the given options and select the cheapest flight\"\"\""
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "a5bf2f5c",
   "metadata": {},
   "outputs": [],
   "source": [
    "reference_evaluation = REFERENCE_PROMPT.format(task=task, scoring=percentage, answer=reference_answer, response=ideal_response, examples=FEW_SHOT_REFERENCE_EXAMPLES)\n",
    "rubric_evaluation = RUBRIC_PROMPT.format(task=task, scoring=percentage, answer=rubric_answer, response=ideal_response, examples=FEW_SHOT_REFERENCE_EXAMPLES)\n",
    "question_evaluation = QUESTION_PROMPT.format(task=task, scoring=percentage, answer=question_answer, response=ideal_response, examples=FEW_SHOT_REFERENCE_EXAMPLES)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 82,
   "id": "1cd01a4c",
   "metadata": {},
   "outputs": [],
   "source": [
    "reference_response = llm_eval(reference_evaluation)\n",
    "rubric_response = llm_eval(rubric_evaluation)\n",
    "question_response = llm_eval(question_evaluation)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 72,
   "id": "72e6ac91",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "REFERENCE: 66.5 6.0 0.0\n",
      "RUBRIC: 65.0 4.0 0.0\n",
      "QUESTION: 87.0 6.5 0.0\n"
     ]
    }
   ],
   "source": [
    "# response3 - the worst one\n",
    "print('REFERENCE:', reference_response_percentage, reference_response_scale, reference_response_binary)\n",
    "print('RUBRIC:', rubric_response_percentage, rubric_response_scale, rubric_response_binary)\n",
    "print('QUESTION:', question_response_percentage, question_response_scale, question_response_binary)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 83,
   "id": "a66014ee",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "REFERENCE: 100.0 10.0 1.0\n",
      "RUBRIC: 88.0 7.5 1.0\n",
      "QUESTION: 87.0 7.5 1.0\n"
     ]
    }
   ],
   "source": [
    "# reference_response - the best one\n",
    "print('REFERENCE:', reference_response_ideal_percentage, reference_response_ideal_scale, reference_response_ideal_binary)\n",
    "print('RUBRIC:', rubric_response_ideal_percentage, rubric_response_ideal_scale, rubric_response_ideal_binary)\n",
    "print('QUESTION:', question_response_ideal_percentage, question_response_ideal_scale, question_response_ideal_binary)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.11.4"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}