benchmark_v2.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>url</th>\n",
       "      <th>html</th>\n",
       "      <th>query</th>\n",
       "      <th>node</th>\n",
       "      <th>code</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>https://github.com</td>\n",
       "      <td>&lt;html lang=\"en\" data-a11y-animated-images=\"sys...</td>\n",
       "      <td>click on start a free entreprise trial</td>\n",
       "      <td>&lt;a class=\"btn-mktg home-campaign-enterprise bt...</td>\n",
       "      <td># Let's proceed step by step.\\n# First we need...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>https://huggingface.co/</td>\n",
       "      <td>&lt;html class=\"\"&gt;&lt;head&gt;\\n\\t\\t&lt;meta charset=\"utf-...</td>\n",
       "      <td>click on datasets</td>\n",
       "      <td>&lt;li&gt;&lt;a class=\"group flex items-center px-2 py-...</td>\n",
       "      <td># Let's proceed step by step.\\n# First we need...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>https://www.irs.gov</td>\n",
       "      <td>&lt;html lang=\"en\" dir=\"ltr\" prefix=\"content: htt...</td>\n",
       "      <td>click on criminal investigation</td>\n",
       "      <td>&lt;/ul&gt;\\n  \\n              &lt;/li&gt;\\n              ...</td>\n",
       "      <td># Let's proceed step by step.\\n# First we need...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>https://www.kaggle.com</td>\n",
       "      <td>&lt;html lang=\"en\"&gt;&lt;head&gt;&lt;meta http-equiv=\"origin...</td>\n",
       "      <td>click on competitions</td>\n",
       "      <td>&lt;div class=\"sc-jiSpbx kcFuZv\"&gt;&lt;div class=\"sc-h...</td>\n",
       "      <td># Let's proceed step by step.\\n# First we need...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>https://en.wikipedia.org/wiki/Main_Page</td>\n",
       "      <td>&lt;html class=\"client-js vector-feature-language...</td>\n",
       "      <td>click on create account</td>\n",
       "      <td>&lt;div id=\"p-vector-user-menu-overflow\" class=\"v...</td>\n",
       "      <td># Let's proceed step by step.\\n# First we need...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5</th>\n",
       "      <td>https://www.google.fr/</td>\n",
       "      <td>&lt;html itemscope=\"\" itemtype=\"http://schema.org...</td>\n",
       "      <td>click on gmail</td>\n",
       "      <td>&lt;div class=\"gb_Id gb_J gb_3f gb_Tf\" data-ogbl=...</td>\n",
       "      <td># Let's proceed step by step.\\n# First we need...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>6</th>\n",
       "      <td>https://www.salesforce.com/</td>\n",
       "      <td>&lt;html lang=\"en\" style=\"--xsf-chat-agent-image:...</td>\n",
       "      <td>click on start free trial</td>\n",
       "      <td>&lt;pbc-button class=\"cta_button__wrapper\" data-l...</td>\n",
       "      <td># Let's proceed step by step.\\n# First we need...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7</th>\n",
       "      <td>https://www.ldlc.com</td>\n",
       "      <td>&lt;html class=\"js cssanimations csstransitions p...</td>\n",
       "      <td>click on configurateur pc</td>\n",
       "      <td>&lt;li&gt;&lt;a href=\"https://www.ldlc.com/configurateu...</td>\n",
       "      <td># Let's proceed step by step.\\n# First we need...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>8</th>\n",
       "      <td>https://www.semrush.com/</td>\n",
       "      <td>&lt;html lang=\"en\"&gt;&lt;head&gt;\\n        &lt;meta charset=...</td>\n",
       "      <td>click on search bar enter domain</td>\n",
       "      <td>&lt;div class=\"index-search__input\"&gt;&lt;input type=\"...</td>\n",
       "      <td># Let's proceed step by step.\\n# First we need...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9</th>\n",
       "      <td>https://www.hubspot.com</td>\n",
       "      <td>&lt;html lang=\"en\"&gt;&lt;head&gt;\\n    &lt;meta charset=\"utf...</td>\n",
       "      <td>click on get started free</td>\n",
       "      <td>&lt;a class=\"\\n  cl-button -secondary -large wf-p...</td>\n",
       "      <td># Let's proceed step by step.\\n# First we need...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                       url  \\\n",
       "0                       https://github.com   \n",
       "1                  https://huggingface.co/   \n",
       "2                      https://www.irs.gov   \n",
       "3                   https://www.kaggle.com   \n",
       "4  https://en.wikipedia.org/wiki/Main_Page   \n",
       "5                   https://www.google.fr/   \n",
       "6              https://www.salesforce.com/   \n",
       "7                     https://www.ldlc.com   \n",
       "8                 https://www.semrush.com/   \n",
       "9                  https://www.hubspot.com   \n",
       "\n",
       "                                                html  \\\n",
       "0  <html lang=\"en\" data-a11y-animated-images=\"sys...   \n",
       "1  <html class=\"\"><head>\\n\\t\\t<meta charset=\"utf-...   \n",
       "2  <html lang=\"en\" dir=\"ltr\" prefix=\"content: htt...   \n",
       "3  <html lang=\"en\"><head><meta http-equiv=\"origin...   \n",
       "4  <html class=\"client-js vector-feature-language...   \n",
       "5  <html itemscope=\"\" itemtype=\"http://schema.org...   \n",
       "6  <html lang=\"en\" style=\"--xsf-chat-agent-image:...   \n",
       "7  <html class=\"js cssanimations csstransitions p...   \n",
       "8  <html lang=\"en\"><head>\\n        <meta charset=...   \n",
       "9  <html lang=\"en\"><head>\\n    <meta charset=\"utf...   \n",
       "\n",
       "                                    query  \\\n",
       "0  click on start a free entreprise trial   \n",
       "1                       click on datasets   \n",
       "2         click on criminal investigation   \n",
       "3                   click on competitions   \n",
       "4                 click on create account   \n",
       "5                          click on gmail   \n",
       "6               click on start free trial   \n",
       "7               click on configurateur pc   \n",
       "8        click on search bar enter domain   \n",
       "9               click on get started free   \n",
       "\n",
       "                                                node  \\\n",
       "0  <a class=\"btn-mktg home-campaign-enterprise bt...   \n",
       "1  <li><a class=\"group flex items-center px-2 py-...   \n",
       "2  </ul>\\n  \\n              </li>\\n              ...   \n",
       "3  <div class=\"sc-jiSpbx kcFuZv\"><div class=\"sc-h...   \n",
       "4  <div id=\"p-vector-user-menu-overflow\" class=\"v...   \n",
       "5  <div class=\"gb_Id gb_J gb_3f gb_Tf\" data-ogbl=...   \n",
       "6  <pbc-button class=\"cta_button__wrapper\" data-l...   \n",
       "7  <li><a href=\"https://www.ldlc.com/configurateu...   \n",
       "8  <div class=\"index-search__input\"><input type=\"...   \n",
       "9  <a class=\"\\n  cl-button -secondary -large wf-p...   \n",
       "\n",
       "                                                code  \n",
       "0  # Let's proceed step by step.\\n# First we need...  \n",
       "1  # Let's proceed step by step.\\n# First we need...  \n",
       "2  # Let's proceed step by step.\\n# First we need...  \n",
       "3  # Let's proceed step by step.\\n# First we need...  \n",
       "4  # Let's proceed step by step.\\n# First we need...  \n",
       "5  # Let's proceed step by step.\\n# First we need...  \n",
       "6  # Let's proceed step by step.\\n# First we need...  \n",
       "7  # Let's proceed step by step.\\n# First we need...  \n",
       "8  # Let's proceed step by step.\\n# First we need...  \n",
       "9  # Let's proceed step by step.\\n# First we need...  "
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import pandas as pd\n",
    "\n",
    "df = pd.read_csv(\"dataset.csv\")\n",
    "df.head(10)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [],
   "source": [
    "from selenium import webdriver\n",
    "from selenium.webdriver.chrome.service import Service\n",
    "from selenium.webdriver.common.by import By\n",
    "from selenium.webdriver.chrome.options import Options\n",
    "from selenium.webdriver.common.keys import Keys\n",
    "import os.path\n",
    "\n",
    "chrome_options = Options()\n",
    "chrome_options.add_argument(\"--headless\") # Ensure GUI is off\n",
    "chrome_options.add_argument(\"--no-sandbox\")\n",
    "\n",
    "homedir = os.path.expanduser(\"~\")\n",
    "chrome_options.binary_location = f\"{homedir}/chrome-linux64/chrome\"\n",
    "webdriver_service = Service(f\"{homedir}/chromedriver-linux64/chromedriver\")\n",
    "\n",
    "driver = webdriver.Chrome(service=webdriver_service, options=chrome_options)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "import ast\n",
    "from selenium.webdriver.remote.webelement import WebElement\n",
    "\n",
    "import re\n",
    "\n",
    "def keep_assignments(code_snippet):\n",
    "    # Regex to match variable assignments. This pattern assumes variable names are valid Python identifiers\n",
    "    # and captures typical assignment statements, excluding those that might appear in comments or strings.\n",
    "    pattern = r'^\\s*[a-zA-Z_][a-zA-Z0-9_]*\\s*=\\s*.+'\n",
    "\n",
    "    # Filter and keep only lines with variable assignments\n",
    "    assignments = [line for line in code_snippet.split('\\n') if re.match(pattern, line)]\n",
    "\n",
    "    # Join the filtered lines back into a string\n",
    "    return \"\\n\".join(assignments)\n",
    "\n",
    "# This function will be used to visit each node in the AST\n",
    "class VariableVisitor(ast.NodeVisitor):\n",
    "    def __init__(self):\n",
    "        super().__init__()\n",
    "        self.output = []\n",
    "    \n",
    "    def visit_Assign(self, node):\n",
    "        \n",
    "        # For each assignment, print the targets (variables)\n",
    "        for target in node.targets:\n",
    "            if isinstance(target, ast.Name):  # Ensure it's a variable assignment\n",
    "                self.output.append(target.id)\n",
    "def get_web_elements(code_snippet):\n",
    "    # Parse the code snippet into an AST\n",
    "    parsed_code = ast.parse(code_snippet)\n",
    "\n",
    "    # Create a visitor instance and use it to visit the nodes in the parsed AST\n",
    "    visitor = VariableVisitor()\n",
    "    visitor.visit(parsed_code)\n",
    "    variables = visitor.output\n",
    "\n",
    "    output = []\n",
    "    variable_type = WebElement\n",
    "    for var_name in variables:\n",
    "            # Use globals() to get the variable from its name since we are in the global scope\n",
    "        var_value = globals().get(var_name, None)\n",
    "        \n",
    "        # Check if the variable exists and its type\n",
    "        if var_value is not None and isinstance(var_value, variable_type):\n",
    "            output.append(var_value)\n",
    "    return output"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:00, ?it/s]"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "10it [00:05,  1.97it/s]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "['<a class=\"btn-mktg home-campaign-enterprise btn-muted-mktg\" data-analytics-event=\"{&quot;category&quot;:&quot;Start a free enterprise trial&quot;,&quot;action&quot;:&quot;click to Start a free enterprise trial&quot;,&quot;label&quot;:&quot;ref_cta:Start a free enterprise trial;&quot;}\" href=\"/organizations/enterprise_plan?ref_cta=Start+a+free+enterprise+trial&amp;ref_loc=Home+campaign+hero&amp;ref_page=%2F\" data-test-selector=\"start-trial-button\">\\n  Start a free enterprise trial\\n  \\n  <svg xmlns=\"http://www.w3.org/2000/svg\" class=\"octicon arrow-symbol-mktg\" width=\"16\" height=\"16\" viewBox=\"0 0 16 16\" fill=\"none\"><path fill=\"currentColor\" d=\"M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z\"></path><path class=\"octicon-chevrow-stem\" stroke=\"currentColor\" d=\"M1.75 8H11\" stroke-width=\"1.5\" stroke-linecap=\"round\"></path></svg>\\n\\n</a>', '<a class=\"group flex items-center px-2 py-0.5 dark:hover:text-gray-400 hover:text-red-700\" href=\"/datasets\"><svg class=\"mr-1.5 text-gray-400 group-hover:text-red-500\" style=\"\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" aria-hidden=\"true\" focusable=\"false\" role=\"img\" width=\"1em\" height=\"1em\" preserveAspectRatio=\"xMidYMid meet\" viewBox=\"0 0 25 25\"><ellipse cx=\"12.5\" cy=\"5\" fill=\"currentColor\" fill-opacity=\"0.25\" rx=\"7.5\" ry=\"2\"></ellipse><path d=\"M12.5 15C16.6421 15 20 14.1046 20 13V20C20 21.1046 16.6421 22 12.5 22C8.35786 22 5 21.1046 5 20V13C5 14.1046 8.35786 15 12.5 15Z\" fill=\"currentColor\" opacity=\"0.5\"></path><path d=\"M12.5 7C16.6421 7 20 6.10457 20 5V11.5C20 12.6046 16.6421 13.5 12.5 13.5C8.35786 13.5 5 12.6046 5 11.5V5C5 6.10457 8.35786 7 12.5 7Z\" fill=\"currentColor\" opacity=\"0.5\"></path><path d=\"M5.23628 12C5.08204 12.1598 5 12.8273 5 13C5 14.1046 8.35786 15 12.5 15C16.6421 15 20 14.1046 20 13C20 12.8273 19.918 12.1598 19.7637 12C18.9311 12.8626 15.9947 13.5 12.5 13.5C9.0053 13.5 6.06886 12.8626 5.23628 12Z\" fill=\"currentColor\"></path></svg> Datasets</a>', '<a href=\"https://www.irs.gov/compliance/criminal-investigation\" data-drupal-link-system-path=\"node/19091\">Criminal Investigation</a>', '<a href=\"/competitions\" data-click-log-id=\"nav-con-competitions\" class=\"sc-jtMbrs bwioyi\">Competitions</a>', '<li id=\"pt-createaccount-2\" class=\"user-links-collapsible-item mw-list-item user-links-collapsible-item\"><a data-mw=\"interface\" href=\"/w/index.php?title=Special:CreateAccount&amp;returnto=Main+Page\" title=\"You are encouraged to create an account and log in; however, it is not mandatory\" class=\"\"><span>Create account</span></a>\\n</li>', '<a class=\"gb_H\" aria-label=\"Gmail (opens a new tab)\" data-pid=\"23\" href=\"https://mail.google.com/mail/&amp;ogbl\" target=\"_top\">Gmail</a>', '<a class=\"cta_button\" data-size=\"\" data-variant=\"primary\" data-stacking=\"default\" aria-label=\"Start free trial: Salesforce Starter Suite\" href=\"https://www.salesforce.com/form/signup/freetrial-salesforce-starter/?d=pb\" target=\"_self\" rel=\"noopener\">\\n        Start free trial\\n        </a>', '<a href=\"https://www.ldlc.com/configurateur/\"><span class=\"icon icon-cat icon-tools\">&nbsp;</span> <span>Configurateur PC</span> </a>', '<div class=\"index-search__input\"><input type=\"text\" class=\"index-search__input-value index-search__input-value--xl svelte-ambpd9\" aria-label=\"Enter domain, keyword or URL\" placeholder=\"Enter domain, keyword or URL\" role=\"combobox\" aria-autocomplete=\"list\" aria-controls=\"suggest-list\" aria-expanded=\"false\"> <div class=\"index-search__input-outline  svelte-ambpd9\"></div></div>', '<a class=\"\\n  cl-button -secondary -large wf-page-header__cta homepage-hero2-cta\" href=\"https://app.hubspot.com/signup-hubspot/crm?hubs_signup-url=www.hubspot.com&amp;hubs_signup-cta=homepage-hero2-cta&amp;__hstc=753710.a1de9ff36a7a860a9d7d109c23651100.1711646589873.1711646589873.1711646589873.1&amp;__hssc=753710.1.1711646589874&amp;__hsfp=3584728142\" rel=\"noreferrer \" data-hs-event-96027165=\"1\">\\n        Get started free\\n        \\n          <span class=\"visually-hidden\">with HubSpot\\'s free tools</span>\\n        \\n      </a>']\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from tqdm import tqdm\n",
    "\n",
    "def load_html(html, driver):\n",
    "    file_path = 'sample_page.html'\n",
    "\n",
    "    with open(file_path, 'w', encoding='utf-8') as file:\n",
    "        file.write(html)\n",
    "        \n",
    "    abs_file_path = os.path.abspath(\"sample_page.html\")\n",
    "\n",
    "    # Use the file:/// protocol to load the local HTML file\n",
    "    driver.get(f\"file:///{abs_file_path}\")\n",
    "\n",
    "output = []\n",
    "\n",
    "for index, row in tqdm(df.iterrows()):\n",
    "    html = row['html']\n",
    "    load_html(html, driver)\n",
    "\n",
    "    code = row['code']\n",
    "    code = keep_assignments(code)\n",
    "\n",
    "    exec(code)\n",
    "\n",
    "    variables = get_web_elements(code)\n",
    "\n",
    "    # We assume there is only one variable defined\n",
    "    variable = variables[0]\n",
    "    outer_html = variable.get_attribute(\"outerHTML\")\n",
    "\n",
    "    output.append(outer_html)\n",
    "    \n",
    "print(output)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/home/azureuser/anaconda3/envs/lavague_test/lib/python3.10/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "\n",
    "from lavague.action_engine import ActionEngine\n",
    "from lavague.defaults import DefaultEmbedder\n",
    "\n",
    "import os\n",
    "from llama_index.llms.azure_openai import AzureOpenAI\n",
    "\n",
    "from lavague.prompts import DEFAULT_PROMPT\n",
    "\n",
    "\n",
    "\n",
    "api_key=os.getenv(\"AZURE_OPENAI_KEY\")\n",
    "api_version=\"2023-05-15\"\n",
    "azure_endpoint = os.getenv(\"AZURE_OPENAI_ENDPOINT\")\n",
    "model = \"gpt-4\"\n",
    "deployment_name = os.getenv(\"AZURE_OPENAI_DEPLOYMENT_NAME\", \"gpt-4-turbo\")\n",
    "\n",
    "class LLM(AzureOpenAI):\n",
    "    def __init__(self):\n",
    "        super().__init__(\n",
    "            model=model,\n",
    "            deployment_name=deployment_name,\n",
    "            api_key=api_key,\n",
    "            azure_endpoint=azure_endpoint,\n",
    "            api_version=api_version,\n",
    "            temperature=0.0\n",
    "        )\n",
    "llm = LLM()\n",
    "embedder = DefaultEmbedder()\n",
    "\n",
    "action_engine = ActionEngine(llm, embedder, streaming=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [],
   "source": [
    "from lavague.prompts import ENHANCED_PROMPT"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "url                                https://www.hubspot.com\n",
       "html     <html lang=\"en\"><head>\\n    <meta charset=\"utf...\n",
       "query                            click on get started free\n",
       "node     <a class=\"\\n  cl-button -secondary -large wf-p...\n",
       "code     # Let's proceed step by step.\\n# First we need...\n",
       "Name: 9, dtype: object"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "row"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "10it [01:54, 11.42s/it]"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "[1, 1, 1, 1, 1, 1, 1, 1, 1, 0]\n"
     ]
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "\n"
     ]
    }
   ],
   "source": [
    "from lavague.action_engine import extract_first_python_code\n",
    "\n",
    "scores = []\n",
    "generated_codes = []\n",
    "generated_outer_htmls = []\n",
    "\n",
    "for index, row in tqdm(df.iterrows()):\n",
    "    node = row['node']\n",
    "    query = row['query']\n",
    "    html = row['html']\n",
    "\n",
    "    prompt = ENHANCED_PROMPT.format(context_str=node, query_str=query)\n",
    "    generated_code = llm.complete(prompt).text\n",
    "    generated_code = extract_first_python_code(generated_code)\n",
    "\n",
    "    load_html(html, driver)\n",
    "\n",
    "    generated_code = keep_assignments(generated_code)\n",
    "    generated_codes.append(generated_code)\n",
    "    try:\n",
    "        exec(generated_code)\n",
    "        variables = get_web_elements(generated_code)\n",
    "\n",
    "        # We assume there is only one variable defined\n",
    "        variable = variables[0]\n",
    "        outer_html = variable.get_attribute(\"outerHTML\")\n",
    "        generated_outer_htmls.append(outer_html)\n",
    "\n",
    "        if outer_html in output[index] or output[index] in outer_html:\n",
    "            scores.append(1)\n",
    "        else:\n",
    "            scores.append(0)\n",
    "    except Exception as e:\n",
    "        scores.append(0)\n",
    "        print(\"Error at index: \", index)\n",
    "        print(\"Non executable code\")\n",
    "        print(\"Code snippet: \", generated_code)\n",
    "        print(\"Error: \", e)\n",
    "print(scores)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<div class=\"index-search__input\"><input type=\"text\" class=\"index-search__input-value index-search__input-value--xl svelte-ambpd9\" aria-label=\"Enter domain, keyword or URL\" placeholder=\"Enter domain, keyword or URL\" role=\"combobox\" aria-autocomplete=\"list\" aria-controls=\"suggest-list\" aria-expanded=\"false\"> <div class=\"index-search__input-outline  svelte-ambpd9\"></div></div>'"
      ]
     },
     "execution_count": 18,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "output[8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'<input type=\"text\" class=\"index-search__input-value index-search__input-value--xl svelte-ambpd9\" aria-label=\"Enter domain, keyword or URL\" placeholder=\"Enter domain, keyword or URL\" role=\"combobox\" aria-autocomplete=\"list\" aria-controls=\"suggest-list\" aria-expanded=\"false\">'"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "generated_outer_htmls[8]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "0it [00:13, ?it/s]\n"
     ]
    },
    {
     "ename": "KeyboardInterrupt",
     "evalue": "",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[16], line 10\u001b[0m\n\u001b[1;32m      7\u001b[0m html \u001b[38;5;241m=\u001b[39m row[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mhtml\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[1;32m      8\u001b[0m query \u001b[38;5;241m=\u001b[39m row[\u001b[38;5;124m'\u001b[39m\u001b[38;5;124mquery\u001b[39m\u001b[38;5;124m'\u001b[39m]\n\u001b[0;32m---> 10\u001b[0m generated_code, retrieved_nodes \u001b[38;5;241m=\u001b[39m \u001b[43maction_engine\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_action\u001b[49m\u001b[43m(\u001b[49m\u001b[43mquery\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mhtml\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     12\u001b[0m load_html(html, driver)\n\u001b[1;32m     14\u001b[0m generated_code \u001b[38;5;241m=\u001b[39m keep_assignments(generated_code)\n",
      "File \u001b[0;32m~/python_projects/LaVague/src/lavague/action_engine.py:130\u001b[0m, in \u001b[0;36mActionEngine.get_action\u001b[0;34m(self, query, html)\u001b[0m\n\u001b[1;32m    119\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mget_action\u001b[39m(\u001b[38;5;28mself\u001b[39m, query: \u001b[38;5;28mstr\u001b[39m, html: \u001b[38;5;28mstr\u001b[39m) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m \u001b[38;5;28mstr\u001b[39m:\n\u001b[1;32m    120\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m    121\u001b[0m \u001b[38;5;124;03m    Generate the code from a query and an html page, only works if streaming=False\u001b[39;00m\n\u001b[1;32m    122\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    128\u001b[0m \u001b[38;5;124;03m        (`str`, `str`): The generated code, and the sources which were used\u001b[39;00m\n\u001b[1;32m    129\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 130\u001b[0m     query_engine \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_query_engine\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhtml\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    131\u001b[0m     response \u001b[38;5;241m=\u001b[39m query_engine\u001b[38;5;241m.\u001b[39mquery(query)\n\u001b[1;32m    132\u001b[0m     source_nodes \u001b[38;5;241m=\u001b[39m response\u001b[38;5;241m.\u001b[39mget_formatted_sources(\n\u001b[1;32m    133\u001b[0m         \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmax_chars_pc\n\u001b[1;32m    134\u001b[0m     )  \u001b[38;5;66;03m# HTML sources with which the code was generated\u001b[39;00m\n",
      "File \u001b[0;32m~/python_projects/LaVague/src/lavague/action_engine.py:95\u001b[0m, in \u001b[0;36mActionEngine.get_query_engine\u001b[0;34m(self, state)\u001b[0m\n\u001b[1;32m     85\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m     86\u001b[0m \u001b[38;5;124;03mGet the llama-index query engine\u001b[39;00m\n\u001b[1;32m     87\u001b[0m \n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m     92\u001b[0m \u001b[38;5;124;03m    `RetrieverQueryEngine`\u001b[39;00m\n\u001b[1;32m     93\u001b[0m \u001b[38;5;124;03m\"\"\"\u001b[39;00m\n\u001b[1;32m     94\u001b[0m html \u001b[38;5;241m=\u001b[39m state\n\u001b[0;32m---> 95\u001b[0m index \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_index\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhtml\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     97\u001b[0m retriever \u001b[38;5;241m=\u001b[39m BM25Retriever\u001b[38;5;241m.\u001b[39mfrom_defaults(\n\u001b[1;32m     98\u001b[0m     index\u001b[38;5;241m=\u001b[39mindex, similarity_top_k\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtop_k\n\u001b[1;32m     99\u001b[0m )\n\u001b[1;32m    101\u001b[0m response_synthesizer \u001b[38;5;241m=\u001b[39m get_response_synthesizer(\n\u001b[1;32m    102\u001b[0m     streaming\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstreaming, llm\u001b[38;5;241m=\u001b[39m\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mllm\n\u001b[1;32m    103\u001b[0m )\n",
      "File \u001b[0;32m~/python_projects/LaVague/src/lavague/action_engine.py:80\u001b[0m, in \u001b[0;36mActionEngine._get_index\u001b[0;34m(self, html)\u001b[0m\n\u001b[1;32m     77\u001b[0m nodes \u001b[38;5;241m=\u001b[39m splitter\u001b[38;5;241m.\u001b[39mget_nodes_from_documents(documents)\n\u001b[1;32m     78\u001b[0m nodes \u001b[38;5;241m=\u001b[39m [node \u001b[38;5;28;01mfor\u001b[39;00m node \u001b[38;5;129;01min\u001b[39;00m nodes \u001b[38;5;28;01mif\u001b[39;00m node\u001b[38;5;241m.\u001b[39mtext]\n\u001b[0;32m---> 80\u001b[0m index \u001b[38;5;241m=\u001b[39m \u001b[43mVectorStoreIndex\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnodes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43membed_model\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43membedding\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     82\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m index\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/llama_index/core/indices/vector_store/base.py:74\u001b[0m, in \u001b[0;36mVectorStoreIndex.__init__\u001b[0;34m(self, nodes, use_async, store_nodes_override, embed_model, insert_batch_size, objects, index_struct, storage_context, callback_manager, transformations, show_progress, service_context, **kwargs)\u001b[0m\n\u001b[1;32m     67\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_embed_model \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m     68\u001b[0m     resolve_embed_model(embed_model, callback_manager\u001b[38;5;241m=\u001b[39mcallback_manager)\n\u001b[1;32m     69\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m embed_model\n\u001b[1;32m     70\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m embed_model_from_settings_or_context(Settings, service_context)\n\u001b[1;32m     71\u001b[0m )\n\u001b[1;32m     73\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_insert_batch_size \u001b[38;5;241m=\u001b[39m insert_batch_size\n\u001b[0;32m---> 74\u001b[0m \u001b[38;5;28;43msuper\u001b[39;49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[38;5;21;43m__init__\u001b[39;49m\u001b[43m(\u001b[49m\n\u001b[1;32m     75\u001b[0m \u001b[43m    \u001b[49m\u001b[43mnodes\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mnodes\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     76\u001b[0m \u001b[43m    \u001b[49m\u001b[43mindex_struct\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mindex_struct\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     77\u001b[0m \u001b[43m    \u001b[49m\u001b[43mservice_context\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mservice_context\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     78\u001b[0m \u001b[43m    \u001b[49m\u001b[43mstorage_context\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mstorage_context\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     79\u001b[0m \u001b[43m    \u001b[49m\u001b[43mshow_progress\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mshow_progress\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     80\u001b[0m \u001b[43m    \u001b[49m\u001b[43mobjects\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mobjects\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     81\u001b[0m \u001b[43m    \u001b[49m\u001b[43mcallback_manager\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mcallback_manager\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     82\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtransformations\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mtransformations\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     83\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m     84\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/llama_index/core/indices/base.py:94\u001b[0m, in \u001b[0;36mBaseIndex.__init__\u001b[0;34m(self, nodes, objects, index_struct, storage_context, callback_manager, transformations, show_progress, service_context, **kwargs)\u001b[0m\n\u001b[1;32m     92\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m index_struct \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m     93\u001b[0m     nodes \u001b[38;5;241m=\u001b[39m nodes \u001b[38;5;129;01mor\u001b[39;00m []\n\u001b[0;32m---> 94\u001b[0m     index_struct \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbuild_index_from_nodes\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m     95\u001b[0m \u001b[43m        \u001b[49m\u001b[43mnodes\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m+\u001b[39;49m\u001b[43m \u001b[49m\u001b[43mobjects\u001b[49m\u001b[43m  \u001b[49m\u001b[38;5;66;43;03m# type: ignore\u001b[39;49;00m\n\u001b[1;32m     96\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m     97\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_index_struct \u001b[38;5;241m=\u001b[39m index_struct\n\u001b[1;32m     98\u001b[0m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_storage_context\u001b[38;5;241m.\u001b[39mindex_store\u001b[38;5;241m.\u001b[39madd_index_struct(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_index_struct)\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/llama_index/core/indices/vector_store/base.py:307\u001b[0m, in \u001b[0;36mVectorStoreIndex.build_index_from_nodes\u001b[0;34m(self, nodes, **insert_kwargs)\u001b[0m\n\u001b[1;32m    299\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28many\u001b[39m(\n\u001b[1;32m    300\u001b[0m     node\u001b[38;5;241m.\u001b[39mget_content(metadata_mode\u001b[38;5;241m=\u001b[39mMetadataMode\u001b[38;5;241m.\u001b[39mEMBED) \u001b[38;5;241m==\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;28;01mfor\u001b[39;00m node \u001b[38;5;129;01min\u001b[39;00m nodes\n\u001b[1;32m    301\u001b[0m ):\n\u001b[1;32m    302\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mValueError\u001b[39;00m(\n\u001b[1;32m    303\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mCannot build index from nodes with no content. \u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    304\u001b[0m         \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mPlease ensure all nodes have content.\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    305\u001b[0m     )\n\u001b[0;32m--> 307\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_build_index_from_nodes\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnodes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minsert_kwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/llama_index/core/indices/vector_store/base.py:279\u001b[0m, in \u001b[0;36mVectorStoreIndex._build_index_from_nodes\u001b[0;34m(self, nodes, **insert_kwargs)\u001b[0m\n\u001b[1;32m    277\u001b[0m     run_async_tasks(tasks)\n\u001b[1;32m    278\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 279\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_add_nodes_to_index\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    280\u001b[0m \u001b[43m        \u001b[49m\u001b[43mindex_struct\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    281\u001b[0m \u001b[43m        \u001b[49m\u001b[43mnodes\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    282\u001b[0m \u001b[43m        \u001b[49m\u001b[43mshow_progress\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_show_progress\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    283\u001b[0m \u001b[43m        \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minsert_kwargs\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    284\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    285\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m index_struct\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/llama_index/core/indices/vector_store/base.py:232\u001b[0m, in \u001b[0;36mVectorStoreIndex._add_nodes_to_index\u001b[0;34m(self, index_struct, nodes, show_progress, **insert_kwargs)\u001b[0m\n\u001b[1;32m    229\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m\n\u001b[1;32m    231\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m nodes_batch \u001b[38;5;129;01min\u001b[39;00m iter_batch(nodes, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_insert_batch_size):\n\u001b[0;32m--> 232\u001b[0m     nodes_batch \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_node_with_embedding\u001b[49m\u001b[43m(\u001b[49m\u001b[43mnodes_batch\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mshow_progress\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    233\u001b[0m     new_ids \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_vector_store\u001b[38;5;241m.\u001b[39madd(nodes_batch, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39minsert_kwargs)\n\u001b[1;32m    235\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_vector_store\u001b[38;5;241m.\u001b[39mstores_text \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_store_nodes_override:\n\u001b[1;32m    236\u001b[0m         \u001b[38;5;66;03m# NOTE: if the vector store doesn't store text,\u001b[39;00m\n\u001b[1;32m    237\u001b[0m         \u001b[38;5;66;03m# we need to add the nodes to the index struct and document store\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/llama_index/core/indices/vector_store/base.py:140\u001b[0m, in \u001b[0;36mVectorStoreIndex._get_node_with_embedding\u001b[0;34m(self, nodes, show_progress)\u001b[0m\n\u001b[1;32m    129\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m_get_node_with_embedding\u001b[39m(\n\u001b[1;32m    130\u001b[0m     \u001b[38;5;28mself\u001b[39m,\n\u001b[1;32m    131\u001b[0m     nodes: Sequence[BaseNode],\n\u001b[1;32m    132\u001b[0m     show_progress: \u001b[38;5;28mbool\u001b[39m \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mFalse\u001b[39;00m,\n\u001b[1;32m    133\u001b[0m ) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m List[BaseNode]:\n\u001b[1;32m    134\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Get tuples of id, node, and embedding.\u001b[39;00m\n\u001b[1;32m    135\u001b[0m \n\u001b[1;32m    136\u001b[0m \u001b[38;5;124;03m    Allows us to store these nodes in a vector store.\u001b[39;00m\n\u001b[1;32m    137\u001b[0m \u001b[38;5;124;03m    Embeddings are called in batches.\u001b[39;00m\n\u001b[1;32m    138\u001b[0m \n\u001b[1;32m    139\u001b[0m \u001b[38;5;124;03m    \"\"\"\u001b[39;00m\n\u001b[0;32m--> 140\u001b[0m     id_to_embed_map \u001b[38;5;241m=\u001b[39m \u001b[43membed_nodes\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    141\u001b[0m \u001b[43m        \u001b[49m\u001b[43mnodes\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_embed_model\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mshow_progress\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mshow_progress\u001b[49m\n\u001b[1;32m    142\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    144\u001b[0m     results \u001b[38;5;241m=\u001b[39m []\n\u001b[1;32m    145\u001b[0m     \u001b[38;5;28;01mfor\u001b[39;00m node \u001b[38;5;129;01min\u001b[39;00m nodes:\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/llama_index/core/indices/utils.py:138\u001b[0m, in \u001b[0;36membed_nodes\u001b[0;34m(nodes, embed_model, show_progress)\u001b[0m\n\u001b[1;32m    135\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    136\u001b[0m         id_to_embed_map[node\u001b[38;5;241m.\u001b[39mnode_id] \u001b[38;5;241m=\u001b[39m node\u001b[38;5;241m.\u001b[39membedding\n\u001b[0;32m--> 138\u001b[0m new_embeddings \u001b[38;5;241m=\u001b[39m \u001b[43membed_model\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_text_embedding_batch\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    139\u001b[0m \u001b[43m    \u001b[49m\u001b[43mtexts_to_embed\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mshow_progress\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mshow_progress\u001b[49m\n\u001b[1;32m    140\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    142\u001b[0m \u001b[38;5;28;01mfor\u001b[39;00m new_id, text_embedding \u001b[38;5;129;01min\u001b[39;00m \u001b[38;5;28mzip\u001b[39m(ids_to_embed, new_embeddings):\n\u001b[1;32m    143\u001b[0m     id_to_embed_map[new_id] \u001b[38;5;241m=\u001b[39m text_embedding\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/llama_index/core/base/embeddings/base.py:255\u001b[0m, in \u001b[0;36mBaseEmbedding.get_text_embedding_batch\u001b[0;34m(self, texts, show_progress, **kwargs)\u001b[0m\n\u001b[1;32m    249\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m idx \u001b[38;5;241m==\u001b[39m \u001b[38;5;28mlen\u001b[39m(texts) \u001b[38;5;241m-\u001b[39m \u001b[38;5;241m1\u001b[39m \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(cur_batch) \u001b[38;5;241m==\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39membed_batch_size:\n\u001b[1;32m    250\u001b[0m     \u001b[38;5;66;03m# flush\u001b[39;00m\n\u001b[1;32m    251\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcallback_manager\u001b[38;5;241m.\u001b[39mevent(\n\u001b[1;32m    252\u001b[0m         CBEventType\u001b[38;5;241m.\u001b[39mEMBEDDING,\n\u001b[1;32m    253\u001b[0m         payload\u001b[38;5;241m=\u001b[39m{EventPayload\u001b[38;5;241m.\u001b[39mSERIALIZED: \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mto_dict()},\n\u001b[1;32m    254\u001b[0m     ) \u001b[38;5;28;01mas\u001b[39;00m event:\n\u001b[0;32m--> 255\u001b[0m         embeddings \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_get_text_embeddings\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcur_batch\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    256\u001b[0m         result_embeddings\u001b[38;5;241m.\u001b[39mextend(embeddings)\n\u001b[1;32m    257\u001b[0m         event\u001b[38;5;241m.\u001b[39mon_end(\n\u001b[1;32m    258\u001b[0m             payload\u001b[38;5;241m=\u001b[39m{\n\u001b[1;32m    259\u001b[0m                 EventPayload\u001b[38;5;241m.\u001b[39mCHUNKS: cur_batch,\n\u001b[1;32m    260\u001b[0m                 EventPayload\u001b[38;5;241m.\u001b[39mEMBEDDINGS: embeddings,\n\u001b[1;32m    261\u001b[0m             },\n\u001b[1;32m    262\u001b[0m         )\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/llama_index/embeddings/huggingface/base.py:204\u001b[0m, in \u001b[0;36mHuggingFaceEmbedding._get_text_embeddings\u001b[0;34m(self, texts)\u001b[0m\n\u001b[1;32m    200\u001b[0m \u001b[38;5;250m\u001b[39m\u001b[38;5;124;03m\"\"\"Get text embeddings.\"\"\"\u001b[39;00m\n\u001b[1;32m    201\u001b[0m texts \u001b[38;5;241m=\u001b[39m [\n\u001b[1;32m    202\u001b[0m     format_text(text, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel_name, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtext_instruction) \u001b[38;5;28;01mfor\u001b[39;00m text \u001b[38;5;129;01min\u001b[39;00m texts\n\u001b[1;32m    203\u001b[0m ]\n\u001b[0;32m--> 204\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_embed\u001b[49m\u001b[43m(\u001b[49m\u001b[43mtexts\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/llama_index/embeddings/huggingface/base.py:161\u001b[0m, in \u001b[0;36mHuggingFaceEmbedding._embed\u001b[0;34m(self, sentences)\u001b[0m\n\u001b[1;32m    156\u001b[0m \u001b[38;5;66;03m# move tokenizer inputs to device\u001b[39;00m\n\u001b[1;32m    157\u001b[0m encoded_input \u001b[38;5;241m=\u001b[39m {\n\u001b[1;32m    158\u001b[0m     key: val\u001b[38;5;241m.\u001b[39mto(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_device) \u001b[38;5;28;01mfor\u001b[39;00m key, val \u001b[38;5;129;01min\u001b[39;00m encoded_input\u001b[38;5;241m.\u001b[39mitems()\n\u001b[1;32m    159\u001b[0m }\n\u001b[0;32m--> 161\u001b[0m model_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_model\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mencoded_input\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    163\u001b[0m context_layer: \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtorch.Tensor\u001b[39m\u001b[38;5;124m\"\u001b[39m \u001b[38;5;241m=\u001b[39m model_output[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m    164\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpooling \u001b[38;5;241m==\u001b[39m Pooling\u001b[38;5;241m.\u001b[39mCLS:\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1509\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1518\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1519\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1523\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/transformers/models/bert/modeling_bert.py:1013\u001b[0m, in \u001b[0;36mBertModel.forward\u001b[0;34m(self, input_ids, attention_mask, token_type_ids, position_ids, head_mask, inputs_embeds, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   1004\u001b[0m head_mask \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mget_head_mask(head_mask, \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39mnum_hidden_layers)\n\u001b[1;32m   1006\u001b[0m embedding_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39membeddings(\n\u001b[1;32m   1007\u001b[0m     input_ids\u001b[38;5;241m=\u001b[39minput_ids,\n\u001b[1;32m   1008\u001b[0m     position_ids\u001b[38;5;241m=\u001b[39mposition_ids,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m   1011\u001b[0m     past_key_values_length\u001b[38;5;241m=\u001b[39mpast_key_values_length,\n\u001b[1;32m   1012\u001b[0m )\n\u001b[0;32m-> 1013\u001b[0m encoder_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mencoder\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m   1014\u001b[0m \u001b[43m    \u001b[49m\u001b[43membedding_output\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1015\u001b[0m \u001b[43m    \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mextended_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1016\u001b[0m \u001b[43m    \u001b[49m\u001b[43mhead_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mhead_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1017\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1018\u001b[0m \u001b[43m    \u001b[49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mencoder_extended_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1019\u001b[0m \u001b[43m    \u001b[49m\u001b[43mpast_key_values\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mpast_key_values\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1020\u001b[0m \u001b[43m    \u001b[49m\u001b[43muse_cache\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43muse_cache\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1021\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1022\u001b[0m \u001b[43m    \u001b[49m\u001b[43moutput_hidden_states\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43moutput_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1023\u001b[0m \u001b[43m    \u001b[49m\u001b[43mreturn_dict\u001b[49m\u001b[38;5;241;43m=\u001b[39;49m\u001b[43mreturn_dict\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m   1024\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1025\u001b[0m sequence_output \u001b[38;5;241m=\u001b[39m encoder_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m   1026\u001b[0m pooled_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpooler(sequence_output) \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mpooler \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1509\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1518\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1519\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1523\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/transformers/models/bert/modeling_bert.py:607\u001b[0m, in \u001b[0;36mBertEncoder.forward\u001b[0;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_values, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m    596\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_gradient_checkpointing_func(\n\u001b[1;32m    597\u001b[0m         layer_module\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__call__\u001b[39m,\n\u001b[1;32m    598\u001b[0m         hidden_states,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    604\u001b[0m         output_attentions,\n\u001b[1;32m    605\u001b[0m     )\n\u001b[1;32m    606\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 607\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m \u001b[43mlayer_module\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    608\u001b[0m \u001b[43m        \u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    609\u001b[0m \u001b[43m        \u001b[49m\u001b[43mattention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    610\u001b[0m \u001b[43m        \u001b[49m\u001b[43mlayer_head_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    611\u001b[0m \u001b[43m        \u001b[49m\u001b[43mencoder_hidden_states\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    612\u001b[0m \u001b[43m        \u001b[49m\u001b[43mencoder_attention_mask\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    613\u001b[0m \u001b[43m        \u001b[49m\u001b[43mpast_key_value\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    614\u001b[0m \u001b[43m        \u001b[49m\u001b[43moutput_attentions\u001b[49m\u001b[43m,\u001b[49m\n\u001b[1;32m    615\u001b[0m \u001b[43m    \u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    617\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m layer_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m    618\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m use_cache:\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1509\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1518\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1519\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1523\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/transformers/models/bert/modeling_bert.py:539\u001b[0m, in \u001b[0;36mBertLayer.forward\u001b[0;34m(self, hidden_states, attention_mask, head_mask, encoder_hidden_states, encoder_attention_mask, past_key_value, output_attentions)\u001b[0m\n\u001b[1;32m    536\u001b[0m     cross_attn_present_key_value \u001b[38;5;241m=\u001b[39m cross_attention_outputs[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m]\n\u001b[1;32m    537\u001b[0m     present_key_value \u001b[38;5;241m=\u001b[39m present_key_value \u001b[38;5;241m+\u001b[39m cross_attn_present_key_value\n\u001b[0;32m--> 539\u001b[0m layer_output \u001b[38;5;241m=\u001b[39m \u001b[43mapply_chunking_to_forward\u001b[49m\u001b[43m(\u001b[49m\n\u001b[1;32m    540\u001b[0m \u001b[43m    \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mfeed_forward_chunk\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mchunk_size_feed_forward\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mseq_len_dim\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mattention_output\u001b[49m\n\u001b[1;32m    541\u001b[0m \u001b[43m\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    542\u001b[0m outputs \u001b[38;5;241m=\u001b[39m (layer_output,) \u001b[38;5;241m+\u001b[39m outputs\n\u001b[1;32m    544\u001b[0m \u001b[38;5;66;03m# if decoder, return the attn key/values as the last output\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/transformers/pytorch_utils.py:237\u001b[0m, in \u001b[0;36mapply_chunking_to_forward\u001b[0;34m(forward_fn, chunk_size, chunk_dim, *input_tensors)\u001b[0m\n\u001b[1;32m    234\u001b[0m     \u001b[38;5;66;03m# concatenate output at same dimension\u001b[39;00m\n\u001b[1;32m    235\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m torch\u001b[38;5;241m.\u001b[39mcat(output_chunks, dim\u001b[38;5;241m=\u001b[39mchunk_dim)\n\u001b[0;32m--> 237\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_fn\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43minput_tensors\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/transformers/models/bert/modeling_bert.py:552\u001b[0m, in \u001b[0;36mBertLayer.feed_forward_chunk\u001b[0;34m(self, attention_output)\u001b[0m\n\u001b[1;32m    550\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mfeed_forward_chunk\u001b[39m(\u001b[38;5;28mself\u001b[39m, attention_output):\n\u001b[1;32m    551\u001b[0m     intermediate_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mintermediate(attention_output)\n\u001b[0;32m--> 552\u001b[0m     layer_output \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43moutput\u001b[49m\u001b[43m(\u001b[49m\u001b[43mintermediate_output\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mattention_output\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    553\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m layer_output\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1509\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1518\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1519\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1523\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/transformers/models/bert/modeling_bert.py:464\u001b[0m, in \u001b[0;36mBertOutput.forward\u001b[0;34m(self, hidden_states, input_tensor)\u001b[0m\n\u001b[1;32m    463\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, hidden_states: torch\u001b[38;5;241m.\u001b[39mTensor, input_tensor: torch\u001b[38;5;241m.\u001b[39mTensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m torch\u001b[38;5;241m.\u001b[39mTensor:\n\u001b[0;32m--> 464\u001b[0m     hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mdense\u001b[49m\u001b[43m(\u001b[49m\u001b[43mhidden_states\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    465\u001b[0m     hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mdropout(hidden_states)\n\u001b[1;32m    466\u001b[0m     hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mLayerNorm(hidden_states \u001b[38;5;241m+\u001b[39m input_tensor)\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/torch/nn/modules/module.py:1511\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1509\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1510\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1511\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_call_impl\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/torch/nn/modules/module.py:1520\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1515\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1516\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1517\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1518\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1519\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1520\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mforward_call\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43margs\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[38;5;241;43m*\u001b[39;49m\u001b[43mkwargs\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m   1522\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   1523\u001b[0m     result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n",
      "File \u001b[0;32m~/anaconda3/envs/lavague_test/lib/python3.10/site-packages/torch/nn/modules/linear.py:116\u001b[0m, in \u001b[0;36mLinear.forward\u001b[0;34m(self, input)\u001b[0m\n\u001b[1;32m    115\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;28minput\u001b[39m: Tensor) \u001b[38;5;241m-\u001b[39m\u001b[38;5;241m>\u001b[39m Tensor:\n\u001b[0;32m--> 116\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[43mF\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mlinear\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;28;43minput\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mweight\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mbias\u001b[49m\u001b[43m)\u001b[49m\n",
      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
     ]
    }
   ],
   "source": [
    "scores = []\n",
    "generated_codes = []\n",
    "\n",
    "generated_outer_htmls = []\n",
    "\n",
    "for index, row in tqdm(df.iterrows()):\n",
    "    html = row['html']\n",
    "    query = row['query']\n",
    "\n",
    "    generated_code, retrieved_nodes = action_engine.get_action(query, html)\n",
    "\n",
    "    load_html(html, driver)\n",
    "\n",
    "    generated_code = keep_assignments(generated_code)\n",
    "    generated_codes.append(generated_code)\n",
    "    try:\n",
    "        exec(generated_code)\n",
    "        variables = get_web_elements(generated_code)\n",
    "\n",
    "        # We assume there is only one variable defined\n",
    "        variable = variables[0]\n",
    "        outer_html = variable.get_attribute(\"outerHTML\")\n",
    "        generated_outer_htmls.append(outer_html)\n",
    "\n",
    "        if outer_html in output[index] or output[index] in outer_html:\n",
    "            scores.append(1)\n",
    "        else:\n",
    "            scores.append(0)\n",
    "    except Exception as e:\n",
    "        generated_outer_htmls.append(\"None\")\n",
    "        scores.append(0)\n",
    "        print(\"Error at index: \", index)\n",
    "        print(\"Non executable code\")\n",
    "        print(\"Code snippet: \", generated_code)\n",
    "        print(\"Error: \", e)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "df[\"scores\"] = scores\n",
    "df[\"generated_codes\"] = generated_codes\n",
    "df[\"generated_outer_htmls\"] = generated_outer_htmls\n",
    "\n",
    "df.to_csv(\"dataset_with_scores.csv\", index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Gold standard code:  # Let's proceed step by step.\n",
      "# First we need to identify the link first, then we can click on it.\n",
      "\n",
      "# Based on the HTML provided, we need to devise the best strategy to select the link.\n",
      "# The link can be identified using the text \"Datasets\"\n",
      "# We can use the XPATH to select the link\n",
      "datasets_link = driver.find_element(By.XPATH, \"//a[contains(text(), 'Datasets')]\")\n",
      "\n",
      "# Then we can click on it\n",
      "datasets_link.click()\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<a class=\"group flex items-center px-2 py-0.5 dark:hover:text-gray-400 hover:text-red-700\" href=\"/datasets\"><svg class=\"mr-1.5 text-gray-400 group-hover:text-red-500\" style=\"\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" aria-hidden=\"true\" focusable=\"false\" role=\"img\" width=\"1em\" height=\"1em\" preserveAspectRatio=\"xMidYMid meet\" viewBox=\"0 0 25 25\"><ellipse cx=\"12.5\" cy=\"5\" fill=\"currentColor\" fill-opacity=\"0.25\" rx=\"7.5\" ry=\"2\"></ellipse><path d=\"M12.5 15C16.6421 15 20 14.1046 20 13V20C20 21.1046 16.6421 22 12.5 22C8.35786 22 5 21.1046 5 20V13C5 14.1046 8.35786 15 12.5 15Z\" fill=\"currentColor\" opacity=\"0.5\"></path><path d=\"M12.5 7C16.6421 7 20 6.10457 20 5V11.5C20 12.6046 16.6421 13.5 12.5 13.5C8.35786 13.5 5 12.6046 5 11.5V5C5 6.10457 8.35786 7 12.5 7Z\" fill=\"currentColor\" opacity=\"0.5\"></path><path d=\"M5.23628 12C5.08204 12.1598 5 12.8273 5 13C5 14.1046 8.35786 15 12.5 15C16.6421 15 20 14.1046 20 13C20 12.8273 19.918 12.1598 19.7637 12C18.9311 12.8626 15.9947 13.5 12.5 13.5C9.0053 13.5 6.06886 12.8626 5.23628 12Z\" fill=\"currentColor\"></path></svg> Datasets</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<a class=\"group flex items-center px-2 py-0.5 dark:hover:text-gray-400 hover:text-red-700\" href=\"/datasets\"><svg class=\"mr-1.5 text-gray-400 group-hover:text-red-500\" style=\"\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" aria-hidden=\"true\" focusable=\"false\" role=\"img\" width=\"1em\" height=\"1em\" preserveAspectRatio=\"xMidYMid meet\" viewBox=\"0 0 25 25\"><ellipse cx=\"12.5\" cy=\"5\" fill=\"currentColor\" fill-opacity=\"0.25\" rx=\"7.5\" ry=\"2\"></ellipse><path d=\"M12.5 15C16.6421 15 20 14.1046 20 13V20C20 21.1046 16.6421 22 12.5 22C8.35786 22 5 21.1046 5 20V13C5 14.1046 8.35786 15 12.5 15Z\" fill=\"currentColor\" opacity=\"0.5\"></path><path d=\"M12.5 7C16.6421 7 20 6.10457 20 5V11.5C20 12.6046 16.6421 13.5 12.5 13.5C8.35786 13.5 5 12.6046 5 11.5V5C5 6.10457 8.35786 7 12.5 7Z\" fill=\"currentColor\" opacity=\"0.5\"></path><path d=\"M5.23628 12C5.08204 12.1598 5 12.8273 5 13C5 14.1046 8.35786 15 12.5 15C16.6421 15 20 14.1046 20 13C20 12.8273 19.918 12.1598 19.7637 12C18.9311 12.8626 15.9947 13.5 12.5 13.5C9.0053 13.5 6.06886 12.8626 5.23628 12Z\" fill=\"currentColor\"></path></svg> Datasets</a>\n",
      "Generated code:  dataset_links = driver.find_elements(By.XPATH, \"//a[contains(@class, 'block p-2')]\")\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "None"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "None\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Gold standard code:  # Let's proceed step by step.\n",
      "# First we need to identify the link first, then we can click on it.\n",
      "\n",
      "# Based on the HTML provided, we need to devise the best strategy to select the link.\n",
      "# The competitions link can be identified using the class name \"sc-gjLLEI sc-fAscey kIbkrq dDqYED\" and the text \"Competitions\"\n",
      "competitions_link = driver.find_element(By.XPATH, \"//*[contains(text(), 'Competitions')]\")\n",
      "\n",
      "# Then we can click on it\n",
      "competitions_link.click()\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<a href=\"/competitions\" data-click-log-id=\"nav-con-competitions\" class=\"sc-jtMbrs bwioyi\">Competitions</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<a href=\"/competitions\" data-click-log-id=\"nav-con-competitions\" class=\"sc-jtMbrs bwioyi\">Competitions</a>\n",
      "Generated code:  competitions_link = driver.find_element(By.XPATH, \"//a[@href='/docs/competitions']\")\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<a href=\"/docs/competitions\" data-click-log-id=\"km-footer-docs-competitions\" class=\"sc-gjLLEI sc-fAscey kIbkrq dDqYED\">Competitions</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<a href=\"/docs/competitions\" data-click-log-id=\"km-footer-docs-competitions\" class=\"sc-gjLLEI sc-fAscey kIbkrq dDqYED\">Competitions</a>\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Gold standard code:  # Let's proceed step by step.\n",
      "# First we need to identify the button first, then we can click on it.\n",
      "\n",
      "# Based on the HTML provided, we need to devise the best strategy to select the button.\n",
      "# The action button can be identified using the ID \"pt-createaccount-2\"\n",
      "action_button = driver.find_element(By.XPATH, \"//*[@id='pt-createaccount-2']\")\n",
      "\n",
      "# Then we can click on it\n",
      "action_button.click()\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<li id=\"pt-createaccount-2\" class=\"user-links-collapsible-item mw-list-item user-links-collapsible-item\"><a data-mw=\"interface\" href=\"/w/index.php?title=Special:CreateAccount&amp;returnto=Main+Page\" title=\"You are encouraged to create an account and log in; however, it is not mandatory\" class=\"\"><span>Create account</span></a>\n",
       "</li>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<li id=\"pt-createaccount-2\" class=\"user-links-collapsible-item mw-list-item user-links-collapsible-item\"><a data-mw=\"interface\" href=\"/w/index.php?title=Special:CreateAccount&amp;returnto=Main+Page\" title=\"You are encouraged to create an account and log in; however, it is not mandatory\" class=\"\"><span>Create account</span></a>\n",
      "</li>\n",
      "Generated code:  create_account_link = driver.find_element(By.XPATH, \"//*[@id='pt-createaccount']/a\")\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<a href=\"/w/index.php?title=Special:CreateAccount&amp;returnto=Main+Page\" title=\"You are encouraged to create an account and log in; however, it is not mandatory\"><span class=\"vector-icon mw-ui-icon-userAdd mw-ui-icon-wikimedia-userAdd\"></span> <span>Create account</span></a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<a href=\"/w/index.php?title=Special:CreateAccount&amp;returnto=Main+Page\" title=\"You are encouraged to create an account and log in; however, it is not mandatory\"><span class=\"vector-icon mw-ui-icon-userAdd mw-ui-icon-wikimedia-userAdd\"></span> <span>Create account</span></a>\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Gold standard code:  # Let's proceed step by step.\n",
      "# First we need to identify the search bar first, then we can click on it.\n",
      "\n",
      "# Based on the HTML provided, we need to devise the best strategy to select the search bar.\n",
      "# The search bar can be identified using the class name \"index-search__input\"\n",
      "search_bar = driver.find_element(By.XPATH, \"//*[@class='index-search__input']\")\n",
      "\n",
      "# Then we can click on it\n",
      "search_bar.click()\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div class=\"index-search__input\"><input type=\"text\" class=\"index-search__input-value index-search__input-value--xl svelte-ambpd9\" aria-label=\"Enter domain, keyword or URL\" placeholder=\"Enter domain, keyword or URL\" role=\"combobox\" aria-autocomplete=\"list\" aria-controls=\"suggest-list\" aria-expanded=\"false\"> <div class=\"index-search__input-outline  svelte-ambpd9\"></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<div class=\"index-search__input\"><input type=\"text\" class=\"index-search__input-value index-search__input-value--xl svelte-ambpd9\" aria-label=\"Enter domain, keyword or URL\" placeholder=\"Enter domain, keyword or URL\" role=\"combobox\" aria-autocomplete=\"list\" aria-controls=\"suggest-list\" aria-expanded=\"false\"> <div class=\"index-search__input-outline  svelte-ambpd9\"></div></div>\n",
      "Generated code:  search_bar = driver.find_element(By.XPATH, \"//input[@placeholder='Enter domain, keyword or URL']\")\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<input type=\"text\" class=\"index-search__input-value index-search__input-value--xl svelte-ambpd9\" aria-label=\"Enter domain, keyword or URL\" placeholder=\"Enter domain, keyword or URL\" role=\"combobox\" aria-autocomplete=\"list\" aria-controls=\"suggest-list\" aria-expanded=\"false\">"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<input type=\"text\" class=\"index-search__input-value index-search__input-value--xl svelte-ambpd9\" aria-label=\"Enter domain, keyword or URL\" placeholder=\"Enter domain, keyword or URL\" role=\"combobox\" aria-autocomplete=\"list\" aria-controls=\"suggest-list\" aria-expanded=\"false\">\n",
      "\n",
      "\n",
      "\n",
      "\n",
      "Gold standard code:  # Let's proceed step by step.\n",
      "# First we need to identify the button first, then we can click on it.\n",
      "\n",
      "# Based on the HTML provided, we need to devise the best strategy to select the button.\n",
      "# The action button can be identified using the class name \"cl-button -secondary -large wf-page-header__cta homepage-hero2-cta\"\n",
      "# We can use the contains function to select the button\n",
      "get_started_free_button = driver.find_element(By.XPATH, \"//*[contains(@class, 'cl-button -secondary -large wf-page-header__cta homepage-hero2-cta')]\")\n",
      "\n",
      "# Then we can click on it\n",
      "get_started_free_button.click()\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<a class=\"\n",
       "  cl-button -secondary -large wf-page-header__cta homepage-hero2-cta\" href=\"https://app.hubspot.com/signup-hubspot/crm?hubs_signup-url=www.hubspot.com&amp;hubs_signup-cta=homepage-hero2-cta&amp;__hstc=753710.2dd10f2fc1a8042f42261521b6eda287.1711594168300.1711594168300.1711594168300.1&amp;__hssc=753710.1.1711594168300&amp;__hsfp=3584728142\" rel=\"noreferrer \" data-hs-event-96027165=\"1\">\n",
       "        Get started free\n",
       "        \n",
       "          <span class=\"visually-hidden\">with HubSpot's free tools</span>\n",
       "        \n",
       "      </a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<a class=\"\n",
      "  cl-button -secondary -large wf-page-header__cta homepage-hero2-cta\" href=\"https://app.hubspot.com/signup-hubspot/crm?hubs_signup-url=www.hubspot.com&amp;hubs_signup-cta=homepage-hero2-cta&amp;__hstc=753710.2dd10f2fc1a8042f42261521b6eda287.1711594168300.1711594168300.1711594168300.1&amp;__hssc=753710.1.1711594168300&amp;__hsfp=3584728142\" rel=\"noreferrer \" data-hs-event-96027165=\"1\">\n",
      "        Get started free\n",
      "        \n",
      "          <span class=\"visually-hidden\">with HubSpot's free tools</span>\n",
      "        \n",
      "      </a>\n",
      "Generated code:  get_started_free_button = driver.find_element(By.XPATH, \"//*[contains(@class, 'homepage-platform-free') or contains(@class, 'homepage-products-free')]\")\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<a class=\"cl-button -secondary -medium homepage-platform-free\" href=\"https://app.hubspot.com/signup-hubspot/crm?hubs_signup-cta=homepage-platform-free&amp;hubs_signup-url=www.hubspot.com&amp;__hstc=753710.9541084b8df4e80fbb678f928a90f60e.1711595731173.1711595731173.1711595731173.1&amp;__hssc=753710.1.1711595731173&amp;__hsfp=3584728142\" rel=\"noreferrer \" data-hs-event-96027165=\"1\">\n",
       "              Get started free\n",
       "              \n",
       "                <span class=\"visually-hidden\">with HubSpot's free tools</span>\n",
       "              \n",
       "            </a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "<a class=\"cl-button -secondary -medium homepage-platform-free\" href=\"https://app.hubspot.com/signup-hubspot/crm?hubs_signup-cta=homepage-platform-free&amp;hubs_signup-url=www.hubspot.com&amp;__hstc=753710.9541084b8df4e80fbb678f928a90f60e.1711595731173.1711595731173.1711595731173.1&amp;__hssc=753710.1.1711595731173&amp;__hsfp=3584728142\" rel=\"noreferrer \" data-hs-event-96027165=\"1\">\n",
      "              Get started free\n",
      "              \n",
      "                <span class=\"visually-hidden\">with HubSpot's free tools</span>\n",
      "              \n",
      "            </a>\n",
      "\n",
      "\n",
      "\n",
      "\n"
     ]
    }
   ],
   "source": [
    "from IPython.display import HTML, display\n",
    "\n",
    "for i, row in df.loc[df[\"scores\"] == 0].iterrows():\n",
    "    code = row[\"code\"]\n",
    "    gold_html = output[i]\n",
    "    generated_code = row[\"generated_codes\"]\n",
    "    generated_html = row[\"generated_outer_htmls\"]\n",
    "    print(\"Gold standard code: \", code)\n",
    "    display(HTML(gold_html))\n",
    "    print(gold_html)\n",
    "    print(\"Generated code: \", generated_code)\n",
    "    display(HTML(generated_html))\n",
    "    print(generated_html)\n",
    "    print(\"\\n\\n\\n\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.5"
      ]
     },
     "execution_count": 11,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import numpy as np\n",
    "\n",
    "accuracy = np.mean(scores)\n",
    "accuracy"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<a class=\"btn-mktg home-campaign-enterprise btn-muted-mktg\" data-analytics-event=\"{&quot;category&quot;:&quot;Start a free enterprise trial&quot;,&quot;action&quot;:&quot;click to Start a free enterprise trial&quot;,&quot;label&quot;:&quot;ref_cta:Start a free enterprise trial;&quot;}\" href=\"/organizations/enterprise_plan?ref_cta=Start+a+free+enterprise+trial&amp;ref_loc=Home+campaign+hero&amp;ref_page=%2F\" data-test-selector=\"start-trial-button\">\n",
       "  Start a free enterprise trial\n",
       "  \n",
       "  <svg xmlns=\"http://www.w3.org/2000/svg\" class=\"octicon arrow-symbol-mktg\" width=\"16\" height=\"16\" viewBox=\"0 0 16 16\" fill=\"none\"><path fill=\"currentColor\" d=\"M7.28033 3.21967C6.98744 2.92678 6.51256 2.92678 6.21967 3.21967C5.92678 3.51256 5.92678 3.98744 6.21967 4.28033L7.28033 3.21967ZM11 8L11.5303 8.53033C11.8232 8.23744 11.8232 7.76256 11.5303 7.46967L11 8ZM6.21967 11.7197C5.92678 12.0126 5.92678 12.4874 6.21967 12.7803C6.51256 13.0732 6.98744 13.0732 7.28033 12.7803L6.21967 11.7197ZM6.21967 4.28033L10.4697 8.53033L11.5303 7.46967L7.28033 3.21967L6.21967 4.28033ZM10.4697 7.46967L6.21967 11.7197L7.28033 12.7803L11.5303 8.53033L10.4697 7.46967Z\"></path><path class=\"octicon-chevrow-stem\" stroke=\"currentColor\" d=\"M1.75 8H11\" stroke-width=\"1.5\" stroke-linecap=\"round\"></path></svg>\n",
       "\n",
       "</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<a class=\"group flex items-center px-2 py-0.5 dark:hover:text-gray-400 hover:text-red-700\" href=\"/datasets\"><svg class=\"mr-1.5 text-gray-400 group-hover:text-red-500\" style=\"\" xmlns=\"http://www.w3.org/2000/svg\" xmlns:xlink=\"http://www.w3.org/1999/xlink\" aria-hidden=\"true\" focusable=\"false\" role=\"img\" width=\"1em\" height=\"1em\" preserveAspectRatio=\"xMidYMid meet\" viewBox=\"0 0 25 25\"><ellipse cx=\"12.5\" cy=\"5\" fill=\"currentColor\" fill-opacity=\"0.25\" rx=\"7.5\" ry=\"2\"></ellipse><path d=\"M12.5 15C16.6421 15 20 14.1046 20 13V20C20 21.1046 16.6421 22 12.5 22C8.35786 22 5 21.1046 5 20V13C5 14.1046 8.35786 15 12.5 15Z\" fill=\"currentColor\" opacity=\"0.5\"></path><path d=\"M12.5 7C16.6421 7 20 6.10457 20 5V11.5C20 12.6046 16.6421 13.5 12.5 13.5C8.35786 13.5 5 12.6046 5 11.5V5C5 6.10457 8.35786 7 12.5 7Z\" fill=\"currentColor\" opacity=\"0.5\"></path><path d=\"M5.23628 12C5.08204 12.1598 5 12.8273 5 13C5 14.1046 8.35786 15 12.5 15C16.6421 15 20 14.1046 20 13C20 12.8273 19.918 12.1598 19.7637 12C18.9311 12.8626 15.9947 13.5 12.5 13.5C9.0053 13.5 6.06886 12.8626 5.23628 12Z\" fill=\"currentColor\"></path></svg> Datasets</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<a href=\"https://www.irs.gov/compliance/criminal-investigation\" data-drupal-link-system-path=\"node/19091\">Criminal Investigation</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<a href=\"/competitions\" data-click-log-id=\"nav-con-competitions\" class=\"sc-jtMbrs bwioyi\">Competitions</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<li id=\"pt-createaccount-2\" class=\"user-links-collapsible-item mw-list-item user-links-collapsible-item\"><a data-mw=\"interface\" href=\"/w/index.php?title=Special:CreateAccount&amp;returnto=Main+Page\" title=\"You are encouraged to create an account and log in; however, it is not mandatory\" class=\"\"><span>Create account</span></a>\n",
       "</li>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<a class=\"gb_H\" aria-label=\"Gmail (opens a new tab)\" data-pid=\"23\" href=\"https://mail.google.com/mail/&amp;ogbl\" target=\"_top\">Gmail</a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<a class=\"cta_button\" data-size=\"\" data-variant=\"primary\" data-stacking=\"default\" aria-label=\"Start free trial: Salesforce Starter Suite\" href=\"https://www.salesforce.com/form/signup/freetrial-salesforce-starter/?d=pb\" target=\"_self\" rel=\"noopener\">\n",
       "        Start free trial\n",
       "        </a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<a href=\"https://www.ldlc.com/configurateur/\"><span class=\"icon icon-cat icon-tools\">&nbsp;</span> <span>Configurateur PC</span> </a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<div class=\"index-search__input\"><input type=\"text\" class=\"index-search__input-value index-search__input-value--xl svelte-ambpd9\" aria-label=\"Enter domain, keyword or URL\" placeholder=\"Enter domain, keyword or URL\" role=\"combobox\" aria-autocomplete=\"list\" aria-controls=\"suggest-list\" aria-expanded=\"false\"> <div class=\"index-search__input-outline  svelte-ambpd9\"></div></div>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "data": {
      "text/html": [
       "<a class=\"\n",
       "  cl-button -secondary -large wf-page-header__cta homepage-hero2-cta\" href=\"https://app.hubspot.com/signup-hubspot/crm?hubs_signup-url=www.hubspot.com&amp;hubs_signup-cta=homepage-hero2-cta&amp;__hstc=753710.e17c494b47d89a6e8909fbe67f28e7c1.1711586640281.1711586640281.1711586640281.1&amp;__hssc=753710.1.1711586640281&amp;__hsfp=3584728142\" rel=\"noreferrer \" data-hs-event-96027165=\"1\">\n",
       "        Get started free\n",
       "        \n",
       "          <span class=\"visually-hidden\">with HubSpot's free tools</span>\n",
       "        \n",
       "      </a>"
      ],
      "text/plain": [
       "<IPython.core.display.HTML object>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from IPython.display import HTML, display\n",
    "\n",
    "for html in output:\n",
    "    display(HTML(html))"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "lavague",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.10.14"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}