OCMChatGPTPaperReading.ipynb

{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "fdcb8ad0",
   "metadata": {},
   "outputs": [],
   "source": [
    "OPENAI_API_KEY =  'YOUR_API_KEY'"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b562b141",
   "metadata": {},
   "outputs": [],
   "source": [
    "import openai\n",
    "from openai import OpenAI\n",
    "import PyPDF2\n",
    "import re\n",
    "import os\n",
    "import pandas as pd\n",
    "import tiktoken\n",
    "import time\n",
    "from sklearn.metrics.pairwise import cosine_similarity\n",
    "import numpy as np\n",
    "import ast\n",
    "import csv\n",
    "\n",
    "\n",
    "def count_tokens(text):\n",
    "    \"\"\"Returns the number of tokens in a text string.\"\"\"\n",
    "    encoding = tiktoken.get_encoding(\"cl100k_base\")\n",
    "    num_tokens = len(encoding.encode(text))\n",
    "    return num_tokens\n",
    "\n",
    "def get_txt_from_pdf(pdf_files,filter_ref = False, combine=False):\n",
    "    \"\"\"Convert pdf files to dataframe\"\"\"\n",
    "    # Create an empty list to store the data\n",
    "    data = []\n",
    "    # Iterate over the PDF\n",
    "    for pdf in pdf_files:\n",
    "        # Fetch the PDF content from the pdf\n",
    "        with open(pdf, 'rb') as pdf_content:\n",
    "            # Create a PDF reader object\n",
    "            pdf_reader = PyPDF2.PdfReader(pdf_content)\n",
    "            # Iterate over all the pages in the PDF\n",
    "            for page_num in range(len(pdf_reader.pages)):\n",
    "                page = pdf_reader.pages[page_num] # Extract the text from the current page\n",
    "                page_text = page.extract_text()\n",
    "                words = page_text.split() # Split the page text into individual words\n",
    "                page_text_join = ' '.join(words) # Join the words back together with a single space between each word\n",
    "\n",
    "                if filter_ref: #filter the reference at the end\n",
    "                    page_text_join = remove_ref(page_text_join)\n",
    "\n",
    "                page_len = len(page_text_join)\n",
    "                div_len = page_len // 4 # Divide the page into 4 parts\n",
    "                page_parts = [page_text_join[i*div_len:(i+1)*div_len] for i in range(4)]\n",
    "            \n",
    "                min_tokens = 40\n",
    "                for i, page_part in enumerate(page_parts):\n",
    "                    if count_tokens(page_part) > min_tokens:\n",
    "                        # Append the data to the list\n",
    "                        data.append({\n",
    "                            'file name': pdf,\n",
    "                            'page number': page_num + 1,\n",
    "                            'page section': i+1,\n",
    "                            'content': page_part,\n",
    "                            'tokens': count_tokens(page_part)\n",
    "                        })\n",
    "    # Create a DataFrame from the data\n",
    "    df = pd.DataFrame(data)\n",
    "    if combine:\n",
    "        df = combine_section(df)\n",
    "    return df\n",
    "\n",
    "\n",
    "def remove_ref(pdf_text):\n",
    "    \"\"\"This function removes reference section from a given PDF text. It uses regular expressions to find the index of the words to be filtered out.\"\"\"\n",
    "    # Regular expression pattern for the words to be filtered out\n",
    "    pattern = r'(REFERENCES|Acknowledgment|ACKNOWLEDGMENT)'\n",
    "    match = re.search(pattern, pdf_text)\n",
    "\n",
    "    if match:\n",
    "        # If a match is found, remove everything after the match\n",
    "        start_index = match.start()\n",
    "        clean_text = pdf_text[:start_index].strip()\n",
    "    else:\n",
    "        # Define a list of regular expression patterns for references\n",
    "        reference_patterns = [\n",
    "            '\\[[\\d\\w]{1,3}\\].+?[\\d]{3,5}\\.','\\[[\\d\\w]{1,3}\\].+?[\\d]{3,5};','\\([\\d\\w]{1,3}\\).+?[\\d]{3,5}\\.','\\[[\\d\\w]{1,3}\\].+?[\\d]{3,5},',\n",
    "            '\\([\\d\\w]{1,3}\\).+?[\\d]{3,5},','\\[[\\d\\w]{1,3}\\].+?[\\d]{3,5}','[\\d\\w]{1,3}\\).+?[\\d]{3,5}\\.','[\\d\\w]{1,3}\\).+?[\\d]{3,5}',\n",
    "            '\\([\\d\\w]{1,3}\\).+?[\\d]{3,5}','^[\\w\\d,\\.– ;)-]+$',\n",
    "        ]\n",
    "\n",
    "        # Find and remove matches with the first eight patterns\n",
    "        for pattern in reference_patterns[:8]:\n",
    "            matches = re.findall(pattern, pdf_text, flags=re.S)\n",
    "            pdf_text = re.sub(pattern, '', pdf_text) if len(matches) > 500 and matches.count('.') < 2 and matches.count(',') < 2 and not matches[-1].isdigit() else pdf_text\n",
    "\n",
    "        # Split the text into lines\n",
    "        lines = pdf_text.split('\\n')\n",
    "\n",
    "        # Strip each line and remove matches with the last two patterns\n",
    "        for i, line in enumerate(lines):\n",
    "            lines[i] = line.strip()\n",
    "            for pattern in reference_patterns[7:]:\n",
    "                matches = re.findall(pattern, lines[i])\n",
    "                lines[i] = re.sub(pattern, '', lines[i]) if len(matches) > 500 and len(re.findall('\\d', matches)) < 8 and len(set(matches)) > 10 and matches.count(',') < 2 and len(matches) > 20 else lines[i]\n",
    "\n",
    "        # Join the lines back together, excluding any empty lines\n",
    "        clean_text = '\\n'.join([line for line in lines if line])\n",
    "\n",
    "    return clean_text\n",
    "\n",
    "      \n",
    "def combine_section(df):\n",
    "    \"\"\"Merge sections, page numbers, add up content, and tokens based on the pdf name.\"\"\"\n",
    "    aggregated_df = df.groupby('file name').agg({\n",
    "        'content': aggregate_content,\n",
    "        'tokens': aggregate_tokens\n",
    "    }).reset_index()\n",
    "\n",
    "    return aggregated_df\n",
    "\n",
    "\n",
    "def aggregate_content(series):\n",
    "    \"\"\"Join all elements in the series with a space separator. \"\"\"\n",
    "    return ' '.join(series)\n",
    "\n",
    "\n",
    "def aggregate_tokens(series):\n",
    "    \"\"\"Sum all elements in the series.\"\"\"\n",
    "    return series.sum()\n",
    "\n",
    "\n",
    "def extract_title(file_name):\n",
    "    \"\"\"Extract the main part of the file name. \"\"\"\n",
    "    title = file_name.split('_')[0]\n",
    "    return title.rstrip('.pdf')\n",
    "\n",
    "\n",
    "def combine_main_SI(df):\n",
    "    \"\"\"Create a new column with the main part of the file name, group the DataFrame by the new column, \n",
    "    and aggregate the content and tokens.\"\"\"\n",
    "    df['main_part'] = df['file name'].apply(extract_title)\n",
    "    merged_df = df.groupby('main_part').agg({\n",
    "        'content': ''.join,\n",
    "        'tokens': sum\n",
    "    }).reset_index()\n",
    "\n",
    "    return merged_df.rename(columns={'main_part': 'file name'})\n",
    "\n",
    "\n",
    "def df_to_csv(df, file_name):\n",
    "    \"\"\"Write a DataFrame to a CSV file.\"\"\"\n",
    "    df.to_csv(file_name, index=False, escapechar='\\\\')\n",
    "\n",
    "\n",
    "def csv_to_df(file_name):\n",
    "    \"\"\"Read a CSV file into a DataFrame.\"\"\"\n",
    "    return pd.read_csv(file_name)\n",
    "\n",
    "\n",
    "\n",
    "def tabulate_condition(df,column_name):\n",
    "    \"\"\"This function converts the text from a ChatGPT conversation into a DataFrame.\n",
    "    It also cleans the DataFrame by dropping additional headers and empty lines.    \"\"\"\n",
    "    \n",
    "    table_text = df[column_name].str.cat(sep='\\n')\n",
    "\n",
    "    # Remove leading and trailing whitespace\n",
    "    table_text = table_text.strip()\n",
    "    \n",
    "    # Split the table into rows\n",
    "    rows = table_text.split('\\n')\n",
    "\n",
    "    # Extract the header row and the divider row\n",
    "    header_row, divider_row, *data_rows = rows\n",
    "\n",
    "    # Extract column names from the header row\n",
    "\n",
    "    column_names = [\n",
    "        'catalyst name', 'catalyst mass', 'GHSV / mL gcat-1 hr-1', 'flow / mL min-1',\n",
    "        'temperature / degree C', '% CH4', '% CH4 conversion', '% C2 selectivity'\n",
    "    ]\n",
    "\n",
    "    # Create a list of dictionaries to store the table data\n",
    "    data = []\n",
    "\n",
    "    dash_pattern = re.compile(r'^-+$')\n",
    "\n",
    "\n",
    "    # Process each data row\n",
    "    for row in data_rows:\n",
    "\n",
    "        \n",
    "\n",
    "        # Split the row into columns\n",
    "        columns = [col.strip() for col in row.split('|') if col.strip()]\n",
    "\n",
    "        if any(dash_pattern.match(col) for col in columns):\n",
    "            continue\n",
    "\n",
    "        if 'N/A' in columns or 'n/a' in columns or 'NA' in columns or 'na' in columns:\n",
    "            continue\n",
    "    \n",
    "        # Create a dictionary to store the row data\n",
    "        row_data = {col_name: col_value for col_name, col_value in zip(column_names, columns)}\n",
    "    \n",
    "        # Append the dictionary to the data list\n",
    "        data.append(row_data)\n",
    "        \n",
    "    df = pd.DataFrame(data)\n",
    "\n",
    "    df = df.drop_duplicates()\n",
    "\n",
    "    \n",
    "    \"\"\"Make df clean by drop additional header and empty lines \"\"\"\n",
    "    def contains_pattern(s, patterns):\n",
    "        return any(re.search(p, s) for p in patterns)\n",
    "\n",
    "    def drop_rows_with_patterns(df, column_name):\n",
    "        #empty cells, N/A cells and header cells\n",
    "        patterns = [\n",
    "            r'^\\s*$',  # Matches empty cells (only whitespace)\n",
    "            r'N/A', r'n/a', r'\\bNA\\b', r'\\bna\\b',  # Matches 'N/A' in different cases\n",
    "            r'catalyst', r'Catalyst',# Matches headers relevant to OCM data\n",
    "        ]\n",
    "        \n",
    "        mask = df[column_name].apply(lambda x: not contains_pattern(str(x), patterns))\n",
    "        filtered_df = df[mask]\n",
    "    \n",
    "        return filtered_df\n",
    "    \n",
    "    \n",
    "    #drop the repeated header\n",
    "    df = drop_rows_with_patterns(df, 'catalyst name')\n",
    "    \n",
    "    #drop the organic synthesis (where the metal source is N/a)    \n",
    "    #filtered_df = drop_rows_with_patterns(drop_rows_with_patterns(drop_rows_with_patterns(df,'metal source'),'metal amount'),'linker amount') \n",
    "\n",
    "    #drop the N/A rows\n",
    "    #filtered_df = filtered_df.dropna(subset=['metal source','metal amount', 'linker amount'])\n",
    "    filtered_df = df.dropna(how='all')\n",
    "\n",
    "\n",
    "    return filtered_df\n",
    "\n",
    "\n",
    "\n",
    "def split_content(input_string, tokens):\n",
    "    \"\"\"Splits a string into chunks based on a maximum token count. \"\"\"\n",
    "\n",
    "    MAX_TOKENS = tokens\n",
    "    split_strings = []\n",
    "    current_string = \"\"\n",
    "    tokens_so_far = 0\n",
    "\n",
    "    for word in input_string.split():\n",
    "        # Check if adding the next word would exceed the max token limit\n",
    "        if tokens_so_far + count_tokens(word) > MAX_TOKENS:\n",
    "            # If we've reached the max tokens, look for the last dot or newline in the current string\n",
    "            last_dot = current_string.rfind(\".\")\n",
    "            last_newline = current_string.rfind(\"\\n\")\n",
    "\n",
    "            # Find the index to cut the current string\n",
    "            cut_index = max(last_dot, last_newline)\n",
    "\n",
    "            # If there's no dot or newline, we'll just cut at the max tokens\n",
    "            if cut_index == -1:\n",
    "                cut_index = MAX_TOKENS\n",
    "\n",
    "            # Add the substring to the result list and reset the current string and tokens_so_far\n",
    "            split_strings.append(current_string[:cut_index + 1].strip())\n",
    "            current_string = current_string[cut_index + 1:].strip()\n",
    "            tokens_so_far = count_tokens(current_string)\n",
    "\n",
    "        # Add the current word to the current string and update the token count\n",
    "        current_string += \" \" + word\n",
    "        tokens_so_far += count_tokens(word)\n",
    "\n",
    "    # Add the remaining current string to the result list\n",
    "    split_strings.append(current_string.strip())\n",
    "\n",
    "    return split_strings\n",
    "\n",
    "\n",
    "def table_text_clean(text):\n",
    "    \"\"\"Cleans the table string and splits it into lines.\"\"\"\n",
    "\n",
    "    # Pattern to find table starts\n",
    "    pattern = r\"\\|\\s*catalyst name\\s*\\|\\s*catalyst mass\\s*\\|.*\"\n",
    "\n",
    "    # Use re.finditer() to find all instances of the pattern in the string and their starting indexes\n",
    "    matches = [match.start() for match in re.finditer(pattern, text, flags=re.IGNORECASE)]\n",
    "\n",
    "    # Count the number of matches\n",
    "    num_matches = len(matches)\n",
    "\n",
    "    # Base table string\n",
    "    table_string = \"\"\"| catalyst name | catalyst mass | GHSV / mL gcat-1 hr-1 | flow / mL min-1 | temperature / degree C | % CH4 | % CH4 conversion | % C2 selectivity |\\n|---------------|--------------|-----------------------|-----------------|------------------------|-------|------------------|------------------|\\n\"\"\"\n",
    "\n",
    "    if num_matches == 0:  # No table in the answer\n",
    "        print(\"No table found in the text: \" + text)\n",
    "        splited_text = ''\n",
    "\n",
    "    else:  # Split the text based on header\n",
    "        splited_text = ''\n",
    "        for i in range(num_matches):\n",
    "            # Get the relevant table slice\n",
    "            splited = text[matches[i]:matches[i + 1]] if i != (num_matches - 1) else text[matches[i]:]\n",
    "\n",
    "            # Remove the text after last '|'\n",
    "            last_pipe_index = splited.rfind('|')\n",
    "            splited = splited[:last_pipe_index + 1]\n",
    "\n",
    "            # Remove the header and \\------\\\n",
    "            pattern_dash = r\"-(\\s*)\\|\"\n",
    "            match = max(re.finditer(pattern_dash, splited), default=None, key=lambda x: x.start())\n",
    "\n",
    "            if not match:\n",
    "                print(\"'-|' pattern not found.\")\n",
    "            else:\n",
    "                first_pipe_index = match.start()\n",
    "                splited = '\\n' + splited[(first_pipe_index + len('-|\\n|') - 1):]  # Start from \"\\\"\n",
    "\n",
    "            splited_text += splited\n",
    "\n",
    "    table_string = table_string + splited_text\n",
    "    return table_string\n",
    "\n",
    "\n",
    "def add_similarity(df, given_embedding):\n",
    "    \"\"\"Adds a 'similarity' column to a dataframe based on cosine similarity with a given embedding.\"\"\"\n",
    "    def calculate_similarity(embedding):\n",
    "        # Check if embedding is a string and convert it to a list of floats if necessary\n",
    "        if isinstance(embedding, str):\n",
    "            embedding = [float(x) for x in embedding.strip('[]').split(',')]\n",
    "        return cosine_similarity([embedding], [given_embedding])[0][0]\n",
    "\n",
    "    df['similarity'] = df['embedding'].apply(calculate_similarity)\n",
    "    return df\n",
    "\n",
    "\n",
    "def select_top_neighbors(df):\n",
    "    \"\"\"Retains top-10 similarity sections and their neighbors in the dataframe and drops the rest.\"\"\"\n",
    "    # Sort dataframe by 'file name' and 'similarity' in descending order\n",
    "    df.sort_values(['file name', 'similarity'], ascending=[True, False], inplace=True)\n",
    "    \n",
    "    # Group dataframe by 'file name' and select the top 10 rows based on similarity\n",
    "    top_10 = df.groupby('file name').head(10)\n",
    "    \n",
    "    # Add neighboring rows (one above and one below) to the selection\n",
    "    neighbors = [i for index in top_10.index for i in (index - 1, index + 1) if 0 <= i < df.shape[0]]\n",
    "\n",
    "    # Create a new dataframe with only the selected rows\n",
    "    selected_df = df.loc[top_10.index.union(neighbors)]\n",
    "    return selected_df\n",
    "\n",
    "\n",
    "def add_emb(df):\n",
    "    \"\"\"Adds an 'embedding' column to a dataframe using OpenAI API.\"\"\"\n",
    "    openai.api_key = api_key\n",
    "    if 'embedding' in df.columns:\n",
    "        print('The dataframe already has embeddings. Please double check.')\n",
    "        return df\n",
    "\n",
    "    embed_msgs = []\n",
    "    for _, row in df.iterrows():\n",
    "        context = row['content']\n",
    "        context_emb = openai.Embedding.create(model=\"text-embedding-ada-002\", input=context)\n",
    "        embed_msgs.append(context_emb['data'][0]['embedding'])\n",
    "\n",
    "    df = df.copy()\n",
    "    df.loc[:, 'embedding'] = embed_msgs\n",
    "    \n",
    "    return df\n",
    "\n",
    "   \n",
    "'''\n",
    "def model_1(df):\n",
    "    \"\"\"Model 1 will turn text in dataframe to a summarized reaction condition table.The dataframe should have a column \"file name\" and a column \"exp content\".\"\"\"\n",
    "    response_msgs = []\n",
    "\n",
    "    for index, row in df.iterrows():\n",
    "        column1_value = row[df.columns[0]]\n",
    "        column2_value = row['content']\n",
    "\n",
    "        max_tokens = 3000\n",
    "        if count_tokens(column2_value) > max_tokens:\n",
    "            context_list = split_content(column2_value, max_tokens)\n",
    "        else:\n",
    "            context_list = [column2_value]\n",
    "\n",
    "        answers = ''  # Collect answers from chatGPT\n",
    "        for context in context_list:\n",
    "            print(\"Start to analyze paper \" + str(column1_value) )\n",
    "            user_heading = f\"This is a section on OCM from paper {column1_value}\\n\\nContext:\\n{context}\"\n",
    "            user_ending = \"\"\"Q: I will provide you with a section of a main paper and another PDF of supplementary information related to Oxidative Coupling of Methane (OCM). Your task is to extract specific reaction conditions and catalyst information from these documents and present them in a table. Here are the detailed instructions:\n",
    "\n",
    "                            Target Information:\n",
    "\n",
    "                            The information is usually found in the conclusion part, but it may also be located elsewhere in the document. Please search throughout the entire documents to ensure no relevant data is missed.\n",
    "                            Data Format:\n",
    "\n",
    "                            Present the extracted information in a table with 8 columns, all in lowercase:\n",
    "                            | catalyst name | catalyst mass | GHSV / mL gcat-1 hr-1 | flow / mL min-1 | temperature / degree C | % CH4 | % CH4 conversion | % C2 selectivity |\n",
    "                            If any information is not provided or you are unsure, use \"N/A\".\n",
    "                            Keywords and Focus:\n",
    "\n",
    "                            The paper will be about OCM. Focus on extracting the conditions related to the described experiments. If multiple temperatures and performance data are presented, ensure that all extracted conditions belong to a single experimental trial.\n",
    "                            Performance Data Proximity:\n",
    "\n",
    "                            Ensure that the conditions and performance information (such as % CH4 conversion and % C2 selectivity) belong to the same experimental trial. Prioritize extracting complete sets of data rather than scattering the focus across multiple sets.\n",
    "                            Information Extraction:\n",
    "\n",
    "                            Ensure the data includes details such as temperature, methane conversion rates, C2 selectivity.\n",
    "                            If the supplementary information provides additional or missing details, incorporate those into the table as well.\n",
    "                            Document Details:\n",
    "\n",
    "                            You will be provided with two PDFs: one of the main paper and another of supplementary information.\n",
    "                            Please ensure accuracy and completeness when filling out the table, ensuring that all information belongs to a single experimental trial and excludes yield data. And, important!!!,only answer the table nothing else please.\n",
    "                            A:\"\"\"   \n",
    "\n",
    "            attempts = 3\n",
    "            while attempts > 0:\n",
    "                try:\n",
    "                    response = client.chat.completions.create(\n",
    "                        model='gpt-4o',\n",
    "                        messages=[{\n",
    "                            \"role\": \"system\",\n",
    "                            \"content\": \"\"\"Answer the question as truthfully as possible using the provided context,\n",
    "                                        and if the answer is not contained within the text below, say \"N/A\" \"\"\"\n",
    "                        },\n",
    "                            {\"role\": \"user\", \"content\": user_heading + user_ending}]\n",
    "                    )\n",
    "                    answer_str = response.choices[0].message.content\n",
    "                    if not answer_str.lower().startswith(\"n/a\"):\n",
    "                        answers += '\\n' + answer_str\n",
    "                    break\n",
    "                except Exception as e:\n",
    "                    attempts -= 1\n",
    "                    if attempts <= 0:\n",
    "                        print(f\"Error: Failed to process paper {column1_value}. Skipping. (model 1)\")\n",
    "                        break\n",
    "                    print(f\"Error: {str(e)}. Retrying in 60 seconds. {attempts} attempts remaining. (model 1)\")\n",
    "                    time.sleep(60)\n",
    "\n",
    "        response_msgs.append(answers)\n",
    "    df = df.copy()\n",
    "    df.loc[:, 'summarized'] = response_msgs\n",
    "    return df\n",
    "\n",
    "'''\n",
    "def model_2(df):\n",
    "    \"\"\"Model 2 has two parts. First, it asks ChatGPT to identify the experiment section,\n",
    "    then it combines the results\"\"\"\n",
    "\n",
    "    response_msgs = []\n",
    "    \n",
    "    prev_paper_name = None  # Initialize the variable. For message printing purpose\n",
    "    total_pages = df.groupby(df.columns[0])[df.columns[1]].max() #  For message printing purpose\n",
    "    \n",
    "    for _, row in df.iterrows():\n",
    "        paper_name = row[df.columns[0]]\n",
    "        page_number = row[df.columns[1]]\n",
    "        # Only print the message when the paper name changes\n",
    "        if paper_name != prev_paper_name:\n",
    "            print(f'Processing paper: {paper_name}. Total pages: {total_pages[paper_name]}')\n",
    "            prev_paper_name = paper_name\n",
    "\n",
    "        context = row['content']\n",
    "\n",
    "        user_msg1 = \"\"\"\n",
    "        \n",
    "          \"\"\"\n",
    "    \n",
    "        user_msg2 = \"\"\"\n",
    "        Question: Does the section contain any conditions for OCM reaction in this list | catalyst name | catalyst mass | GHSV / mL gcat-1 hr-1 | flow / mL min-1 | temperature / degree C | % CH4 | % CH4 conversion | % C2 selectivity |?\n",
    "        Only provide the answer in 'Yes' or 'No'.Rather 'false yes' no 'false no' please.\n",
    "        \"\"\"\n",
    "\n",
    "        attempts = 3\n",
    "        while attempts > 0:\n",
    "            try:\n",
    "                response = client.chat.completions.create(\n",
    "                    model='gpt-4o',\n",
    "                    messages=[\n",
    "                        {\"role\": \"system\", \"content\": \"Does the section contain at least one conditions for catalyst for OCM reaction in this list | catalyst name | catalyst mass | GHSV / mL gcat-1 hr-1 | flow / mL min-1 | temperature / degree C | % CH4 | % CH4 conversion | % C2 selectivity |?Only provide the answer in 'Yes' or 'No'.Rather 'false yes' no 'false no' please.\"},\n",
    "                        {\"role\": \"user\", \"content\": user_msg1 + context + user_msg2}\n",
    "                    ]\n",
    "                )\n",
    "                answers = response.choices[0].message.content\n",
    "                break\n",
    "\n",
    "            except Exception as e:\n",
    "                attempts -= 1\n",
    "                if attempts > 0:\n",
    "                    print(f\"Error: {str(e)}. Retrying in 60 seconds. {attempts} attempts remaining. (model 2)\")\n",
    "                    time.sleep(60)\n",
    "                else:\n",
    "                    print(f\"Error: Failed to process paper {paper_name}. Skipping. (model 2)\")\n",
    "                    answers = \"No\"\n",
    "                    break\n",
    "\n",
    "        response_msgs.append(answers)\n",
    "    df = df.copy()\n",
    "    df.loc[:,'classification'] = response_msgs\n",
    "\n",
    "\n",
    "    # The following section creates a new dataframe after applying some transformations to the old dataframe\n",
    "    # Create a boolean mask for rows where 'results' starts with 'No'\n",
    "    mask_no = df[\"classification\"].str.startswith(\"No\")\n",
    "    # Create a boolean mask for rows where both the row above and below have 'No' in the 'results' column\n",
    "    mask_surrounded_by_no = mask_no.shift(1, fill_value=False) & mask_no.shift(-1, fill_value=False)\n",
    "    # Combine the two masks with an AND operation\n",
    "    mask_to_remove = mask_no & mask_surrounded_by_no\n",
    "    # Invert the mask and filter the DataFrame\n",
    "    filtered_df = df[~mask_to_remove]\n",
    "    #combined\n",
    "    combined_df= combine_main_SI(combine_section(filtered_df ))\n",
    "    #call model 1 to summarized results\n",
    "    add_table_df = model_1(combined_df)\n",
    "    return add_table_df \n",
    "\n",
    "\n",
    "def model_3(df, prompt_choice=\"synthesis\", classfication = True):\n",
    "    \"\"\"Input a dataframe in broken separation, ~300 tokens, separated by pages and sections. This function will filter the unnecessary sections.\"\"\"\n",
    "\n",
    "    # Define the prompt\n",
    "    prompts = {\n",
    "        \"synthesis\": \"Provide a detailed description of the experimental section or synthesis method used in this research. This section should cover essential information such as the compound name (e.g., MOF-5, ZIF-1, Cu(Bpdc), compound 1, etc.), metal source (e.g., ZrCl4, CuCl2, AlCl3, zinc nitrate, iron acetate, etc.), organic linker (e.g., terephthalate acid, H2BDC, H2PZDC, H4Por, etc.), amount (e.g., 25mg, 1.02g, 100mmol, 0.2mol, etc.), solvent (e.g., N,N Dimethylformamide, DMF, DCM, DEF, NMP, water, EtOH, etc.), solvent volume (e.g., 12mL, 100mL, 1L, 0.1mL, etc.), reaction temperature (e.g., 120°C, 293K, 100C, room temperature, reflux, etc.), and reaction time (e.g., 120h, 1 day, 1d, 1h, 0.5h, 30min, a week, etc.).\",\n",
    "        \"TGA\": \"\"\"Identify the section discussing thermogravimetric analysis (TGA) and thermal stability. This section typically includes information about weight-loss steps (e.g., 20%, 30%, 29.5%) and a decomposition temperature range (e.g., 450°C, 515°C) or a plateau.\"\"\",\n",
    "        \"sorption\": \"Identify the section discussing nitrogen (N2) sorption, argon sorption, Brunauer-Emmett-Teller (BET) surface area, Langmuir surface area, and porosity. This section typically reports values such as 1000 m2/g, 100 cm3/g STP, and includes pore diameter or pore size expressed in units of Ångströms (Å)\"\n",
    "    }\n",
    "        \n",
    "    #other than \"synthesis\", \"TGA\", \"sorption\"),the prompt choice can be the name of the linker to be searched for.\n",
    "    # If the choice is not one of the predefined ones (\"synthesis\", \"TGA\", \"sorption\"), it defaults to a generic prompt for the linker.\n",
    "    prompt = prompts.get(prompt_choice, f\"Provide the full name of linker ({prompt_choice}) or denoted as {prompt_choice} in chemicals, abstract, introduction or experimental section.\")\n",
    "    \n",
    "    # Create an embedding for the chosen prompt using OpenAI's embedding model\n",
    "    prompt_result = client.Embedding.create(model=\"text-embedding-ada-002\", input=prompt)\n",
    "    # Extract the embedding data from the result\n",
    "    prompt_emb = prompt_result.data[0].embedding\n",
    "\n",
    "    # If the dataframe does not already have an 'embedding' column, add one. This is done by calling the add_emb function on the dataframe\n",
    "    if 'embedding' not in df.columns:\n",
    "        df_with_emb = add_emb(df)\n",
    "    else:\n",
    "        df_with_emb  = df\n",
    "\n",
    "    # Add a 'similarity' column to the dataframe by comparing the embeddings.This is done by calling the add_similarity function on the dataframe and the prompt embedding\n",
    "    df_2 = add_similarity(df_with_emb, prompt_emb)\n",
    "\n",
    "    # Filter the dataframe to only include rows with top similarity and their neighbors\n",
    "    df_3 = select_top_neighbors(df_2)\n",
    "\n",
    "    # If the classification parameter is True, pass the dataframe to model_2 for further processing\n",
    "    if classfication:\n",
    "        return model_2(df_3)\n",
    "\n",
    "    # If the classification parameter is False, return the filtered dataframe as is\n",
    "    return df_3\n",
    "\n",
    "\n",
    "\n",
    "def load_paper(filename):\n",
    "    \"\"\"Crate a dataframe\"\"\"\n",
    "    if os.path.exists(filename):\n",
    "        dataframe = pd.read_csv(filename)\n",
    "        return dataframe\n",
    "    else:\n",
    "        #load pdf names\n",
    "        \n",
    "        with open('pdf_pool.csv', 'r') as file:\n",
    "            reader = csv.reader(file)\n",
    "            pdf_pool = [row[0] for row in reader]\n",
    "        dataframe = get_txt_from_pdf(pdf_pool,combine = False, filter_ref = True)\n",
    "    \n",
    "        #store the dataframe\n",
    "        df_to_csv(dataframe, filename)\n",
    "\n",
    "        \n",
    "def load_paper_emb(filename):\n",
    "    \"\"\"Crate a dataframe that includes embedding information\"\"\"\n",
    "    if os.path.exists(filename):\n",
    "        paper_df_emb  = pd.read_csv(filename)\n",
    "        paper_df_emb['embedding'] = paper_df_emb['embedding'].apply(ast.literal_eval)\n",
    "        \n",
    "    else: #load paper and create embedding\n",
    "        paper_df_emb = add_emb(load_paper())\n",
    "    #store embedding to csv\n",
    "        df_to_csv(paper_df_emb, filename)\n",
    "    \n",
    "    return paper_df_emb\n",
    "\n",
    "\n",
    "def check_system(syn_df, paper_df, paper_df_emb):\n",
    "    \"\"\"Check if the data is correctly loaded\"\"\"\n",
    "    # check if openai.api_key is not placeholder\n",
    "    if openai.api_key  == \"Add Your OpenAI API KEY Here.\":\n",
    "        print(\"Error: Please replace openai.api_key with your actual key.\")\n",
    "        return False\n",
    "\n",
    "    # check if 'content' column exists in syn_df\n",
    "    if 'content' not in syn_df.columns:\n",
    "        print(\"Error: 'content' column is missing in syn_df.\")\n",
    "        return False\n",
    "\n",
    "    # check if 'paper_df' has at least four columns\n",
    "    expected_columns = ['file name', 'page number', 'page section', 'content']\n",
    "    if not all(col in paper_df.columns for col in expected_columns):\n",
    "        print(\"Error: 'paper_df' should have these columns: 'file name', 'page number', 'page section', 'content'.\")\n",
    "        return False\n",
    "\n",
    "    # check if 'embedding' column exists in paper_df_emb\n",
    "    if 'embedding' not in paper_df_emb.columns:\n",
    "        print(\"Error: 'embedding' column is missing in paper_df_emb.\")\n",
    "        return False\n",
    "\n",
    "    print(\"All checks passed.\")\n",
    "    return True\n",
    "\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "e693c9d9",
   "metadata": {},
   "outputs": [],
   "source": [
    "def model_1(df):\n",
    "    \"\"\"Model 1 will turn text in dataframe to a summarized reaction condition table.\n",
    "    The dataframe should have a column 'file name' and a column 'content'.\"\"\"\n",
    "    \n",
    "    max_tokens = 3000\n",
    "    aggregated_context = \"\"\n",
    "    file_names = df[df.columns[0]].unique()\n",
    "    \n",
    "    for index, row in df.iterrows():\n",
    "        column1_value = row[df.columns[0]]\n",
    "        column2_value = row['content']\n",
    "\n",
    "        if count_tokens(column2_value) > max_tokens:\n",
    "            context_list = split_content(column2_value, max_tokens)\n",
    "        else:\n",
    "            context_list = [column2_value]\n",
    "\n",
    "        aggregated_context += \"\\n\".join(context_list) + \"\\n\"\n",
    "    \n",
    "    # Split the aggregated context into chunks if it exceeds max_tokens\n",
    "    context_list = split_content(aggregated_context, max_tokens) if count_tokens(aggregated_context) > max_tokens else [aggregated_context]\n",
    "\n",
    "    response_msgs = []\n",
    "    for context in context_list:\n",
    "        user_heading = f\"This is a combined section on OCM from papers {', '.join(file_names)}\\n\\nContext:\\n{context}\"\n",
    "        user_ending = \"\"\"Q: I will provide you with sections of main papers and supplementary information related to Oxidative Coupling of Methane (OCM). Your task is to extract specific reaction conditions and catalyst information from these documents and present them in a table. Here are the detailed instructions:\n",
    "\n",
    "                        Target Information:\n",
    "\n",
    "                        The information is usually found in the conclusion part, but it may also be located elsewhere in the document. Please search throughout the entire documents to ensure no relevant data is missed.\n",
    "                        Data Format:\n",
    "\n",
    "                        Present the extracted information in a table with 8 columns, all in lowercase:\n",
    "                        | catalyst name | catalyst mass | GHSV / mL gcat-1 hr-1 | flow / mL min-1 | temperature / degree C | % CH4 | % CH4 conversion | % C2 selectivity |\n",
    "                        If any information is not provided or you are unsure, use \"N/A\".\n",
    "                        Keywords and Focus:\n",
    "\n",
    "                        The paper will be about OCM. Focus on extracting the conditions related to the described experiments. If multiple temperatures and performance data are presented, ensure that all extracted conditions belong to a single experimental trial.\n",
    "                        Performance Data Proximity:\n",
    "\n",
    "                        Ensure that the conditions and performance information (such as % CH4 conversion and % C2 selectivity) belong to the same experimental trial. Prioritize extracting complete sets of data rather than scattering the focus across multiple sets.\n",
    "                        Information Extraction:\n",
    "\n",
    "                        Ensure the data includes details such as temperature, methane conversion rates, C2 selectivity.\n",
    "                        If the supplementary information provides additional or missing details, incorporate those into the table as well.\n",
    "                        Document Details:\n",
    "\n",
    "                        You will be provided with two PDFs: one of the main paper and another of supplementary information.\n",
    "                        Please ensure accuracy and completeness when filling out the table, ensuring that all information belongs to a single experimental trial and excludes yield data. And, important!!!, only answer the table nothing else please.\n",
    "                        Important: Do not include any rows that contain only dashes ('------'). Only include rows with actual data.\n",
    "                        A:\"\"\"\n",
    "\n",
    "        attempts = 3\n",
    "        while attempts > 0:\n",
    "            try:\n",
    "                response = client.chat.completions.create(\n",
    "                    model='gpt-4o',\n",
    "                    messages=[{\n",
    "                        \"role\": \"system\",\n",
    "                        \"content\": \"\"\"Answer the question as truthfully as possible using the provided context,\n",
    "                                    and if the answer is not contained within the text below, say \"N/A\" \"\"\"\n",
    "                    },\n",
    "                        {\"role\": \"user\", \"content\": user_heading + user_ending}]\n",
    "                )\n",
    "                answer_str = response.choices[0].message.content\n",
    "                if not answer_str.lower().startswith(\"n/a\"):\n",
    "                    response_msgs.append(answer_str)\n",
    "                    print(answer_str)\n",
    "                break\n",
    "            except Exception as e:\n",
    "                attempts -= 1\n",
    "                if attempts <= 0:\n",
    "                    print(f\"Error: Failed to process combined content. Skipping. (model 1)\")\n",
    "                    response_msgs.append(\"N/A\")\n",
    "                    break\n",
    "                print(f\"Error: {str(e)}. Retrying in 60 seconds. {attempts} attempts remaining. (model 1)\")\n",
    "                time.sleep(60)\n",
    "\n",
    "    df = df.copy()\n",
    "    df['summarized'] = \"\\n\".join(response_msgs)\n",
    "\n",
    "    return df\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "968f0b6c",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "'\\n#Load all dataframes\\nclient = openai(api_key = OPENAI_API_KEY, )\\nsyn_df = pd.read_csv(\"GTHpaper_info.csv\")\\npaper_df=load_paper(\"GTHpaper_parser.csv\")\\npaper_df_emb = load_paper_emb(\"228paper_emb.csv\")\\ncheck_system(syn_df, paper_df, paper_df_emb)\\n\\n#Run for Model 1\\nmodel_1_table = tabulate_condition(model_1(syn_df),\"summarized\")\\n\\n#Run for Model 2\\nmodel_2_table = tabulate_condition(model_2(paper_df),\"summarized\")\\n\\n#Run for Model 3\\nmodel_3_table_2 = tabulate_condition( model_3(paper_df_emb),\"summarized\")\\n'"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "'''\n",
    "#Load all dataframes\n",
    "client = openai(api_key = OPENAI_API_KEY, )\n",
    "syn_df = pd.read_csv(\"GTHpaper_info.csv\")\n",
    "paper_df=load_paper(\"GTHpaper_parser.csv\")\n",
    "paper_df_emb = load_paper_emb(\"228paper_emb.csv\")\n",
    "check_system(syn_df, paper_df, paper_df_emb)\n",
    "\n",
    "#Run for Model 1\n",
    "model_1_table = tabulate_condition(model_1(syn_df),\"summarized\")\n",
    "\n",
    "#Run for Model 2\n",
    "model_2_table = tabulate_condition(model_2(paper_df),\"summarized\")\n",
    "\n",
    "#Run for Model 3\n",
    "model_3_table_2 = tabulate_condition( model_3(paper_df_emb),\"summarized\")\n",
    "'''"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "4f1d97b6",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Processing paper: 1main.pdf. Total pages: 5\n",
      "Processing paper: 1supple.pdf. Total pages: 6\n",
      "| catalyst name | catalyst mass | GHSV / mL gcat-1 hr-1 | flow / mL min-1 | temperature / degree C | % CH4 | % CH4 conversion | % C2 selectivity |\n",
      "|---------------|---------------|------------------------|----------------|-----------------------|--------|-----------------|------------------|\n",
      "| Sr-La2O3 nanofibers | N/A | N/A | N/A | 500 | N/A | 35 | 47 |\n",
      "| Sr-La2O3 nanofibers (8.6 wt% Sr) | N/A | N/A | N/A | 650 | N/A | N/A | 20 |\n",
      "\n",
      "\n",
      "| catalyst name   | catalyst mass | GHSV / mL gcat-1 hr-1 | flow / mL min-1 | temperature / degree C | % CH4 | % CH4 conversion | % C2 selectivity |\n",
      "|-----------------|---------------|-----------------------|-----------------|------------------------|-------|------------------|-----------------|\n",
      "| 8.6 wt% Sr-La2O3 | 0.2 g        | 72000                 | 240             | 500                    | 75    | ~35              | 47              |\n",
      "| Sr-La2O3         | N/A           | N/A                   | N/A             | 650                    | 75    | N/A              | N/A             |\n"
     ]
    }
   ],
   "source": [
    "client = OpenAI(api_key = OPENAI_API_KEY, )\n",
    "paper_df=load_paper(\"GTHpaper_parser.csv\")\n",
    "\n",
    "model_2_answer = model_2(paper_df)\n",
    "\n",
    "\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "2ec120c4",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_to_csv(model_2_answer,\"table_answer.csv\")\n",
    "model_2_table = tabulate_condition(model_2_answer,\"summarized\")\n",
    "table_cleaned = model_2_table.dropna()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "2cd90128",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_to_csv(table_cleaned,\"table.csv\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "670e011b",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "      catalyst name catalyst mass GHSV / mL gcat-1 hr-1 flow / mL min-1  \\\n",
      "3  8.6 wt% Sr-La2O3         0.2 g                 72000             240   \n",
      "\n",
      "  temperature / degree C % CH4 % CH4 conversion % C2 selectivity  \n",
      "3                    500    75              ~35               47  \n"
     ]
    }
   ],
   "source": [
    "print(table_cleaned)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.8.10"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}