data preprocessing

json.ipynb

@@ -0,0 +1,100 @@
+{
+ "cells": [
+  {
+   "cell_type": "code",
+   "execution_count": 1,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import pdfplumber\n",
+    "import pandas as pd\n",
+    "import re\n",
+    "import json\n",
+    "\n",
+    "headers = [\n",
+    "    r\"Product Name\", r\"Cat No.\", r\"Company\", r\"Synonyms\"\n",
+    "]\n",
+    "\n",
+    "def extract_text_from_pdf(pdf_path):\n",
+    "    with pdfplumber.open(pdf_path) as pdf:\n",
+    "        full_text = []\n",
+    "        for page in pdf.pages:\n",
+    "            page_text = page.extract_text()\n",
+    "            if page_text:\n",
+    "                full_text.append(page_text)\n",
+    "    return \"\\n\".join(full_text)\n",
+    "\n",
+    "def segment_text_based_on_headers(text):\n",
+    "    segments = re.split('|'.join(headers), text, flags=re.IGNORECASE)\n",
+    "    headers_found = re.findall('|'.join(headers), text, flags=re.IGNORECASE)\n",
+    "    \n",
+    "    categorized_text = {}\n",
+    "    for i, header in enumerate(headers_found):\n",
+    "        categorized_text[header.strip()] = segments[i + 1].strip() if i + 1 < len(segments) else \"\"\n",
+    "    \n",
+    "    return categorized_text\n",
+    "\n",
+    "def extract_table_data(pdf_path):\n",
+    "    table_data = []\n",
+    "    with pdfplumber.open(pdf_path) as pdf:\n",
+    "        for page in pdf.pages:\n",
+    "            tables = page.extract_tables()\n",
+    "            for table in tables:\n",
+    "                table_data.extend(table)\n",
+    "    return table_data\n",
+    "\n",
+    "def get_cleaned_text_remove_paragraph(pdf_path):\n",
+    "    with pdfplumber.open(pdf_path) as pdf:\n",
+    "        all_text = []\n",
+    "        for page in pdf.pages:\n",
+    "            page_text = page.extract_text()\n",
+    "            if page_text:\n",
+    "                page_text = re.sub(r\"Page \\d+ of \\d+\", \"\", page_text) \n",
+    "                page_text = re.sub(r\"Specification File\", \"\", page_text)\n",
+    "                page_text = re.sub(r\"(?s)Disclaimer.*?(\\n\\n|\\Z)\", \"\", page_text) \n",
+    "                all_text.append(page_text.strip())\n",
+    "    return \"\\n\\n\".join(all_text)\n",
+    "\n",
+    "def save_pdf_data_to_json(pdf_text_path, pdf_table_path, json_file):\n",
+    "    cleaned_text = get_cleaned_text_remove_paragraph(pdf_text_path)\n",
+    "    segmented_text = segment_text_based_on_headers(cleaned_text)\n",
+    "    table_data = extract_table_data(pdf_table_path)\n",
+    "    \n",
+    "    data = {\n",
+    "        \"text_data\": segmented_text,\n",
+    "        \"table_data\": table_data\n",
+    "    }\n",
+    "    \n",
+    "    with open(json_file, \"w\") as file:\n",
+    "        json.dump(data, file, indent=4)\n",
+    "\n",
+    "pdf_path_text = 'A:/dp/data_preprocessor/data/citric-acid-gran-cert-acs-kg.pdf'\n",
+    "pdf_path_table = 'A:/dp/data_preprocessor/data/phenol-liquid-cert-.pdf'\n",
+    "json_file = 'A:/dp/data_preprocessor/combined_data.json'\n",
+    "\n",
+    "save_pdf_data_to_json(pdf_path_text, pdf_path_table, json_file)"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "Python 3",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.12.4"
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}