[CAI-177] - Docker read dynamic URLs and create the index automatically

apps/chatbot/docker/app.local.Dockerfile

@@ -3,7 +3,15 @@ ARG DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && \
   apt-get install -y \
-  curl
+  curl \
+  wget \
+  zip
+
+RUN wget https://github.com/rphrp1985/selenium_support/raw/main/chrome_114_amd64.deb && \
+  apt-get install -y ./chrome_114_amd64.deb && \
+  wget https://chromedriver.storage.googleapis.com/114.0.5735.90/chromedriver_linux64.zip && \
+  unzip chromedriver_linux64.zip && \
+  mv chromedriver /usr/bin/chromedriver
 
 ENV PYTHONPATH=/app
 

apps/chatbot/docker/compose.yaml

@@ -16,7 +16,7 @@ services:
         condition: service_started
     networks:
       - ntw
-  
+
   dynamodb:
     image: amazon/dynamodb-local:2.5.2
     environment:
@@ -27,7 +27,7 @@ services:
       - "8000:8000"
     networks:
       - ntw
-  
+
   redis:
     image: redis/redis-stack:7.2.0-v13
     ports:

apps/chatbot/docker/docker-compose-build-local.sh

@@ -0,0 +1,2 @@
+#!/bin/bash
+docker compose --env-file .env -f docker/compose.yaml -p chatbot build

apps/chatbot/docker/docker-compose-run-create_index.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-docker compose -f docker/compose.yaml -p chatbot run create_index
+docker compose --env-file .env -f docker/compose.yaml -p chatbot run create_index

apps/chatbot/docker/docker-compose-up-api.sh

@@ -1,2 +1,2 @@
 #!/bin/bash
-docker compose -f docker/compose.yaml -p chatbot up api
+docker compose --env-file .env -f docker/compose.yaml -p chatbot up api

apps/chatbot/src/modules/vector_database.py

@@ -10,9 +10,6 @@
 from datetime import datetime
 from bs4 import BeautifulSoup
 from selenium import webdriver
-# from selenium.webdriver.chrome.options import Options
-# from selenium.webdriver.chrome.service import Service
-# from chromedriver_py import binary_path
 from typing import List, Tuple
 
 from llama_index.core import (
@@ -147,22 +144,25 @@ def create_documentation(
     hash_table = {}
     empty_pages = []
 
-    # full_text = ""
-
+    driver_exe_path = "/usr/bin/chromedriver"
+    if os.path.exists(driver_exe_path):
+        driver_service = webdriver.ChromeService(executable_path=driver_exe_path)
+    else:
+        driver_service = None
+    driver_options = webdriver.ChromeOptions()
+    driver_options.add_argument('--headless')
+    driver_options.add_argument('--disable-gpu')
+    driver_options.add_argument('--no-sandbox')
+    driver_options.add_argument('--disable-dev-shm-usage')
+
     for file in tqdm.tqdm(html_files, total=len(html_files), desc="Extracting HTML"):
 
-        # FIX: resolve webdriver.Chrome "self.assert_process_still_running" error in docker
         if file in dynamic_htmls or "/webinars/" in file or "/api/" in file:
-        # if 6 == 9:
             url = file.replace(documentation_dir, f"{website_url}/").replace(".html", "")
-
-            # svc = webdriver.ChromeService(executable_path=binary_path)
-            # service = Service(executable_path=binary_path)
-            # options = webdriver.ChromeOptions()
-            # options.add_argument('--headless=new')
-            # options.add_argument('--no-sandbox')
-            # options.add_argument('user-agent=fake-useragent')
-            driver = webdriver.Chrome() #(service=service, options=options)
+            driver = webdriver.Chrome(
+                options=driver_options,
+                service=driver_service
+            )
 
             driver.get(url)
             time.sleep(5)
@@ -172,7 +172,6 @@ def create_documentation(
             title, text = html2markdown(open(file))
 
         if text is None or text == "" or text == "None" or text=="404\n\n#### Pagina non trovata\n\nLa pagina che stai cercando non esiste":
-            # print(file)
             empty_pages.append(file)
 
         else:
@@ -231,7 +230,7 @@ def build_automerging_index_redis(
             key=key,
             val=value
         )
-    logging.info(f"[vector_database.py - build_automerging_index_redis] hash_table_{INDEX_ID} is now on Redis.")
+    logging.info(f"[vector_database.py - build_automerging_index_redis] hash_table_{NEW_INDEX_ID} is now on Redis.")
 
     logging.info(f"[vector_database.py - build_automerging_index_redis] Creating index {NEW_INDEX_ID} ...")
     nodes = Settings.node_parser.get_nodes_from_documents(documents)