Merge branch 'main' into project-RAG-GUI

README.md

@@ -1,9 +1,11 @@
 <h1 align="center">GRAG</h1>
 
-[![License: AGPL v3](https://img.shields.io/badge/License-AGPL_v3-blue.svg)](https://www.gnu.org/licenses/agpl-3.0)
+![GitHub License](https://img.shields.io/github/license/arjbingly/Capstone_5)
+![Linting](https://img.shields.io/github/actions/workflow/status/arjbingly/Capstone_5/linting.yml?label=Docs&labelColor=yellow)
+![GitHub Actions Workflow Status](https://img.shields.io/github/actions/workflow/status/arjbingly/Capstone_5/build_linting.yml?label=Linting)
+![Static Badge](https://img.shields.io/badge/Tests-passing-darggreen)
 ![Static Badge](https://img.shields.io/badge/docstring%20style-google-yellow)
 ![Static Badge](https://img.shields.io/badge/linter%20-ruff-yellow)
-![Linting](https://img.shields.io/github/actions/workflow/status/arjbingly/Capstone_5/ruff_linting.yml?label=Docs&labelColor=yellow)
 ![Static Badge](https://img.shields.io/badge/buildstyle-hatchling-purple?labelColor=white)
 ![Static Badge](https://img.shields.io/badge/codestyle-pyflake-purple?labelColor=white)
 ![GitHub Issues or Pull Requests](https://img.shields.io/github/issues-pr/arjbingly/Capstone_5)

ci/Jenkinsfile

@@ -10,11 +10,11 @@ pipeline {
         CMAKE_ARGS = "-DLLAMA_CUBLAS=on"
         PATH="/usr/local/cuda-12.3/bin:$PATH"
         LD_LIBRARY_PATH="/usr/local/cuda-12.3/lib64:$LD_LIBRARY_PATH"
+        GIT_SSH_COMMAND="ssh -o StrictHostKeyChecking=accept-new"
         }
 
 
     stages {
-
         stage('Checkout') {
             steps {
                     cleanWs()
@@ -80,7 +80,6 @@ pipeline {
                      withChecks('Static Type Checks'){
                          junit 'mypy-report.xml'
                      }
-
                 }
             }
         }
@@ -102,18 +101,40 @@ pipeline {
                      withChecks('Integration Tests'){
                         junit 'pytest-report.xml'
                      }
-
-                     cleanWs(
-                        cleanWhenNotBuilt: false,
-                        deleteDirs: true,
-                        disableDeferredWipeout: true,
-                        notFailBuild: true,
-                        patterns: [[pattern: '.gitignore', type: 'INCLUDE'],
-                                   [pattern: '.propsfile', type: 'EXCLUDE']]
-                             )
                 }
             }
         }
-
+    }
+    post {
+        fixed{
+            sshagent(credentials: ['GithubKey']){
+                sh 'git checkout main'
+                sh 'python3 ci/modify_test_status.py'
+                sh 'git add README.md'
+                sh 'git commit -m "test status updated"'
+                sh 'git push'
+            }       
+
+        }
+        regression{
+            sshagent(credentials: ['GithubKey']){
+                sh 'git checkout main'
+                sh 'python3 ci/modify_test_status.py --fail'
+                sh 'git add README.md'
+                sh 'git commit -m "test status updated"'
+                sh 'git push'
+            }
+
+        }
+        cleanup{
+            cleanWs(
+                cleanWhenNotBuilt: false,
+                deleteDirs: true,
+                disableDeferredWipeout: true,
+                notFailBuild: true,
+                patterns: [[pattern: '.gitignore', type: 'INCLUDE'],
+                           [pattern: '.propsfile', type: 'EXCLUDE']]
+                    )
+        }
     }
 }

ci/modify_test_status.py

@@ -0,0 +1,22 @@
+import argparse
+
+if __name__ == '__main__':
+
+    parser = argparse.ArgumentParser()
+    parser.add_argument('-f', '--fail', action='store_true')
+    args = parser.parse_args()
+    readme_path = 'README.md'  # Path to your README file
+
+    with open(readme_path, 'r') as file:
+        lines = file.readlines()
+
+    for i, line in enumerate(lines):
+        if 'img.shields.io' in line and ('passing' in line or 'failing' in line):
+            if args.fail:
+                lines[i] = line.replace('passing', 'failing').replace('darggreen', 'red')
+            else:
+                lines[i] = line.replace('failing', 'passing').replace('red', 'darggreen')
+            print('README file has been updated.')
+
+    with open(readme_path, 'w') as file:
+        file.writelines(lines)

cookbook/Basic-RAG/BasicRAG_ingest.py

@@ -8,19 +8,16 @@
 
 # from grag.components.vectordb.chroma_client import ChromaClient
 
-# Initialize the client by giving a collection name - DeepLakeClient or ChromaClient
-client = DeepLakeClient(collection_name="arxiv")
-# client = ChromaClient(collection_name="arxiv")
+SYNC = True  # Run synchronously (slow)
+ASYNC = True  # Run asynchronously 
 
-# Initialize the retriever by passing in the client 
-# Note that: if no client is passed the retriever class will take config from the config file.
+client = DeepLakeClient(collection_name="ci_test")
+# client = ChromaClient(collection_name="ci_test")
 retriever = Retriever(vectordb=client)
 
-# The path to the folder with pdfs
-dir_path = Path(__file__).parents[2] / "data/pdf"
+dir_path = Path(__file__).parents[2] / "data/test/pdfs/new_papers"
 
-# Either
-#   1. To run synchronously (slow)
-# retriever.ingest(dir_path)
-#   2. To run asynchronously
-asyncio.run(retriever.aingest(dir_path))
+if SYNC:
+    retriever.ingest(dir_path)
+elif ASYNC:
+    asyncio.run(retriever.aingest(dir_path))

src/grag/components/llm.py

@@ -2,6 +2,7 @@
 
 import os
 from pathlib import Path
+from typing import Optional, Union
 
 import torch
 from langchain.callbacks.manager import CallbackManager
@@ -86,7 +87,7 @@ def model_path(self):
             self.base_dir / self.model_name / f"ggml-model-{self.quantization}.gguf"
         )
 
-    def hf_pipeline(self, is_local=False):
+    def hf_pipeline(self, is_local: Optional[bool] = False):
         """Loads the model using Hugging Face transformers.
 
         Args:
@@ -162,7 +163,8 @@ def llama_cpp(self):
         return llm
 
     def load_model(
-        self, model_name=None, pipeline=None, quantization=None, is_local=None
+        self, model_name: Optional[str] = None, pipeline: Optional[str] = None, quantization: Optional[str] = None,
+        is_local: Optional[bool] = None
     ):
         """Loads the model based on the specified pipeline and model name.