From a52c0488c9691a54bbd467020d7f33921f5eff0b Mon Sep 17 00:00:00 2001 From: Saurav Prateek Date: Wed, 18 Dec 2024 22:44:13 +0530 Subject: [PATCH 1/8] Added DocString for method in chat_models module --- .../langchain_google_vertexai/chat_models.py | 23 +++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/libs/vertexai/langchain_google_vertexai/chat_models.py b/libs/vertexai/langchain_google_vertexai/chat_models.py index ebb9464a..73f470a6 100644 --- a/libs/vertexai/langchain_google_vertexai/chat_models.py +++ b/libs/vertexai/langchain_google_vertexai/chat_models.py @@ -442,6 +442,29 @@ def _parse_content(raw_content: str | Dict[Any, Any]) -> Dict[Any, Any]: def _parse_examples(examples: List[BaseMessage]) -> List[InputOutputTextPair]: + """Parse the list of messages. The method expects the messages to be in the order of Human Message followed by an AI Message. + + Args: + examples: The list of messages to be parsed + Returns: + A parsed example list. + Raises: + ValueError: + + - If an odd number of examples are given as argument. + - If an instance of Human Message is not found at every even index in the input examples list. + - If an instance of AI Message is not found at every odd index in the input examples list. + + A valid list examples can be as follows: + .. code-block:: python + examples = [ + HumanMessage(content = "A first sample Human Message"), + AIMessage(content = "A first sample AI Message"), + HumanMessage(content = "A second sample Human Message"), + AIMessage(content = "A second sample AI Message"), + ] + """ + if len(examples) % 2 != 0: raise ValueError( f"Expect examples to have an even amount of messages, got {len(examples)}." From 15d0109a5dca2debd1dd2fb90df0c22f814b155e Mon Sep 17 00:00:00 2001 From: Saurav Prateek Date: Wed, 18 Dec 2024 22:49:41 +0530 Subject: [PATCH 2/8] modified docstring --- libs/vertexai/langchain_google_vertexai/chat_models.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libs/vertexai/langchain_google_vertexai/chat_models.py b/libs/vertexai/langchain_google_vertexai/chat_models.py index 73f470a6..fc4eef78 100644 --- a/libs/vertexai/langchain_google_vertexai/chat_models.py +++ b/libs/vertexai/langchain_google_vertexai/chat_models.py @@ -442,10 +442,10 @@ def _parse_content(raw_content: str | Dict[Any, Any]) -> Dict[Any, Any]: def _parse_examples(examples: List[BaseMessage]) -> List[InputOutputTextPair]: - """Parse the list of messages. The method expects the messages to be in the order of Human Message followed by an AI Message. + """Parse the list of examples. The method expects the examples to be in the order of Human Message followed by an AI Message. Args: - examples: The list of messages to be parsed + examples: The list of examples to be parsed Returns: A parsed example list. Raises: From d566eb63d655ee1183b6c1692dd4abed2d6f0bab Mon Sep 17 00:00:00 2001 From: Saurav Prateek Date: Sun, 22 Dec 2024 15:08:41 +0530 Subject: [PATCH 3/8] Added intro for Vertex AI library --- .../langchain_google_vertexai/__init__.py | 106 ++++++++++++++++++ .../langchain_google_vertexai/chat_models.py | 23 ---- 2 files changed, 106 insertions(+), 23 deletions(-) diff --git a/libs/vertexai/langchain_google_vertexai/__init__.py b/libs/vertexai/langchain_google_vertexai/__init__.py index bd057448..31983658 100644 --- a/libs/vertexai/langchain_google_vertexai/__init__.py +++ b/libs/vertexai/langchain_google_vertexai/__init__.py @@ -1,3 +1,109 @@ +""" +## langchain-google-vertexai + +This module contains the LangChain integrations for Google Cloud generative models. + +## Installation + +```bash +pip install -U langchain-google-vertexai +``` + +## Chat Models + +`ChatVertexAI` class exposes models such as `gemini-pro` and `chat-bison`. + +To use, you should have Google Cloud project with APIs enabled, and configured credentials. Initialize the model as: + +```python +from langchain_google_vertexai import ChatVertexAI + +llm = ChatVertexAI(model_name="gemini-pro") +llm.invoke("Sing a ballad of LangChain.") +``` + +You can use other models, e.g. `chat-bison`: + +```python +from langchain_google_vertexai import ChatVertexAI + +llm = ChatVertexAI(model_name="chat-bison", temperature=0.3) +llm.invoke("Sing a ballad of LangChain.") +``` + +#### Multimodal inputs + +Gemini vision model supports image inputs when providing a single chat message. Example: + +```python +from langchain_core.messages import HumanMessage +from langchain_google_vertexai import ChatVertexAI + +llm = ChatVertexAI(model_name="gemini-pro-vision") +# example +message = HumanMessage( + content=[ + { + "type": "text", + "text": "What's in this image?", + }, # You can optionally provide text parts + {"type": "image_url", "image_url": {"url": "https://picsum.photos/seed/picsum/200/300"}}, + ] +) +llm.invoke([message]) +``` + +The value of `image_url` can be any of the following: + +- A public image URL +- An accessible gcs file (e.g., "gcs://path/to/file.png") +- A base64 encoded image (e.g., `data:image/png;base64,abcd124`) + +## Embeddings + +You can use Google Cloud's embeddings models as: + +```python +from langchain_google_vertexai import VertexAIEmbeddings + +embeddings = VertexAIEmbeddings() +embeddings.embed_query("hello, world!") +``` + +## LLMs + +You can use Google Cloud's generative AI models as Langchain LLMs: + +```python +from langchain_core.prompts import PromptTemplate +from langchain_google_vertexai import ChatVertexAI + +template = \"""Question: {question} + +Answer: Let's think step by step.\""" +prompt = PromptTemplate.from_template(template) + +llm = ChatVertexAI(model_name="gemini-pro") +chain = prompt | llm + +question = "Who was the president of the USA in 1994?" +print(chain.invoke({"question": question})) +``` + +You can use Gemini and Palm models, including code-generations ones: + +```python + +from langchain_google_vertexai import VertexAI + +llm = VertexAI(model_name="code-bison", max_output_tokens=1000, temperature=0.3) + +question = "Write a python function that checks if a string is a valid email address" + +output = llm(question) +``` +""" + from google.cloud.aiplatform_v1beta1.types import ( FunctionCallingConfig, FunctionDeclaration, diff --git a/libs/vertexai/langchain_google_vertexai/chat_models.py b/libs/vertexai/langchain_google_vertexai/chat_models.py index fc4eef78..ebb9464a 100644 --- a/libs/vertexai/langchain_google_vertexai/chat_models.py +++ b/libs/vertexai/langchain_google_vertexai/chat_models.py @@ -442,29 +442,6 @@ def _parse_content(raw_content: str | Dict[Any, Any]) -> Dict[Any, Any]: def _parse_examples(examples: List[BaseMessage]) -> List[InputOutputTextPair]: - """Parse the list of examples. The method expects the examples to be in the order of Human Message followed by an AI Message. - - Args: - examples: The list of examples to be parsed - Returns: - A parsed example list. - Raises: - ValueError: - - - If an odd number of examples are given as argument. - - If an instance of Human Message is not found at every even index in the input examples list. - - If an instance of AI Message is not found at every odd index in the input examples list. - - A valid list examples can be as follows: - .. code-block:: python - examples = [ - HumanMessage(content = "A first sample Human Message"), - AIMessage(content = "A first sample AI Message"), - HumanMessage(content = "A second sample Human Message"), - AIMessage(content = "A second sample AI Message"), - ] - """ - if len(examples) % 2 != 0: raise ValueError( f"Expect examples to have an even amount of messages, got {len(examples)}." From 558ca2c5e0763cc84bfaab6b0ed5a98eebf89888 Mon Sep 17 00:00:00 2001 From: Saurav Prateek Date: Tue, 24 Dec 2024 12:20:41 +0530 Subject: [PATCH 4/8] Resolved lint issue: Line too long --- libs/vertexai/langchain_google_vertexai/__init__.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libs/vertexai/langchain_google_vertexai/__init__.py b/libs/vertexai/langchain_google_vertexai/__init__.py index 31983658..7c0444f1 100644 --- a/libs/vertexai/langchain_google_vertexai/__init__.py +++ b/libs/vertexai/langchain_google_vertexai/__init__.py @@ -13,7 +13,8 @@ `ChatVertexAI` class exposes models such as `gemini-pro` and `chat-bison`. -To use, you should have Google Cloud project with APIs enabled, and configured credentials. Initialize the model as: +To use, you should have Google Cloud project with APIs enabled, and configured +credentials. Initialize the model as: ```python from langchain_google_vertexai import ChatVertexAI From 72c5dad7cf951aeaad7870367d6be847f60e566e Mon Sep 17 00:00:00 2001 From: Saurav Prateek Date: Tue, 7 Jan 2025 18:47:26 +0530 Subject: [PATCH 5/8] Added documentation for MaaS and Vector Stores --- .../langchain_google_vertexai/__init__.py | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/libs/vertexai/langchain_google_vertexai/__init__.py b/libs/vertexai/langchain_google_vertexai/__init__.py index 7c0444f1..2dd016fd 100644 --- a/libs/vertexai/langchain_google_vertexai/__init__.py +++ b/libs/vertexai/langchain_google_vertexai/__init__.py @@ -9,6 +9,40 @@ pip install -U langchain-google-vertexai ``` +## Supported Models (MaaS: Model-as-a-Service) + +1. Llama +2. Mistral + +Integration on Google Cloud Vertex AI Model-as-a-Service. + +For more information, see: + https://cloud.google.com/blog/products/ai-machine-learning/llama-3-1-on-vertex-ai + +#### Setup + +You need to enable a corresponding MaaS model (Google Cloud UI console -> +Vertex AI -> Model Garden -> search for a model you need and click enable) + +You must have the langchain-google-vertexai Python package installed +.. code-block:: bash + + pip install -U langchain-google-vertexai + +And either: + - Have credentials configured for your environment + (gcloud, workload identity, etc...) + - Store the path to a service account JSON file as the + GOOGLE_APPLICATION_CREDENTIALS environment variable + +This codebase uses the google.auth library which first looks for the application +credentials variable mentioned above, and then looks for system-level auth. + +For more information, see: +https://cloud.google.com/docs/authentication/application-default-credentials#GAC +and +https://googleapis.dev/python/google-auth/latest/reference/google.auth.html#module-google.auth. + ## Chat Models `ChatVertexAI` class exposes models such as `gemini-pro` and `chat-bison`. @@ -102,6 +136,17 @@ question = "Write a python function that checks if a string is a valid email address" output = llm(question) + +## Vector Stores + +#### Vector Search Vector Store GCS + +VertexAI VectorStore that handles the search and indexing using Vector Search +and stores the documents in Google Cloud Storage. + +#### Vector Search Vector Store Datastore + +VectorSearch with DatasTore document storage. ``` """ From 975c9c399de4b99d8e5eca810320f7dd49b17617 Mon Sep 17 00:00:00 2001 From: Saurav Prateek Date: Fri, 10 Jan 2025 14:25:11 +0530 Subject: [PATCH 6/8] Added more documentation for integrated models in Model Garden --- .../langchain_google_vertexai/__init__.py | 71 +++++++++++++++++++ 1 file changed, 71 insertions(+) diff --git a/libs/vertexai/langchain_google_vertexai/__init__.py b/libs/vertexai/langchain_google_vertexai/__init__.py index 2dd016fd..3b6f193d 100644 --- a/libs/vertexai/langchain_google_vertexai/__init__.py +++ b/libs/vertexai/langchain_google_vertexai/__init__.py @@ -13,6 +13,8 @@ 1. Llama 2. Mistral +3. Gemma +4. Vision models Integration on Google Cloud Vertex AI Model-as-a-Service. @@ -43,6 +45,75 @@ and https://googleapis.dev/python/google-auth/latest/reference/google.auth.html#module-google.auth. +## Chat Language Model vs Base Language Models + +A "base model" is a large language model trained on a wide variety of text data, +providing a general-purpose foundation for various tasks. + +While a "chat model" is a specialized version of a base model specifically fine-tuned +to excel in conversational interactions, meaning it's optimized to understand and +respond to dialogue within a chat context, often with features like maintaining +conversational context and generating engaging responses. + +## Gemma models + +Gemma is a family of lightweight, state-of-the-art open models built from the same +research and technology used to create the Gemini models. Developed by Google DeepMind +and other teams across Google. + +We currently support the following Gemma models. + +1. GemmaChatLocalHF: Gemma Chat local model from Hugging Face. +2. GemmaChatLocalKaggle - Gemma chat local model from Kaggle. +3. GemmaLocalHF - Gemma Local model from Hugging Face +4. GemmaLocalKaggle - Gemma Local model from Kaggle + +## Vision models + +### Image Captioning + +Generate Captions from Image. Implementation of the Image Captioning model + +The model takes a list of prompts as an input where each prompt must be a +string that represents an image. Currently supported are: + +1. Google Cloud Storage URI +2. B64 encoded string +3. Local file path +4. Remote url + +The model returns Captions generated from every prompt (Image). + +### Image Generation + +Generates images from text prompt. + +The message must be a list of only one element with one part i.e. the user prompt. + +### Visual QnA Chat + +Answers questions about an image. Chat implementation of a visual QnA model. + +The model takes a list of messages. The first message should contain a string +representation of the image. Currently supported are: + +1. Google Cloud Storage URI +2. B64 encoded string +3. Local file path +4. Remote url + +There has to be at least another message with the first question. +The model returns the generated answer for the questions asked. + +### Image Editor + +Given an image and a prompt, edit the image. Currently only supports mask free editing. + +The message must be a list of only one element with two part: + +1. The image as a dictionary: { 'type': 'image_url', 'image_url': {'url': } } +2. The user prompt. + ## Chat Models `ChatVertexAI` class exposes models such as `gemini-pro` and `chat-bison`. From 973fb3461236c4714a45041f867f0b8c99616e8c Mon Sep 17 00:00:00 2001 From: Leonid Kuligin Date: Mon, 13 Jan 2025 16:36:41 +0100 Subject: [PATCH 7/8] adjusted intro --- .../langchain_google_vertexai/__init__.py | 218 ++---------------- 1 file changed, 17 insertions(+), 201 deletions(-) diff --git a/libs/vertexai/langchain_google_vertexai/__init__.py b/libs/vertexai/langchain_google_vertexai/__init__.py index 3b6f193d..a4f45283 100644 --- a/libs/vertexai/langchain_google_vertexai/__init__.py +++ b/libs/vertexai/langchain_google_vertexai/__init__.py @@ -1,37 +1,26 @@ -""" -## langchain-google-vertexai +"""**LangChain Google Generative AI Integration** -This module contains the LangChain integrations for Google Cloud generative models. +This module contains the LangChain integrations for Vertex AI service - Google foundational models, third-party foundational modela available on Vertex Model Garden and. -## Installation +**Supported integrations models** -```bash -pip install -U langchain-google-vertexai -``` - -## Supported Models (MaaS: Model-as-a-Service) - -1. Llama -2. Mistral -3. Gemma -4. Vision models +1. Google's founational models: Gemini family, Codey, embeddings - `ChatVertexAI`, `VertexAI`, `VertexAIEmbeddings`. +2. Other Google's foundational models: Imagen - `VertexAIImageCaptioning`, `VertexAIImageCaptioningChat`, `VertexAIImageEditorChat`, `VertexAIImageGeneratorChat`, `VertexAIVisualQnAChat`. +3. Third-party foundational models available as a an API (mdel-as-a-service) on Vertex Model Garden (Mistral, Llama, Anthropic) - `model_garden.ChatAnthropicVertex`, `model_garden_maas.VertexModelGardenLlama`, `model_garden_maas.VertexModelGardenMistral`. +4. Third-party foundational models deployed on Vertex AI endpoints from Vertex Model Garden or Hugginface - `VertexAIModelGarden`. +5. Gemma deployed on Vertex AI endpoints or locally - `GemmaChatLocalHF`, `GemmaChatLocalKaggle`, `GemmaChatVertexAIModelGarden`, `GemmaLocalHF`, `GemmaLocalKaggle`, `GemmaVertexAIModelGarden`. +5. Vector Search on Vertex AI - `VectorSearchVectorStore`, `VectorSearchVectorStoreDatastore`, `VectorSearchVectorStoreGCS`. +6. Vertex AI evaluators for generative AI - `VertexPairWiseStringEvaluator`, `VertexStringEvaluator`. -Integration on Google Cloud Vertex AI Model-as-a-Service. +Take a look at detailed documentation for each class for further details. -For more information, see: - https://cloud.google.com/blog/products/ai-machine-learning/llama-3-1-on-vertex-ai - -#### Setup - -You need to enable a corresponding MaaS model (Google Cloud UI console -> -Vertex AI -> Model Garden -> search for a model you need and click enable) +**Installation** -You must have the langchain-google-vertexai Python package installed -.. code-block:: bash - - pip install -U langchain-google-vertexai +```bash +pip install -U langchain-google-vertexai +``` -And either: +You need to enable needed Google Cloud APIs (depending on the integration you're using) and set up credentials by either: - Have credentials configured for your environment (gcloud, workload identity, etc...) - Store the path to a service account JSON file as the @@ -45,181 +34,8 @@ and https://googleapis.dev/python/google-auth/latest/reference/google.auth.html#module-google.auth. -## Chat Language Model vs Base Language Models - -A "base model" is a large language model trained on a wide variety of text data, -providing a general-purpose foundation for various tasks. - -While a "chat model" is a specialized version of a base model specifically fine-tuned -to excel in conversational interactions, meaning it's optimized to understand and -respond to dialogue within a chat context, often with features like maintaining -conversational context and generating engaging responses. - -## Gemma models - -Gemma is a family of lightweight, state-of-the-art open models built from the same -research and technology used to create the Gemini models. Developed by Google DeepMind -and other teams across Google. - -We currently support the following Gemma models. - -1. GemmaChatLocalHF: Gemma Chat local model from Hugging Face. -2. GemmaChatLocalKaggle - Gemma chat local model from Kaggle. -3. GemmaLocalHF - Gemma Local model from Hugging Face -4. GemmaLocalKaggle - Gemma Local model from Kaggle - -## Vision models - -### Image Captioning - -Generate Captions from Image. Implementation of the Image Captioning model - -The model takes a list of prompts as an input where each prompt must be a -string that represents an image. Currently supported are: - -1. Google Cloud Storage URI -2. B64 encoded string -3. Local file path -4. Remote url - -The model returns Captions generated from every prompt (Image). - -### Image Generation - -Generates images from text prompt. - -The message must be a list of only one element with one part i.e. the user prompt. - -### Visual QnA Chat - -Answers questions about an image. Chat implementation of a visual QnA model. - -The model takes a list of messages. The first message should contain a string -representation of the image. Currently supported are: - -1. Google Cloud Storage URI -2. B64 encoded string -3. Local file path -4. Remote url - -There has to be at least another message with the first question. -The model returns the generated answer for the questions asked. - -### Image Editor - -Given an image and a prompt, edit the image. Currently only supports mask free editing. - -The message must be a list of only one element with two part: - -1. The image as a dictionary: { 'type': 'image_url', 'image_url': {'url': } } -2. The user prompt. - -## Chat Models - -`ChatVertexAI` class exposes models such as `gemini-pro` and `chat-bison`. - -To use, you should have Google Cloud project with APIs enabled, and configured -credentials. Initialize the model as: -```python -from langchain_google_vertexai import ChatVertexAI - -llm = ChatVertexAI(model_name="gemini-pro") -llm.invoke("Sing a ballad of LangChain.") -``` - -You can use other models, e.g. `chat-bison`: - -```python -from langchain_google_vertexai import ChatVertexAI - -llm = ChatVertexAI(model_name="chat-bison", temperature=0.3) -llm.invoke("Sing a ballad of LangChain.") -``` - -#### Multimodal inputs - -Gemini vision model supports image inputs when providing a single chat message. Example: - -```python -from langchain_core.messages import HumanMessage -from langchain_google_vertexai import ChatVertexAI - -llm = ChatVertexAI(model_name="gemini-pro-vision") -# example -message = HumanMessage( - content=[ - { - "type": "text", - "text": "What's in this image?", - }, # You can optionally provide text parts - {"type": "image_url", "image_url": {"url": "https://picsum.photos/seed/picsum/200/300"}}, - ] -) -llm.invoke([message]) -``` - -The value of `image_url` can be any of the following: - -- A public image URL -- An accessible gcs file (e.g., "gcs://path/to/file.png") -- A base64 encoded image (e.g., `data:image/png;base64,abcd124`) - -## Embeddings - -You can use Google Cloud's embeddings models as: - -```python -from langchain_google_vertexai import VertexAIEmbeddings - -embeddings = VertexAIEmbeddings() -embeddings.embed_query("hello, world!") -``` - -## LLMs - -You can use Google Cloud's generative AI models as Langchain LLMs: - -```python -from langchain_core.prompts import PromptTemplate -from langchain_google_vertexai import ChatVertexAI - -template = \"""Question: {question} - -Answer: Let's think step by step.\""" -prompt = PromptTemplate.from_template(template) - -llm = ChatVertexAI(model_name="gemini-pro") -chain = prompt | llm - -question = "Who was the president of the USA in 1994?" -print(chain.invoke({"question": question})) -``` - -You can use Gemini and Palm models, including code-generations ones: - -```python - -from langchain_google_vertexai import VertexAI - -llm = VertexAI(model_name="code-bison", max_output_tokens=1000, temperature=0.3) - -question = "Write a python function that checks if a string is a valid email address" - -output = llm(question) - -## Vector Stores - -#### Vector Search Vector Store GCS - -VertexAI VectorStore that handles the search and indexing using Vector Search -and stores the documents in Google Cloud Storage. - -#### Vector Search Vector Store Datastore - -VectorSearch with DatasTore document storage. -``` -""" +""" # noqa: E501 from google.cloud.aiplatform_v1beta1.types import ( FunctionCallingConfig, From 0d85706178577cac42ab300c2dd0f1ce540bc9d7 Mon Sep 17 00:00:00 2001 From: Leonid Kuligin Date: Mon, 13 Jan 2025 17:27:12 +0100 Subject: [PATCH 8/8] Update __init__.py --- libs/vertexai/langchain_google_vertexai/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libs/vertexai/langchain_google_vertexai/__init__.py b/libs/vertexai/langchain_google_vertexai/__init__.py index a4f45283..2d03d79f 100644 --- a/libs/vertexai/langchain_google_vertexai/__init__.py +++ b/libs/vertexai/langchain_google_vertexai/__init__.py @@ -2,7 +2,7 @@ This module contains the LangChain integrations for Vertex AI service - Google foundational models, third-party foundational modela available on Vertex Model Garden and. -**Supported integrations models** +**Supported integrations** 1. Google's founational models: Gemini family, Codey, embeddings - `ChatVertexAI`, `VertexAI`, `VertexAIEmbeddings`. 2. Other Google's foundational models: Imagen - `VertexAIImageCaptioning`, `VertexAIImageCaptioningChat`, `VertexAIImageEditorChat`, `VertexAIImageGeneratorChat`, `VertexAIVisualQnAChat`.