Merge branch 'master' into obvec

.github/CODEOWNERS

@@ -0,0 +1,2 @@
+/.github/   @efriis @baskaryan @ccurme
+/libs/packages.yml   @efriis

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,7 +1,7 @@
 Thank you for contributing to LangChain!
 
 - [ ] **PR title**: "package: description"
-  - Where "package" is whichever of langchain, community, core, etc. is being modified. Use "docs: ..." for purely docs changes, "templates: ..." for template changes, "infra: ..." for CI changes.
+  - Where "package" is whichever of langchain, community, core, etc. is being modified. Use "docs: ..." for purely docs changes, "infra: ..." for CI changes.
   - Example: "community: add foobar LLM"
 
 

.github/scripts/check_diff.py

@@ -37,7 +37,6 @@
 PY_312_MAX_PACKAGES = [
     f"libs/partners/{integration}"
     for integration in [
-        "anthropic",
         "chroma",
         "couchbase",
         "huggingface",
@@ -307,7 +306,7 @@ def _get_configs_for_multi_dirs(
                 f"Unknown lib: {file}. check_diff.py likely needs "
                 "an update for this new library!"
             )
-        elif any(file.startswith(p) for p in ["docs/", "templates/", "cookbook/"]):
+        elif any(file.startswith(p) for p in ["docs/", "cookbook/"]):
             if file.startswith("docs/"):
                 docs_edited = True
             dirs_to_run["lint"].add(".")

.github/scripts/get_min_versions.py

@@ -23,6 +23,7 @@
     "langchain-community",
     "langchain",
     "langchain-text-splitters",
+    "numpy",
     "SQLAlchemy",
 ]
 

.github/scripts/prep_api_docs_build.py

@@ -11,7 +11,9 @@
 def load_packages_yaml() -> Dict[str, Any]:
     """Load and parse the packages.yml file."""
     with open("langchain/libs/packages.yml", "r") as f:
-        return yaml.safe_load(f)
+        all_packages = yaml.safe_load(f)
+
+    return {k: v for k, v in all_packages.items() if k["repo"]}
 
 
 def get_target_dir(package_name: str) -> Path:
@@ -23,24 +25,19 @@ def get_target_dir(package_name: str) -> Path:
     return base_path / "partners" / package_name_short
 
 
-def clean_target_directories(packages: Dict[str, Any]) -> None:
+def clean_target_directories(packages: list) -> None:
     """Remove old directories that will be replaced."""
-    for package in packages["packages"]:
-        if package["repo"] != "langchain-ai/langchain":
-            target_dir = get_target_dir(package["name"])
-            if target_dir.exists():
-                print(f"Removing {target_dir}")
-                shutil.rmtree(target_dir)
+    for package in packages:
+
+        target_dir = get_target_dir(package["name"])
+        if target_dir.exists():
+            print(f"Removing {target_dir}")
+            shutil.rmtree(target_dir)
 
 
-def move_libraries(packages: Dict[str, Any]) -> None:
+def move_libraries(packages: list) -> None:
     """Move libraries from their source locations to the target directories."""
-    for package in packages["packages"]:
-        # Skip if it's the main langchain repo or disabled
-        if package["repo"] == "langchain-ai/langchain" or package.get(
-            "disabled", False
-        ):
-            continue
+    for package in packages:
 
         repo_name = package["repo"].split("/")[1]
         source_path = package["path"]
@@ -68,7 +65,14 @@ def main():
     """Main function to orchestrate the library sync process."""
     try:
         # Load packages configuration
-        packages = load_packages_yaml()
+        package_yaml = load_packages_yaml()
+        packages = [
+            p
+            for p in package_yaml["packages"]
+            if not p.get("disabled", False)
+            and p["repo"].startswith("langchain-ai/")
+            and p["repo"] != "langchain-ai/langchain"
+        ]
 
         # Clean target directories
         clean_target_directories(packages)

.github/workflows/_integration_test.yml

@@ -41,12 +41,6 @@ jobs:
         shell: bash
         run: poetry run pip install "boto3<2" "google-cloud-aiplatform<2"
 
-      - name: 'Authenticate to Google Cloud'
-        id: 'auth'
-        uses: google-github-actions/auth@v2
-        with:
-          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
-
       - name: Run integration tests
         shell: bash
         env:
@@ -85,6 +79,7 @@ jobs:
           VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }}
           COHERE_API_KEY: ${{ secrets.COHERE_API_KEY }}
           UPSTAGE_API_KEY: ${{ secrets.UPSTAGE_API_KEY }}
+          XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
         run: |
           make integration_tests
 

.github/workflows/_release.yml

@@ -103,14 +103,19 @@ jobs:
           if [ -z "$PREV_TAG" ]; then
             REGEX="^$PKG_NAME==\\d+\\.\\d+\\.\\d+\$"
             echo $REGEX
-            PREV_TAG=$(git tag --sort=-creatordate | grep -P $REGEX || true | head -1)
+            PREV_TAG=$(git tag --sort=-creatordate | (grep -P $REGEX || true) | head -1)
           fi
 
-          # confirm prev-tag actually exists in git repo with git tag
-          GIT_TAG_RESULT=$(git tag -l "$PREV_TAG")
-          if [ -z "$GIT_TAG_RESULT" ]; then
-            echo "Previous tag $PREV_TAG not found in git repo"
-            exit 1
+          # if PREV_TAG is empty, let it be empty
+          if [ -z "$PREV_TAG" ]; then
+            echo "No previous tag found - first release"
+          else
+            # confirm prev-tag actually exists in git repo with git tag
+            GIT_TAG_RESULT=$(git tag -l "$PREV_TAG")
+            if [ -z "$GIT_TAG_RESULT" ]; then
+              echo "Previous tag $PREV_TAG not found in git repo"
+              exit 1
+            fi
           fi
 
 
@@ -262,12 +267,6 @@ jobs:
           make tests
         working-directory: ${{ inputs.working-directory }}
 
-      - name: 'Authenticate to Google Cloud'
-        id: 'auth'
-        uses: google-github-actions/auth@v2
-        with:
-          credentials_json: '${{ secrets.GOOGLE_CREDENTIALS }}'
-
       - name: Import integration test dependencies
         run: poetry install --with test,test_integration
         working-directory: ${{ inputs.working-directory }}
@@ -309,6 +308,7 @@ jobs:
           VOYAGE_API_KEY: ${{ secrets.VOYAGE_API_KEY }}
           UPSTAGE_API_KEY: ${{ secrets.UPSTAGE_API_KEY }}
           FIREWORKS_API_KEY: ${{ secrets.FIREWORKS_API_KEY }}
+          XAI_API_KEY: ${{ secrets.XAI_API_KEY }}
         run: make integration_tests
         working-directory: ${{ inputs.working-directory }}
 

.github/workflows/api_doc_build.yml

@@ -37,9 +37,9 @@ jobs:
           # Get unique repositories
           REPOS=$(echo "$REPOS_UNSORTED" | sort -u)
           
-          # Checkout each unique repository
+          # Checkout each unique repository that is in langchain-ai org
           for repo in $REPOS; do
-            if [ "$repo" != "langchain-ai/langchain" ]; then
+            if [[ "$repo" != "langchain-ai/langchain" && "$repo" == langchain-ai/* ]]; then
               REPO_NAME=$(echo $repo | cut -d'/' -f2)
               echo "Checking out $repo to $REPO_NAME"
               git clone --depth 1 https://github.com/$repo.git $REPO_NAME

MIGRATE.md

@@ -1,11 +1,11 @@
 # Migrating
 
-Please see the following guides for migratin LangChain code:
+Please see the following guides for migrating LangChain code:
 
 * Migrate to [LangChain v0.3](https://python.langchain.com/docs/versions/v0_3/)
 * Migrate to [LangChain v0.2](https://python.langchain.com/docs/versions/v0_2/)
 * Migrating from [LangChain 0.0.x Chains](https://python.langchain.com/docs/versions/migrating_chains/)
-* Upgrate to [LangGraph Memory](https://python.langchain.com/docs/versions/migrating_memory/)
+* Upgrade to [LangGraph Memory](https://python.langchain.com/docs/versions/migrating_memory/)
 
-The [LangChain CLI](https://python.langchain.com/docs/versions/v0_3/#migrate-using-langchain-cli) can help automatically upgrade your code to use non deprecated imports. 
+The [LangChain CLI](https://python.langchain.com/docs/versions/v0_3/#migrate-using-langchain-cli) can help you automatically upgrade your code to use non-deprecated imports. 
 This will be especially helpful if you're still on either version 0.0.x or 0.1.x of LangChain.

Makefile

@@ -66,12 +66,12 @@ spell_fix:
 
 ## lint: Run linting on the project.
 lint lint_package lint_tests:
-	poetry run ruff check docs templates cookbook
-	poetry run ruff format docs templates cookbook --diff
-	poetry run ruff check --select I docs templates cookbook
-	git grep 'from langchain import' docs/docs templates cookbook | grep -vE 'from langchain import (hub)' && exit 1 || exit 0
+	poetry run ruff check docs cookbook
+	poetry run ruff format docs cookbook cookbook --diff
+	poetry run ruff check --select I docs cookbook
+	git grep 'from langchain import' docs/docs cookbook | grep -vE 'from langchain import (hub)' && exit 1 || exit 0
 
 ## format: Format the project files.
 format format_diff:
-	poetry run ruff format docs templates cookbook
-	poetry run ruff check --select I --fix docs templates cookbook
+	poetry run ruff format docs cookbook
+	poetry run ruff check --select I --fix docs cookbook

README.md

@@ -59,7 +59,8 @@ For these applications, LangChain simplifies the entire application lifecycle:
 
 - **[LangGraph Cloud](https://langchain-ai.github.io/langgraph/cloud/)**: Turn your LangGraph applications into production-ready APIs and Assistants.
 
-![Diagram outlining the hierarchical organization of the LangChain framework, displaying the interconnected parts across multiple layers.](docs/static/svg/langchain_stack_062024.svg "LangChain Architecture Overview")
+![Diagram outlining the hierarchical organization of the LangChain framework, displaying the interconnected parts across multiple layers.](docs/static/svg/langchain_stack_112024.svg#gh-light-mode-only "LangChain Architecture Overview")
+![Diagram outlining the hierarchical organization of the LangChain framework, displaying the interconnected parts across multiple layers.](docs/static/svg/langchain_stack_112024_dark.svg#gh-dark-mode-only "LangChain Architecture Overview")
 
 ## 🧱 What can you build with LangChain?
 
@@ -128,7 +129,7 @@ Please see [here](https://python.langchain.com) for full documentation, which in
 
 - [🦜🛠️ LangSmith](https://docs.smith.langchain.com/): Trace and evaluate your language model applications and intelligent agents to help you move from prototype to production.
 - [🦜🕸️ LangGraph](https://langchain-ai.github.io/langgraph/): Create stateful, multi-actor applications with LLMs. Integrates smoothly with LangChain, but can be used without it.
-- [🦜🏓 LangServe](https://python.langchain.com/docs/langserve): Deploy LangChain runnables and chains as REST APIs.
+- [🦜🕸️ LangGraph Platform](https://langchain-ai.github.io/langgraph/concepts/#langgraph-platform): Deploy LLM applications built with LangGraph into production.
 
 ## 💁 Contributing
 

cookbook/README.md

@@ -62,4 +62,5 @@ Notebook | Description
 [wikibase_agent.ipynb](https://github.com/langchain-ai/langchain/tree/master/cookbook/wikibase_agent.ipynb) | Create a simple wikibase agent that utilizes sparql generation, with testing done on http://wikidata.org.
 [oracleai_demo.ipynb](https://github.com/langchain-ai/langchain/tree/master/cookbook/oracleai_demo.ipynb) | This guide outlines how to utilize Oracle AI Vector Search alongside Langchain for an end-to-end RAG pipeline, providing step-by-step examples. The process includes loading documents from various sources using OracleDocLoader, summarizing them either within or outside the database with OracleSummary, and generating embeddings similarly through OracleEmbeddings. It also covers chunking documents according to specific requirements using Advanced Oracle Capabilities from OracleTextSplitter, and finally, storing and indexing these documents in a Vector Store for querying with OracleVS.
 [rag-locally-on-intel-cpu.ipynb](https://github.com/langchain-ai/langchain/tree/master/cookbook/rag-locally-on-intel-cpu.ipynb) | Perform Retrieval-Augmented-Generation (RAG) on locally downloaded open-source models using langchain and open source tools and execute it on Intel Xeon CPU. We showed an example of how to apply RAG on Llama 2 model and enable it to answer the queries related to Intel Q1 2024 earnings release.
-[visual_RAG_vdms.ipynb](https://github.com/langchain-ai/langchain/tree/master/cookbook/visual_RAG_vdms.ipynb) | Performs Visual Retrieval-Augmented-Generation (RAG) using videos and scene descriptions generated by open source models.
+[visual_RAG_vdms.ipynb](https://github.com/langchain-ai/langchain/tree/master/cookbook/visual_RAG_vdms.ipynb) | Performs Visual Retrieval-Augmented-Generation (RAG) using videos and scene descriptions generated by open source models.
+[contextual_rag.ipynb](https://github.com/langchain-ai/langchain/tree/master/cookbook/contextual_rag.ipynb) | Performs contextual retrieval-augmented generation (RAG) prepending chunk-specific explanatory context to each chunk before embedding.