Skip to content

Commit

Permalink
Add CI
Browse files Browse the repository at this point in the history
  • Loading branch information
Rai220 committed Oct 10, 2024
1 parent 51f9902 commit 4f356d9
Show file tree
Hide file tree
Showing 15 changed files with 1,065 additions and 3 deletions.
93 changes: 93 additions & 0 deletions .github/actions/poetry_setup/action.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,93 @@
# An action for setting up poetry install with caching.
# Using a custom action since the default action does not
# take poetry install groups into account.
# Action code from:
# https://github.com/actions/setup-python/issues/505#issuecomment-1273013236
name: poetry-install-with-caching
description: Poetry install with support for caching of dependency groups.

inputs:
python-version:
description: Python version, supporting MAJOR.MINOR only
required: true

poetry-version:
description: Poetry version
required: true

cache-key:
description: Cache key to use for manual handling of caching
required: true

working-directory:
description: Directory whose poetry.lock file should be cached
required: true

runs:
using: composite
steps:
- uses: actions/setup-python@v5
name: Setup python ${{ inputs.python-version }}
id: setup-python
with:
python-version: ${{ inputs.python-version }}

- uses: actions/cache@v4
id: cache-bin-poetry
name: Cache Poetry binary - Python ${{ inputs.python-version }}
env:
SEGMENT_DOWNLOAD_TIMEOUT_MIN: "1"
with:
path: |
/opt/pipx/venvs/poetry
# This step caches the poetry installation, so make sure it's keyed on the poetry version as well.
key: bin-poetry-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-${{ inputs.poetry-version }}

- name: Refresh shell hashtable and fixup softlinks
if: steps.cache-bin-poetry.outputs.cache-hit == 'true'
shell: bash
env:
POETRY_VERSION: ${{ inputs.poetry-version }}
PYTHON_VERSION: ${{ inputs.python-version }}
run: |
set -eux
# Refresh the shell hashtable, to ensure correct `which` output.
hash -r
# `actions/cache@v3` doesn't always seem able to correctly unpack softlinks.
# Delete and recreate the softlinks pipx expects to have.
rm /opt/pipx/venvs/poetry/bin/python
cd /opt/pipx/venvs/poetry/bin
ln -s "$(which "python$PYTHON_VERSION")" python
chmod +x python
cd /opt/pipx_bin/
ln -s /opt/pipx/venvs/poetry/bin/poetry poetry
chmod +x poetry
# Ensure everything got set up correctly.
/opt/pipx/venvs/poetry/bin/python --version
/opt/pipx_bin/poetry --version
- name: Install poetry
if: steps.cache-bin-poetry.outputs.cache-hit != 'true'
shell: bash
env:
POETRY_VERSION: ${{ inputs.poetry-version }}
PYTHON_VERSION: ${{ inputs.python-version }}
# Install poetry using the python version installed by setup-python step.
run: pipx install "poetry==$POETRY_VERSION" --python '${{ steps.setup-python.outputs.python-path }}' --verbose

- name: Restore pip and poetry cached dependencies
uses: actions/cache@v4
env:
SEGMENT_DOWNLOAD_TIMEOUT_MIN: "4"
WORKDIR: ${{ inputs.working-directory == '' && '.' || inputs.working-directory }}
with:
path: |
~/.cache/pip
~/.cache/pypoetry/virtualenvs
~/.cache/pypoetry/cache
~/.cache/pypoetry/artifacts
${{ env.WORKDIR }}/.venv
key: py-deps-${{ runner.os }}-${{ runner.arch }}-py-${{ inputs.python-version }}-poetry-${{ inputs.poetry-version }}-${{ inputs.cache-key }}-${{ hashFiles(format('{0}/**/poetry.lock', env.WORKDIR)) }}
32 changes: 32 additions & 0 deletions .github/scripts/check_diff.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
import json
import sys

DIRS = {
"libs/langchain_gigachat",
}

if __name__ == "__main__":
files = sys.argv[1:]
dirs_to_run = set()

if len(files) == 300:
# max diff length is 300 files - there are likely files missing
raise ValueError("Max diff reached. Please manually run CI on changed libs.")

for file in files:
if any(
file.startswith(dir_)
for dir_ in (
".github/workflows",
".github/tools",
".github/actions",
".github/scripts/check_diff.py",
)
):
dirs_to_run.update(DIRS)
elif "libs/langchain_gigachat" in file:
dirs_to_run.update({"libs/langchain_gigachat"})
else:
pass
json_output = json.dumps(list(dirs_to_run))
print(f"dirs-to-run={json_output}")
85 changes: 85 additions & 0 deletions .github/scripts/get_min_versions.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,85 @@
import sys

if sys.version_info >= (3, 11):
import tomllib
else:
# for python 3.10 and below, which doesnt have stdlib tomllib
import tomli as tomllib

from packaging.version import parse as parse_version
import re

MIN_VERSION_LIBS = ["langchain-core"]

SKIP_IF_PULL_REQUEST = ["langchain-core"]


def get_min_version(version: str) -> str:
# base regex for x.x.x with cases for rc/post/etc
# valid strings: https://peps.python.org/pep-0440/#public-version-identifiers
vstring = r"\d+(?:\.\d+){0,2}(?:(?:a|b|rc|\.post|\.dev)\d+)?"
# case ^x.x.x
_match = re.match(f"^\\^({vstring})$", version)
if _match:
return _match.group(1)

# case >=x.x.x,<y.y.y
_match = re.match(f"^>=({vstring}),<({vstring})$", version)
if _match:
_min = _match.group(1)
_max = _match.group(2)
assert parse_version(_min) < parse_version(_max)
return _min

# case x.x.x
_match = re.match(f"^({vstring})$", version)
if _match:
return _match.group(1)

raise ValueError(f"Unrecognized version format: {version}")


def get_min_version_from_toml(toml_path: str, versions_for: str):
# Parse the TOML file
with open(toml_path, "rb") as file:
toml_data = tomllib.load(file)

# Get the dependencies from tool.poetry.dependencies
dependencies = toml_data["tool"]["poetry"]["dependencies"]

# Initialize a dictionary to store the minimum versions
min_versions = {}

# Iterate over the libs in MIN_VERSION_LIBS
for lib in MIN_VERSION_LIBS:
if versions_for == "pull_request" and lib in SKIP_IF_PULL_REQUEST:
# some libs only get checked on release because of simultaneous
# changes
continue
# Check if the lib is present in the dependencies
if lib in dependencies:
# Get the version string
version_string = dependencies[lib]

if isinstance(version_string, dict):
version_string = version_string["version"]

# Use parse_version to get the minimum supported version from version_string
min_version = get_min_version(version_string)

# Store the minimum version in the min_versions dictionary
min_versions[lib] = min_version

return min_versions


if __name__ == "__main__":
# Get the TOML file path from the command line argument
toml_file = sys.argv[1]
versions_for = sys.argv[2]
assert versions_for in ["release", "pull_request"]

# Call the function to get the minimum versions
min_versions = get_min_version_from_toml(toml_file, versions_for)

print(" ".join([f"{lib}=={version}" for lib, version in min_versions.items()]))
54 changes: 54 additions & 0 deletions .github/workflows/_all_ci.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
---
name: langchain CI

on:
workflow_call:
inputs:
working-directory:
required: true
type: string
description: "From which folder this pipeline executes"
workflow_dispatch:
inputs:
working-directory:
required: true
type: choice
default: 'libs/vertexai'
options:
- libs/langchain_gigachat


# If another push to the same PR or branch happens while this workflow is still running,
# cancel the earlier run in favor of the next run.
#
# There's no point in testing an outdated version of the code. GitHub only allows
# a limited number of job runners to be active at the same time, so it's better to cancel
# pointless jobs early so that more useful jobs can run sooner.
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}-${{ inputs.working-directory }}
cancel-in-progress: true

env:
POETRY_VERSION: "1.7.1"

jobs:
lint:
name: "-"
uses: ./.github/workflows/_lint.yml
with:
working-directory: ${{ inputs.working-directory }}
secrets: inherit

test:
name: "-"
uses: ./.github/workflows/_test.yml
with:
working-directory: ${{ inputs.working-directory }}
secrets: inherit

# compile-integration-tests:
# name: "-"
# uses: ./.github/workflows/_compile_integration_test.yml
# with:
# working-directory: ${{ inputs.working-directory }}
# secrets: inherit
39 changes: 39 additions & 0 deletions .github/workflows/_codespell.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
---
name: make spell_check

on:
workflow_call:
inputs:
working-directory:
required: true
type: string
description: "From which folder this pipeline executes"

permissions:
contents: read

jobs:
codespell:
name: (Check for spelling errors)
runs-on: ubuntu-latest

steps:
- name: Checkout
uses: actions/checkout@v4

- name: Install Dependencies
run: |
pip install toml
- name: Extract Ignore Words List
working-directory: ${{ inputs.working-directory }}
run: |
# Use a Python script to extract the ignore words list from pyproject.toml
python ../../.github/workflows/extract_ignored_words_list.py
id: extract_ignore_words

- name: Codespell
uses: codespell-project/actions-codespell@v2
with:
skip: guide_imports.json
ignore_words_list: ${{ steps.extract_ignore_words.outputs.ignore_words_list }}
56 changes: 56 additions & 0 deletions .github/workflows/_compile_integration_test.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,56 @@
name: compile-integration-test

on:
workflow_call:
inputs:
working-directory:
required: true
type: string
description: "From which folder this pipeline executes"

env:
POETRY_VERSION: "1.7.1"

jobs:
build:
defaults:
run:
working-directory: ${{ inputs.working-directory }}
runs-on: ubuntu-latest
strategy:
matrix:
python-version:
- "3.9"
- "3.10"
- "3.11"
name: "poetry run pytest -m compile tests/integration_tests #${{ matrix.python-version }}"
steps:
- uses: actions/checkout@v4

- name: Set up Python ${{ matrix.python-version }} + Poetry ${{ env.POETRY_VERSION }}
uses: "./.github/actions/poetry_setup"
with:
python-version: ${{ matrix.python-version }}
poetry-version: ${{ env.POETRY_VERSION }}
working-directory: ${{ inputs.working-directory }}
cache-key: compile-integration

- name: Install integration dependencies
shell: bash
run: poetry install --with=test_integration,test

- name: Check integration tests compile
shell: bash
run: poetry run pytest -m compile tests/integration_tests

- name: Ensure the tests did not create any additional files
shell: bash
run: |
set -eu
STATUS="$(git status)"
echo "$STATUS"
# grep will exit non-zero if the target message isn't found,
# and `set -e` above will cause the step to fail.
echo "$STATUS" | grep 'nothing to commit, working tree clean'
Loading

0 comments on commit 4f356d9

Please sign in to comment.