From ebabbd437415173c1673693b7394db3ca210a5d7 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Wed, 6 Nov 2024 21:29:21 -0500 Subject: [PATCH 01/10] feat: Adding support for Milvus as a Vector database Signed-off-by: Francisco Javier Arceo --- setup.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 5a6581cc85..26347c7893 100644 --- a/setup.py +++ b/setup.py @@ -150,6 +150,8 @@ QDRANT_REQUIRED = ["qdrant-client>=1.12.0"] +MILVUS_REQUIRED = ["pymilvus"] + CI_REQUIRED = ( [ "build", @@ -220,6 +222,7 @@ + OPENTELEMETRY + FAISS_REQUIRED + QDRANT_REQUIRED + + MILVUS_REQUIRED ) DOCS_REQUIRED = CI_REQUIRED @@ -291,7 +294,8 @@ "couchbase": COUCHBASE_REQUIRED, "opentelemetry": OPENTELEMETRY, "faiss": FAISS_REQUIRED, - "qdrant": QDRANT_REQUIRED + "qdrant": QDRANT_REQUIRED, + "milvus": MILVUS_REQUIRED, }, include_package_data=True, license="Apache", From 57646fbe3abd2f9070f894781f551117994ccd76 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Thu, 7 Nov 2024 23:24:16 -0500 Subject: [PATCH 02/10] updated requirements files Signed-off-by: Francisco Javier Arceo --- .../requirements/py3.10-ci-requirements.txt | 64 +++++++++++------- .../requirements/py3.10-requirements.txt | 8 +-- .../requirements/py3.11-ci-requirements.txt | 66 ++++++++++++------- .../requirements/py3.11-requirements.txt | 8 +-- .../requirements/py3.9-ci-requirements.txt | 62 ++++++++++------- .../requirements/py3.9-requirements.txt | 8 +-- 6 files changed, 132 insertions(+), 84 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 9ac4937582..68548f98d9 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -51,7 +51,7 @@ attrs==24.2.0 # aiohttp # jsonschema # referencing -azure-core==1.31.0 +azure-core==1.32.0 # via # azure-identity # azure-storage-blob @@ -65,9 +65,9 @@ babel==2.16.0 # sphinx beautifulsoup4==4.12.3 # via nbconvert -bigtree==0.21.3 +bigtree==0.22.1 # via feast (setup.py) -bleach==6.1.0 +bleach==6.2.0 # via nbconvert boto3==1.35.36 # via @@ -160,7 +160,7 @@ decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.20.2 +deltalake==0.21.0 # via feast (setup.py) deprecation==2.1.0 # via python-keycloak @@ -172,7 +172,7 @@ docker==7.1.0 # via testcontainers docutils==0.19 # via sphinx -duckdb==1.1.2 +duckdb==1.1.3 # via ibis-framework elastic-transport==8.15.1 # via elasticsearch @@ -180,6 +180,8 @@ elasticsearch==8.15.1 # via feast (setup.py) entrypoints==0.4 # via altair +environs==9.5.0 + # via pymilvus exceptiongroup==1.2.2 # via # anyio @@ -220,7 +222,7 @@ google-api-core[grpc]==2.22.0 # google-cloud-core # google-cloud-datastore # google-cloud-storage -google-auth==2.35.0 +google-auth==2.36.0 # via # google-api-core # google-cloud-bigquery @@ -264,7 +266,7 @@ great-expectations==0.18.22 # via feast (setup.py) grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable -grpcio==1.67.0 +grpcio==1.67.1 # via # feast (setup.py) # google-api-core @@ -275,6 +277,7 @@ grpcio==1.67.0 # grpcio-status # grpcio-testing # grpcio-tools + # pymilvus # qdrant-client grpcio-health-checking==1.62.3 # via feast (setup.py) @@ -442,14 +445,18 @@ markupsafe==3.0.2 # jinja2 # nbconvert # werkzeug -marshmallow==3.23.0 - # via great-expectations +marshmallow==3.23.1 + # via + # environs + # great-expectations matplotlib-inline==0.1.7 # via # ipykernel # ipython mdurl==0.1.2 # via markdown-it-py +milvus-lite==2.4.10 + # via pymilvus minio==7.1.0 # via feast (setup.py) mistune==3.0.2 @@ -548,6 +555,7 @@ pandas==2.2.3 # google-cloud-bigquery # great-expectations # ibis-framework + # pymilvus # snowflake-connector-python pandocfilters==1.5.1 # via nbconvert @@ -612,6 +620,7 @@ protobuf==4.25.5 # grpcio-tools # mypy-protobuf # proto-plus + # pymilvus # substrait psutil==5.9.0 # via @@ -677,6 +686,8 @@ pyjwt[crypto]==2.9.0 # msal # singlestoredb # snowflake-connector-python +pymilvus==2.4.9 + # via feast (setup.py) pymssql==2.3.1 # via feast (setup.py) pymysql==1.1.1 @@ -709,7 +720,7 @@ pytest-asyncio==0.23.8 # via feast (setup.py) pytest-benchmark==3.4.1 # via feast (setup.py) -pytest-cov==5.0.0 +pytest-cov==6.0.0 # via feast (setup.py) pytest-env==1.1.3 # via feast (setup.py) @@ -736,7 +747,9 @@ python-dateutil==2.9.0.post0 # pandas # trino python-dotenv==1.0.1 - # via uvicorn + # via + # environs + # uvicorn python-json-logger==2.0.7 # via jupyter-events python-keycloak==4.2.2 @@ -763,7 +776,7 @@ pyzmq==26.2.0 # ipykernel # jupyter-client # jupyter-server -qdrant-client==1.12.0 +qdrant-client==1.12.1 # via feast (setup.py) redis==4.6.0 # via feast (setup.py) @@ -772,7 +785,7 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications # jupyter-events -regex==2024.9.11 +regex==2024.11.6 # via # feast (setup.py) # parsimonious @@ -811,9 +824,9 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rich==13.9.3 +rich==13.9.4 # via ibis-framework -rpds-py==0.20.0 +rpds-py==0.21.0 # via # jsonschema # referencing @@ -823,7 +836,7 @@ ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.7.1 +ruff==0.7.2 # via feast (setup.py) s3transfer==0.10.3 # via boto3 @@ -831,12 +844,13 @@ scipy==1.14.1 # via great-expectations send2trash==1.8.3 # via jupyter-server -setuptools==75.2.0 +setuptools==75.3.0 # via # grpcio-tools # jupyterlab # kubernetes # pip-tools + # pymilvus # singlestoredb singlestoredb==1.7.2 # via feast (setup.py) @@ -844,7 +858,6 @@ six==1.16.0 # via # asttokens # azure-core - # bleach # geomet # happybase # kubernetes @@ -934,10 +947,11 @@ tornado==6.4.1 # jupyterlab # notebook # terminado -tqdm==4.66.6 +tqdm==4.67.0 # via # feast (setup.py) # great-expectations + # milvus-lite traitlets==5.14.3 # via # comm @@ -955,7 +969,7 @@ traitlets==5.14.3 # nbformat trino==0.330.0 # via feast (setup.py) -typeguard==4.4.0 +typeguard==4.4.1 # via feast (setup.py) types-cffi==1.16.0.20240331 # via types-pyopenssl @@ -963,7 +977,7 @@ types-protobuf==3.19.22 # via # feast (setup.py) # mypy-protobuf -types-pymysql==1.1.0.20240524 +types-pymysql==1.1.0.20241103 # via feast (setup.py) types-pyopenssl==24.1.0.20240722 # via types-redis @@ -979,7 +993,7 @@ types-redis==4.6.0.20241004 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==75.2.0.20241025 +types-setuptools==75.3.0.20241105 # via # feast (setup.py) # types-cffi @@ -1017,6 +1031,8 @@ tzlocal==5.2 # via # great-expectations # trino +ujson==5.10.0 + # via pymilvus uri-template==1.3.0 # via jsonschema urllib3==2.2.3 @@ -1060,7 +1076,7 @@ websocket-client==1.8.0 # kubernetes websockets==13.1 # via uvicorn -werkzeug==3.0.6 +werkzeug==3.1.2 # via moto wheel==0.44.0 # via @@ -1074,7 +1090,7 @@ wrapt==1.16.0 # testcontainers xmltodict==0.14.2 # via moto -yarl==1.16.0 +yarl==1.17.1 # via aiohttp zipp==3.20.2 # via importlib-metadata diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index dd2ed6951c..c64480da5d 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -10,7 +10,7 @@ attrs==24.2.0 # via # jsonschema # referencing -bigtree==0.21.3 +bigtree==0.22.1 # via feast (setup.py) certifi==2024.8.30 # via requests @@ -122,7 +122,7 @@ referencing==0.35.1 # jsonschema-specifications requests==2.32.3 # via feast (setup.py) -rpds-py==0.20.0 +rpds-py==0.21.0 # via # jsonschema # referencing @@ -146,9 +146,9 @@ toolz==1.0.0 # via # dask # partd -tqdm==4.66.6 +tqdm==4.67.0 # via feast (setup.py) -typeguard==4.4.0 +typeguard==4.4.1 # via feast (setup.py) typing-extensions==4.12.2 # via diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index feaafa36e3..60706a8909 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -40,7 +40,7 @@ async-lru==2.0.4 # via jupyterlab async-property==0.2.2 # via python-keycloak -async-timeout==4.0.3 +async-timeout==5.0.1 # via redis atpublic==5.0 # via ibis-framework @@ -49,7 +49,7 @@ attrs==24.2.0 # aiohttp # jsonschema # referencing -azure-core==1.31.0 +azure-core==1.32.0 # via # azure-identity # azure-storage-blob @@ -63,9 +63,9 @@ babel==2.16.0 # sphinx beautifulsoup4==4.12.3 # via nbconvert -bigtree==0.21.3 +bigtree==0.22.1 # via feast (setup.py) -bleach==6.1.0 +bleach==6.2.0 # via nbconvert boto3==1.35.36 # via @@ -158,7 +158,7 @@ decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.20.2 +deltalake==0.21.0 # via feast (setup.py) deprecation==2.1.0 # via python-keycloak @@ -170,7 +170,7 @@ docker==7.1.0 # via testcontainers docutils==0.19 # via sphinx -duckdb==1.1.2 +duckdb==1.1.3 # via ibis-framework elastic-transport==8.15.1 # via elasticsearch @@ -178,6 +178,8 @@ elasticsearch==8.15.1 # via feast (setup.py) entrypoints==0.4 # via altair +environs==9.5.0 + # via pymilvus execnet==2.1.1 # via pytest-xdist executing==2.1.0 @@ -213,7 +215,7 @@ google-api-core[grpc]==2.22.0 # google-cloud-core # google-cloud-datastore # google-cloud-storage -google-auth==2.35.0 +google-auth==2.36.0 # via # google-api-core # google-cloud-bigquery @@ -257,7 +259,7 @@ great-expectations==0.18.22 # via feast (setup.py) grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable -grpcio==1.67.0 +grpcio==1.67.1 # via # feast (setup.py) # google-api-core @@ -268,6 +270,7 @@ grpcio==1.67.0 # grpcio-status # grpcio-testing # grpcio-tools + # pymilvus # qdrant-client grpcio-health-checking==1.62.3 # via feast (setup.py) @@ -433,14 +436,18 @@ markupsafe==3.0.2 # jinja2 # nbconvert # werkzeug -marshmallow==3.23.0 - # via great-expectations +marshmallow==3.23.1 + # via + # environs + # great-expectations matplotlib-inline==0.1.7 # via # ipykernel # ipython mdurl==0.1.2 # via markdown-it-py +milvus-lite==2.4.10 + # via pymilvus minio==7.1.0 # via feast (setup.py) mistune==3.0.2 @@ -539,6 +546,7 @@ pandas==2.2.3 # google-cloud-bigquery # great-expectations # ibis-framework + # pymilvus # snowflake-connector-python pandocfilters==1.5.1 # via nbconvert @@ -603,6 +611,7 @@ protobuf==4.25.5 # grpcio-tools # mypy-protobuf # proto-plus + # pymilvus # substrait psutil==5.9.0 # via @@ -668,6 +677,8 @@ pyjwt[crypto]==2.9.0 # msal # singlestoredb # snowflake-connector-python +pymilvus==2.4.9 + # via feast (setup.py) pymssql==2.3.1 # via feast (setup.py) pymysql==1.1.1 @@ -700,7 +711,7 @@ pytest-asyncio==0.23.8 # via feast (setup.py) pytest-benchmark==3.4.1 # via feast (setup.py) -pytest-cov==5.0.0 +pytest-cov==6.0.0 # via feast (setup.py) pytest-env==1.1.3 # via feast (setup.py) @@ -727,7 +738,9 @@ python-dateutil==2.9.0.post0 # pandas # trino python-dotenv==1.0.1 - # via uvicorn + # via + # environs + # uvicorn python-json-logger==2.0.7 # via jupyter-events python-keycloak==4.2.2 @@ -754,7 +767,7 @@ pyzmq==26.2.0 # ipykernel # jupyter-client # jupyter-server -qdrant-client==1.12.0 +qdrant-client==1.12.1 # via feast (setup.py) redis==4.6.0 # via feast (setup.py) @@ -763,7 +776,7 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications # jupyter-events -regex==2024.9.11 +regex==2024.11.6 # via # feast (setup.py) # parsimonious @@ -802,9 +815,9 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rich==13.9.3 +rich==13.9.4 # via ibis-framework -rpds-py==0.20.0 +rpds-py==0.21.0 # via # jsonschema # referencing @@ -814,7 +827,7 @@ ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.7.1 +ruff==0.7.2 # via feast (setup.py) s3transfer==0.10.3 # via boto3 @@ -822,12 +835,13 @@ scipy==1.14.1 # via great-expectations send2trash==1.8.3 # via jupyter-server -setuptools==75.2.0 +setuptools==75.3.0 # via # grpcio-tools # jupyterlab # kubernetes # pip-tools + # pymilvus # singlestoredb singlestoredb==1.7.2 # via feast (setup.py) @@ -835,7 +849,6 @@ six==1.16.0 # via # asttokens # azure-core - # bleach # geomet # happybase # kubernetes @@ -915,10 +928,11 @@ tornado==6.4.1 # jupyterlab # notebook # terminado -tqdm==4.66.6 +tqdm==4.67.0 # via # feast (setup.py) # great-expectations + # milvus-lite traitlets==5.14.3 # via # comm @@ -936,7 +950,7 @@ traitlets==5.14.3 # nbformat trino==0.330.0 # via feast (setup.py) -typeguard==4.4.0 +typeguard==4.4.1 # via feast (setup.py) types-cffi==1.16.0.20240331 # via types-pyopenssl @@ -944,7 +958,7 @@ types-protobuf==3.19.22 # via # feast (setup.py) # mypy-protobuf -types-pymysql==1.1.0.20240524 +types-pymysql==1.1.0.20241103 # via feast (setup.py) types-pyopenssl==24.1.0.20240722 # via types-redis @@ -960,7 +974,7 @@ types-redis==4.6.0.20241004 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==75.2.0.20241025 +types-setuptools==75.3.0.20241105 # via # feast (setup.py) # types-cffi @@ -993,6 +1007,8 @@ tzlocal==5.2 # via # great-expectations # trino +ujson==5.10.0 + # via pymilvus uri-template==1.3.0 # via jsonschema urllib3==2.2.3 @@ -1036,7 +1052,7 @@ websocket-client==1.8.0 # kubernetes websockets==13.1 # via uvicorn -werkzeug==3.0.6 +werkzeug==3.1.2 # via moto wheel==0.44.0 # via @@ -1050,7 +1066,7 @@ wrapt==1.16.0 # testcontainers xmltodict==0.14.2 # via moto -yarl==1.16.0 +yarl==1.17.1 # via aiohttp zipp==3.20.2 # via importlib-metadata diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index c9833ca07b..7d8ee79e05 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -10,7 +10,7 @@ attrs==24.2.0 # via # jsonschema # referencing -bigtree==0.21.3 +bigtree==0.22.1 # via feast (setup.py) certifi==2024.8.30 # via requests @@ -120,7 +120,7 @@ referencing==0.35.1 # jsonschema-specifications requests==2.32.3 # via feast (setup.py) -rpds-py==0.20.0 +rpds-py==0.21.0 # via # jsonschema # referencing @@ -142,9 +142,9 @@ toolz==1.0.0 # via # dask # partd -tqdm==4.66.6 +tqdm==4.67.0 # via feast (setup.py) -typeguard==4.4.0 +typeguard==4.4.1 # via feast (setup.py) typing-extensions==4.12.2 # via diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index 30eab84822..ceae3980f6 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -51,7 +51,7 @@ attrs==24.2.0 # aiohttp # jsonschema # referencing -azure-core==1.31.0 +azure-core==1.32.0 # via # azure-identity # azure-storage-blob @@ -67,9 +67,9 @@ beautifulsoup4==4.12.3 # via nbconvert bidict==0.23.1 # via ibis-framework -bigtree==0.21.3 +bigtree==0.22.1 # via feast (setup.py) -bleach==6.1.0 +bleach==6.2.0 # via nbconvert boto3==1.35.36 # via @@ -162,7 +162,7 @@ decorator==5.1.1 # via ipython defusedxml==0.7.1 # via nbconvert -deltalake==0.20.2 +deltalake==0.21.0 # via feast (setup.py) deprecation==2.1.0 # via python-keycloak @@ -182,6 +182,8 @@ elasticsearch==8.15.1 # via feast (setup.py) entrypoints==0.4 # via altair +environs==9.5.0 + # via pymilvus exceptiongroup==1.2.2 # via # anyio @@ -222,7 +224,7 @@ google-api-core[grpc]==2.22.0 # google-cloud-core # google-cloud-datastore # google-cloud-storage -google-auth==2.35.0 +google-auth==2.36.0 # via # google-api-core # google-cloud-bigquery @@ -266,7 +268,7 @@ great-expectations==0.18.22 # via feast (setup.py) grpc-google-iam-v1==0.13.1 # via google-cloud-bigtable -grpcio==1.67.0 +grpcio==1.67.1 # via # feast (setup.py) # google-api-core @@ -277,6 +279,7 @@ grpcio==1.67.0 # grpcio-status # grpcio-testing # grpcio-tools + # pymilvus # qdrant-client grpcio-health-checking==1.62.3 # via feast (setup.py) @@ -451,14 +454,18 @@ markupsafe==3.0.2 # jinja2 # nbconvert # werkzeug -marshmallow==3.23.0 - # via great-expectations +marshmallow==3.23.1 + # via + # environs + # great-expectations matplotlib-inline==0.1.7 # via # ipykernel # ipython mdurl==0.1.2 # via markdown-it-py +milvus-lite==2.4.10 + # via pymilvus minio==7.1.0 # via feast (setup.py) mistune==3.0.2 @@ -556,6 +563,7 @@ pandas==2.2.3 # google-cloud-bigquery # great-expectations # ibis-framework + # pymilvus # snowflake-connector-python pandocfilters==1.5.1 # via nbconvert @@ -620,6 +628,7 @@ protobuf==4.25.5 # grpcio-tools # mypy-protobuf # proto-plus + # pymilvus # substrait psutil==5.9.0 # via @@ -685,6 +694,8 @@ pyjwt[crypto]==2.9.0 # msal # singlestoredb # snowflake-connector-python +pymilvus==2.4.9 + # via feast (setup.py) pymssql==2.3.1 # via feast (setup.py) pymysql==1.1.1 @@ -717,7 +728,7 @@ pytest-asyncio==0.23.8 # via feast (setup.py) pytest-benchmark==3.4.1 # via feast (setup.py) -pytest-cov==5.0.0 +pytest-cov==6.0.0 # via feast (setup.py) pytest-env==1.1.3 # via feast (setup.py) @@ -744,7 +755,9 @@ python-dateutil==2.9.0.post0 # pandas # trino python-dotenv==1.0.1 - # via uvicorn + # via + # environs + # uvicorn python-json-logger==2.0.7 # via jupyter-events python-keycloak==4.2.2 @@ -771,7 +784,7 @@ pyzmq==26.2.0 # ipykernel # jupyter-client # jupyter-server -qdrant-client==1.12.0 +qdrant-client==1.12.1 # via feast (setup.py) redis==4.6.0 # via feast (setup.py) @@ -780,7 +793,7 @@ referencing==0.35.1 # jsonschema # jsonschema-specifications # jupyter-events -regex==2024.9.11 +regex==2024.11.6 # via # feast (setup.py) # parsimonious @@ -819,9 +832,9 @@ rfc3986-validator==0.1.1 # via # jsonschema # jupyter-events -rich==13.9.3 +rich==13.9.4 # via ibis-framework -rpds-py==0.20.0 +rpds-py==0.21.0 # via # jsonschema # referencing @@ -831,7 +844,7 @@ ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.7.1 +ruff==0.7.2 # via feast (setup.py) s3transfer==0.10.3 # via boto3 @@ -839,12 +852,13 @@ scipy==1.13.1 # via great-expectations send2trash==1.8.3 # via jupyter-server -setuptools==75.2.0 +setuptools==75.3.0 # via # grpcio-tools # jupyterlab # kubernetes # pip-tools + # pymilvus # singlestoredb singlestoredb==1.7.2 # via feast (setup.py) @@ -852,7 +866,6 @@ six==1.16.0 # via # asttokens # azure-core - # bleach # geomet # happybase # kubernetes @@ -942,10 +955,11 @@ tornado==6.4.1 # jupyterlab # notebook # terminado -tqdm==4.66.6 +tqdm==4.67.0 # via # feast (setup.py) # great-expectations + # milvus-lite traitlets==5.14.3 # via # comm @@ -963,7 +977,7 @@ traitlets==5.14.3 # nbformat trino==0.330.0 # via feast (setup.py) -typeguard==4.4.0 +typeguard==4.4.1 # via feast (setup.py) types-cffi==1.16.0.20240331 # via types-pyopenssl @@ -971,7 +985,7 @@ types-protobuf==3.19.22 # via # feast (setup.py) # mypy-protobuf -types-pymysql==1.1.0.20240524 +types-pymysql==1.1.0.20241103 # via feast (setup.py) types-pyopenssl==24.1.0.20240722 # via types-redis @@ -987,7 +1001,7 @@ types-redis==4.6.0.20241004 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==75.2.0.20241025 +types-setuptools==75.3.0.20241105 # via # feast (setup.py) # types-cffi @@ -1027,6 +1041,8 @@ tzlocal==5.2 # via # great-expectations # trino +ujson==5.10.0 + # via pymilvus uri-template==1.3.0 # via jsonschema urllib3==1.26.20 @@ -1071,7 +1087,7 @@ websocket-client==1.8.0 # kubernetes websockets==13.1 # via uvicorn -werkzeug==3.0.6 +werkzeug==3.1.2 # via moto wheel==0.44.0 # via @@ -1085,7 +1101,7 @@ wrapt==1.16.0 # testcontainers xmltodict==0.14.2 # via moto -yarl==1.16.0 +yarl==1.17.1 # via aiohttp zipp==3.20.2 # via importlib-metadata diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index ec46a195c1..2e3ffa9038 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -10,7 +10,7 @@ attrs==24.2.0 # via # jsonschema # referencing -bigtree==0.21.3 +bigtree==0.22.1 # via feast (setup.py) certifi==2024.8.30 # via requests @@ -124,7 +124,7 @@ referencing==0.35.1 # jsonschema-specifications requests==2.32.3 # via feast (setup.py) -rpds-py==0.20.0 +rpds-py==0.21.0 # via # jsonschema # referencing @@ -148,9 +148,9 @@ toolz==1.0.0 # via # dask # partd -tqdm==4.66.6 +tqdm==4.67.0 # via feast (setup.py) -typeguard==4.4.0 +typeguard==4.4.1 # via feast (setup.py) typing-extensions==4.12.2 # via From 744112b003b3cbae190c159a030391051ea42985 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Sun, 10 Nov 2024 23:19:11 -0500 Subject: [PATCH 03/10] adding configuration Signed-off-by: Francisco Javier Arceo --- Makefile | 19 ++++ .../universal/online_store/milvus.py | 49 +++++++++ .../online_store/test_online_retrieval.py | 103 ++++++++++++++++++ 3 files changed, 171 insertions(+) create mode 100644 sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py diff --git a/Makefile b/Makefile index 4f0f887615..71a71e7ae9 100644 --- a/Makefile +++ b/Makefile @@ -340,6 +340,25 @@ test-python-universal-cassandra-no-cloud-providers: not test_snowflake" \ sdk/python/tests + test-python-universal-milvus-online: + PYTHONPATH='.' \ + FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.milvus_repo_configuration \ + PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.milvus\ + python -m pytest -n 8 --integration \ + -k "not test_universal_cli and \ + not test_go_feature_server and \ + not test_feature_logging and \ + not test_reorder_columns and \ + not test_logged_features_validation and \ + not test_lambda_materialization_consistency and \ + not test_offline_write and \ + not test_push_features_to_offline_store and \ + not gcs_registry and \ + not s3_registry and \ + not test_universal_types and \ + not test_snowflake" \ + sdk/python/tests + test-python-universal-singlestore-online: PYTHONPATH='.' \ FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.singlestore_repo_configuration \ diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py b/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py new file mode 100644 index 0000000000..c8f1dcd87b --- /dev/null +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py @@ -0,0 +1,49 @@ +import os +import time +from typing import Any, Dict + +from pymilvus import connections +from testcontainers.core.container import DockerContainer + +from tests.integration.feature_repos.universal.online_store_creator import ( + OnlineStoreCreator, +) + + +class MilvusOnlineStoreCreator(OnlineStoreCreator): + def __init__(self, project_name: str, **kwargs): + super().__init__(project_name) + self.container = DockerContainer("milvusdb/milvus:v2.2.9").with_exposed_ports(19530) + + def create_online_store(self) -> Dict[str, Any]: + self.container.start() + # Wait for Milvus server to be ready + host = "localhost" + port = self.container.get_exposed_port(19530) + + max_attempts = 12 + for attempt in range(1, max_attempts + 1): + try: + print(f"Attempting to connect to Milvus at {host}:{port}, attempt {attempt}") + connections.connect(alias='default', host=host, port=port) + if connections.has_connection(alias='default'): + print("Successfully connected to Milvus") + break + except Exception as e: + print(f"Connection attempt failed: {e}") + time.sleep(5) + else: + raise RuntimeError("Cannot connect to Milvus server after multiple attempts") + + return { + "type": "milvus", + "host": host, + "port": int(port), + "index_type": "IVF_FLAT", + "metric_type": "L2", + "embedding_dim": 128, # Adjust based on your embedding dimension + "vector_enabled": True, + } + + def teardown(self): + self.container.stop() diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index 83184643f3..ffa8aee20a 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -11,12 +11,14 @@ from pandas.testing import assert_frame_equal from feast import FeatureStore, RepoConfig +from feast.infra.online_stores.contrib.milvus import MilvusOnlineStoreConfig from feast.errors import FeatureViewNotFoundException from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import FloatList as FloatListProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import RegistryConfig from feast.utils import _utc_now +from feast.infra.provider import Provider from tests.integration.feature_repos.universal.feature_views import TAGS from tests.utils.cli_repo_creator import CliRunner, get_example_repo @@ -561,3 +563,104 @@ def test_sqlite_vec_import() -> None: """).fetchall() result = [(rowid, round(distance, 2)) for rowid, distance in result] assert result == [(2, 2.39), (1, 2.39)] + +def test_milvus_get_online_documents() -> None: + """ + Test retrieving documents from the online store in local mode. + """ +def test_milvus_get_online_documents() -> None: + """ + Test retrieving documents from the online store in local mode using Milvus. + """ + n = 10 # number of samples - note: we'll actually double it + vector_length = 8 + runner = CliRunner() + with runner.local_repo( + get_example_repo("example_feature_repo_1.py"), "file" + ) as store: + # Configure the online store to use Milvus + new_config = RepoConfig( + project=store.config.project, + registry=store.config.registry, + provider=store.config.provider, + online_store=MilvusOnlineStoreConfig( + type="milvus", + host="localhost", + port=19530, + index_type="IVF_FLAT", + metric_type="L2", + embedding_dim=vector_length, + vector_enabled=True, + ), + entity_key_serialization_version=store.config.entity_key_serialization_version, + ) + store = FeatureStore(config=new_config, repo_path=store.repo_path) + # Apply the new configuration + store.apply([]) + + # Write some data to the feature view + document_embeddings_fv = store.get_feature_view(name="document_embeddings") + + provider: Provider = store._get_provider() + + item_keys = [ + EntityKeyProto( + join_keys=["item_id"], entity_values=[ValueProto.Value(int64_val=i)] + ) + for i in range(n) + ] + data = [] + for item_key in item_keys: + embedding_vector = np.random.random(vector_length).tolist() + data.append( + ( + item_key, + { + "Embeddings": ValueProto.Value( + float_list_val=FloatListProto(val=embedding_vector) + ) + }, + _utc_now(), + _utc_now(), + ) + ) + + provider.online_write_batch( + config=store.config, + table=document_embeddings_fv, + data=data, + progress=None, + ) + + documents_df = pd.DataFrame( + { + "item_id": [i for i in range(n)], + "Embeddings": [ + np.random.random(vector_length).tolist() for _ in range(n) + ], + "event_timestamp": [_utc_now() for _ in range(n)], + } + ) + + store.write_to_online_store( + feature_view_name="document_embeddings", + df=documents_df, + ) + + # For Milvus, get the collection and check the number of entities + collection = provider._online_store._get_collection( + store.config, document_embeddings_fv + ) + record_count = collection.num_entities + assert record_count == len(data) + documents_df.shape[0] + + query_embedding = np.random.random(vector_length).tolist() + + # Retrieve online documents using Milvus + result = store.retrieve_online_documents( + feature="document_embeddings:Embeddings", query=query_embedding, top_k=3 + ).to_dict() + + assert "Embeddings" in result + assert "distance" in result + assert len(result["distance"]) == 3 From 7e196cbda192595ce72ce81d94931b0bdf146701 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Mon, 11 Nov 2024 22:01:43 -0500 Subject: [PATCH 04/10] updated Signed-off-by: Francisco Javier Arceo --- Makefile | 2 +- .../milvus_online_store/__init__.py | 0 .../milvus_online_store/milvus.py | 366 ++++++++++++++++++ .../milvus_repo_configuration.py | 10 + .../universal/online_store/milvus.py | 17 +- .../online_store/test_online_retrieval.py | 9 +- 6 files changed, 391 insertions(+), 13 deletions(-) create mode 100644 sdk/python/feast/infra/online_stores/milvus_online_store/__init__.py create mode 100644 sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py create mode 100644 sdk/python/feast/infra/online_stores/milvus_online_store/milvus_repo_configuration.py diff --git a/Makefile b/Makefile index 71a71e7ae9..1ea73ba16e 100644 --- a/Makefile +++ b/Makefile @@ -342,7 +342,7 @@ test-python-universal-cassandra-no-cloud-providers: test-python-universal-milvus-online: PYTHONPATH='.' \ - FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.milvus_repo_configuration \ + FULL_REPO_CONFIGS_MODULE=sdk.python.feast.infra.online_stores.milvus_online_store.milvus_repo_configuration \ PYTEST_PLUGINS=sdk.python.tests.integration.feature_repos.universal.online_store.milvus\ python -m pytest -n 8 --integration \ -k "not test_universal_cli and \ diff --git a/sdk/python/feast/infra/online_stores/milvus_online_store/__init__.py b/sdk/python/feast/infra/online_stores/milvus_online_store/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py new file mode 100644 index 0000000000..757b6469a0 --- /dev/null +++ b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py @@ -0,0 +1,366 @@ +from datetime import datetime +from typing import Any, Callable, Dict, List, Literal, Optional, Sequence, Tuple + +from pydantic import StrictStr +from pymilvus import ( + Collection, + CollectionSchema, + DataType, + FieldSchema, + connections, +) + +from feast import Entity +from feast.feature_view import FeatureView +from feast.infra.infra_object import InfraObject +from feast.infra.key_encoding_utils import ( + deserialize_entity_key, + serialize_entity_key, +) +from feast.infra.online_stores.online_store import OnlineStore +from feast.infra.online_stores.vector_store import VectorStoreConfig +from feast.protos.feast.core.InfraObject_pb2 import InfraObject as InfraObjectProto +from feast.protos.feast.core.Registry_pb2 import Registry as RegistryProto +from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto +from feast.protos.feast.types.Value_pb2 import Value as ValueProto +from feast.repo_config import FeastConfigBaseModel, RepoConfig +from feast.utils import ( + _build_retrieve_online_document_record, + to_naive_utc, +) + + +class MilvusOnlineStoreConfig(FeastConfigBaseModel, VectorStoreConfig): + """Online store config for Milvus vector database""" + + type: Literal["milvus", "feast.infra.online_stores.milvus.MilvusOnlineStore"] = ( + "milvus" + ) + """Online store type selector""" + + host: StrictStr = "localhost" + """Hostname for Milvus server""" + + port: int = 19530 + """Port for Milvus server""" + + index_type: str = "IVF_FLAT" + """Index type for Milvus collection""" + + metric_type: str = "L2" + """Distance metric type""" + + embedding_dim: int = 128 + """Dimension of the embedding vectors""" + + vector_enabled: bool = True + """Flag to enable vector search""" + + +class MilvusOnlineStore(OnlineStore): + """ + Milvus implementation of the online store interface. + + Attributes: + _collections: Dictionary to cache Milvus collections. + """ + + _collections: Dict[str, Collection] = {} + + def _connect(self, config: RepoConfig): + connections.connect( + alias="default", + host=config.online_store.host, + port=str(config.online_store.port), + ) + + def _get_collection(self, config: RepoConfig, table: FeatureView) -> Collection: + collection_name = _table_id(config.project, table) + if collection_name not in self._collections: + self._connect(config) + + fields = [ + FieldSchema( + name="pk", dtype=DataType.INT64, is_primary=True, auto_id=True + ), + FieldSchema(name="entity_key", dtype=DataType.VARCHAR, max_length=512), + FieldSchema( + name="feature_name", dtype=DataType.VARCHAR, max_length=256 + ), + FieldSchema(name="value", dtype=DataType.BINARY_VECTOR, dim=8 * 1024), + FieldSchema( + name="vector_value", + dtype=DataType.FLOAT_VECTOR, + dim=config.online_store.embedding_dim, + ), + FieldSchema(name="event_ts", dtype=DataType.INT64), + FieldSchema(name="created_ts", dtype=DataType.INT64), + ] + schema = CollectionSchema( + fields=fields, description="Feast feature view data" + ) + collection = Collection( + name=collection_name, schema=schema, using="default" + ) + if not collection.has_index(): + index_params = { + "index_type": config.online_store.index_type, + "metric_type": config.online_store.metric_type, + "params": {"nlist": 128}, + } + collection.create_index( + field_name="vector_value", index_params=index_params + ) + collection.load() + self._collections[collection_name] = collection + return self._collections[collection_name] + + def online_write_batch( + self, + config: RepoConfig, + table: FeatureView, + data: List[ + Tuple[ + EntityKeyProto, + Dict[str, ValueProto], + datetime, + Optional[datetime], + ] + ], + progress: Optional[Callable[[int], Any]], + ) -> None: + collection = self._get_collection(config, table) + entity_keys = [] + feature_names = [] + values = [] + vector_values = [] + event_tss = [] + created_tss = [] + + for entity_key, values_dict, timestamp, created_ts in data: + entity_key_str = serialize_entity_key( + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version, + ).hex() + timestamp_int = int(to_naive_utc(timestamp).timestamp() * 1e6) + created_ts_int = ( + int(to_naive_utc(created_ts).timestamp() * 1e6) if created_ts else 0 + ) + for feature_name, val in values_dict.items(): + entity_keys.append(entity_key_str) + feature_names.append(feature_name) + values.append(val.SerializeToString()) + if config.online_store.vector_enabled: + vector_values.append(val.float_list_val.val) + else: + vector_values.append([0.0] * config.online_store.embedding_dim) + event_tss.append(timestamp_int) + created_tss.append(created_ts_int) + if progress: + progress(1) + + if entity_keys: + insert_data = { + "entity_key": entity_keys, + "feature_name": feature_names, + "value": values, + "vector_value": vector_values, + "event_ts": event_tss, + "created_ts": created_tss, + } + collection.insert(insert_data) + collection.flush() + + def online_read( + self, + config: RepoConfig, + table: FeatureView, + entity_keys: List[EntityKeyProto], + requested_features: Optional[List[str]] = None, + ) -> List[Tuple[Optional[datetime], Optional[Dict[str, ValueProto]]]]: + collection = self._get_collection(config, table) + results = [] + + for entity_key in entity_keys: + entity_key_str = serialize_entity_key( + entity_key, + entity_key_serialization_version=config.entity_key_serialization_version, + ).hex() + expr = f"entity_key == '{entity_key_str}'" + if requested_features: + features_str = ", ".join([f"'{f}'" for f in requested_features]) + expr += f" && feature_name in [{features_str}]" + + res = collection.query( + expr, + output_fields=["feature_name", "value", "event_ts"], + consistency_level="Strong", + ) + + res_dict = {} + res_ts = None + for r in res: + feature_name = r["feature_name"] + val_bin = r["value"] + val = ValueProto() + val.ParseFromString(val_bin) + res_dict[feature_name] = val + res_ts = datetime.fromtimestamp(r["event_ts"] / 1e6) + if not res_dict: + results.append((None, None)) + else: + results.append((res_ts, res_dict)) + return results + + def update( + self, + config: RepoConfig, + tables_to_delete: Sequence[FeatureView], + tables_to_keep: Sequence[FeatureView], + entities_to_delete: Sequence[Entity], + entities_to_keep: Sequence[Entity], + partial: bool, + ): + self._connect(config) + for table in tables_to_keep: + self._get_collection(config, table) + for table in tables_to_delete: + collection_name = _table_id(config.project, table) + collection = Collection(name=collection_name) + if collection.exists(): + collection.drop() + self._collections.pop(collection_name, None) + + def plan( + self, config: RepoConfig, desired_registry_proto: RegistryProto + ) -> List[InfraObject]: + project = config.project + + infra_objects: List[InfraObject] = [ + MilvusTable( + host=config.online_store.host, + port=config.online_store.port, + name=_table_id(project, FeatureView.from_proto(view)), + ) + for view in [ + *desired_registry_proto.feature_views, + *desired_registry_proto.stream_feature_views, + ] + ] + return infra_objects + + def teardown( + self, + config: RepoConfig, + tables: Sequence[FeatureView], + entities: Sequence[Entity], + ): + self._connect(config) + for table in tables: + collection_name = _table_id(config.project, table) + collection = Collection(name=collection_name) + if collection.exists(): + collection.drop() + self._collections.pop(collection_name, None) + + def retrieve_online_documents( + self, + config: RepoConfig, + table: FeatureView, + requested_feature: str, + embedding: List[float], + top_k: int, + distance_metric: Optional[str] = None, + ) -> List[ + Tuple[ + Optional[datetime], + Optional[EntityKeyProto], + Optional[ValueProto], + Optional[ValueProto], + Optional[ValueProto], + ] + ]: + collection = self._get_collection(config, table) + if not config.online_store.vector_enabled: + raise ValueError("Vector search is not enabled in the online store config") + + search_params = { + "metric_type": distance_metric or config.online_store.metric_type, + "params": {"nprobe": 10}, + } + expr = f"feature_name == '{requested_feature}'" + + results = collection.search( + data=[embedding], + anns_field="vector_value", + param=search_params, + limit=top_k, + expr=expr, + output_fields=["entity_key", "value", "event_ts"], + consistency_level="Strong", + ) + + result_list = [] + for hits in results: + for hit in hits: + entity_key_str = hit.entity.get("entity_key") + val_bin = hit.entity.get("value") + val = ValueProto() + val.ParseFromString(val_bin) + distance = hit.distance + event_ts = datetime.fromtimestamp(hit.entity.get("event_ts") / 1e6) + entity_key = deserialize_entity_key( + bytes.fromhex(entity_key_str), + config.entity_key_serialization_version, + ) + result_list.append( + _build_retrieve_online_document_record( + entity_key, + val.SerializeToString(), + embedding, + distance, + event_ts, + config.entity_key_serialization_version, + ) + ) + return result_list + + +def _table_id(project: str, table: FeatureView) -> str: + return f"{project}_{table.name}" + + +class MilvusTable(InfraObject): + """ + A Milvus collection managed by Feast. + + Attributes: + host: The host of the Milvus server. + port: The port of the Milvus server. + name: The name of the collection. + """ + + host: str + port: int + + def __init__(self, host: str, port: int, name: str): + super().__init__(name) + self.host = host + self.port = port + self._connect() + + def _connect(self): + connections.connect(alias="default", host=self.host, port=str(self.port)) + + def to_infra_object_proto(self) -> InfraObjectProto: + # Implement serialization if needed + pass + + def update(self): + # Implement update logic if needed + pass + + def teardown(self): + collection = Collection(name=self.name) + if collection.exists(): + collection.drop() diff --git a/sdk/python/feast/infra/online_stores/milvus_online_store/milvus_repo_configuration.py b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus_repo_configuration.py new file mode 100644 index 0000000000..90df2a338a --- /dev/null +++ b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus_repo_configuration.py @@ -0,0 +1,10 @@ +from tests.integration.feature_repos.integration_test_repo_config import ( + IntegrationTestRepoConfig, +) +from tests.integration.feature_repos.universal.online_store.milvus import ( + MilvusOnlineStoreCreator, +) + +FULL_REPO_CONFIGS = [ + IntegrationTestRepoConfig(online_store_creator=MilvusOnlineStoreCreator), +] diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py b/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py index c8f1dcd87b..00501f69a9 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py @@ -1,4 +1,3 @@ -import os import time from typing import Any, Dict @@ -13,7 +12,9 @@ class MilvusOnlineStoreCreator(OnlineStoreCreator): def __init__(self, project_name: str, **kwargs): super().__init__(project_name) - self.container = DockerContainer("milvusdb/milvus:v2.2.9").with_exposed_ports(19530) + self.container = DockerContainer("milvusdb/milvus:v2.2.9").with_exposed_ports( + 19530 + ) def create_online_store(self) -> Dict[str, Any]: self.container.start() @@ -24,16 +25,20 @@ def create_online_store(self) -> Dict[str, Any]: max_attempts = 12 for attempt in range(1, max_attempts + 1): try: - print(f"Attempting to connect to Milvus at {host}:{port}, attempt {attempt}") - connections.connect(alias='default', host=host, port=port) - if connections.has_connection(alias='default'): + print( + f"Attempting to connect to Milvus at {host}:{port}, attempt {attempt}" + ) + connections.connect(alias="default", host=host, port=port) + if connections.has_connection(alias="default"): print("Successfully connected to Milvus") break except Exception as e: print(f"Connection attempt failed: {e}") time.sleep(5) else: - raise RuntimeError("Cannot connect to Milvus server after multiple attempts") + raise RuntimeError( + "Cannot connect to Milvus server after multiple attempts" + ) return { "type": "milvus", diff --git a/sdk/python/tests/unit/online_store/test_online_retrieval.py b/sdk/python/tests/unit/online_store/test_online_retrieval.py index ffa8aee20a..602715e5df 100644 --- a/sdk/python/tests/unit/online_store/test_online_retrieval.py +++ b/sdk/python/tests/unit/online_store/test_online_retrieval.py @@ -11,14 +11,14 @@ from pandas.testing import assert_frame_equal from feast import FeatureStore, RepoConfig -from feast.infra.online_stores.contrib.milvus import MilvusOnlineStoreConfig from feast.errors import FeatureViewNotFoundException +from feast.infra.online_stores.milvus_online_store.milvus import MilvusOnlineStoreConfig +from feast.infra.provider import Provider from feast.protos.feast.types.EntityKey_pb2 import EntityKey as EntityKeyProto from feast.protos.feast.types.Value_pb2 import FloatList as FloatListProto from feast.protos.feast.types.Value_pb2 import Value as ValueProto from feast.repo_config import RegistryConfig from feast.utils import _utc_now -from feast.infra.provider import Provider from tests.integration.feature_repos.universal.feature_views import TAGS from tests.utils.cli_repo_creator import CliRunner, get_example_repo @@ -564,10 +564,7 @@ def test_sqlite_vec_import() -> None: result = [(rowid, round(distance, 2)) for rowid, distance in result] assert result == [(2, 2.39), (1, 2.39)] -def test_milvus_get_online_documents() -> None: - """ - Test retrieving documents from the online store in local mode. - """ + def test_milvus_get_online_documents() -> None: """ Test retrieving documents from the online store in local mode using Milvus. From e879a89437b9b72e52f27a86f62abf85b5c750bc Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Thu, 14 Nov 2024 22:46:46 -0500 Subject: [PATCH 05/10] changed things and linted Signed-off-by: Francisco Javier Arceo --- .../universal/online_store/milvus.py | 30 +++++++------------ 1 file changed, 10 insertions(+), 20 deletions(-) diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py b/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py index 00501f69a9..3ccf2b92d6 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py @@ -1,8 +1,7 @@ -import time from typing import Any, Dict -from pymilvus import connections from testcontainers.core.container import DockerContainer +from testcontainers.core.waiting_utils import wait_for_logs from tests.integration.feature_repos.universal.online_store_creator import ( OnlineStoreCreator, @@ -12,7 +11,7 @@ class MilvusOnlineStoreCreator(OnlineStoreCreator): def __init__(self, project_name: str, **kwargs): super().__init__(project_name) - self.container = DockerContainer("milvusdb/milvus:v2.2.9").with_exposed_ports( + self.container = DockerContainer("milvusdb/milvus:v2.4.4").with_exposed_ports( 19530 ) @@ -22,23 +21,14 @@ def create_online_store(self) -> Dict[str, Any]: host = "localhost" port = self.container.get_exposed_port(19530) - max_attempts = 12 - for attempt in range(1, max_attempts + 1): - try: - print( - f"Attempting to connect to Milvus at {host}:{port}, attempt {attempt}" - ) - connections.connect(alias="default", host=host, port=port) - if connections.has_connection(alias="default"): - print("Successfully connected to Milvus") - break - except Exception as e: - print(f"Connection attempt failed: {e}") - time.sleep(5) - else: - raise RuntimeError( - "Cannot connect to Milvus server after multiple attempts" - ) + log_string_to_wait_for = "database system is ready to accept connections" + wait_for_logs( + container=self.container, predicate=log_string_to_wait_for, timeout=10 + ) + init_log_string_to_wait_for = "Milvus DB init process complete" + wait_for_logs( + container=self.container, predicate=init_log_string_to_wait_for, timeout=10 + ) return { "type": "milvus", From b35fd9e5efa45e453510d42d7192f0d529274356 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Fri, 15 Nov 2024 11:11:36 -0500 Subject: [PATCH 06/10] updating testcontainers to include the Milvus container Signed-off-by: Francisco Javier Arceo --- .../requirements/py3.10-ci-requirements.txt | 64 ++++++++++--------- .../requirements/py3.10-requirements.txt | 16 ++--- .../requirements/py3.11-ci-requirements.txt | 60 ++++++++--------- .../requirements/py3.11-requirements.txt | 14 ++-- .../requirements/py3.9-ci-requirements.txt | 60 ++++++++--------- .../requirements/py3.9-requirements.txt | 12 ++-- 6 files changed, 116 insertions(+), 110 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index 68548f98d9..c2b5225207 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -4,7 +4,7 @@ aiobotocore==2.15.2 # via feast (setup.py) aiohappyeyeballs==2.4.3 # via aiohttp -aiohttp==3.10.10 +aiohttp==3.11.2 # via aiobotocore aioitertools==0.12.0 # via aiobotocore @@ -40,7 +40,7 @@ async-lru==2.0.4 # via jupyterlab async-property==0.2.2 # via python-keycloak -async-timeout==4.0.3 +async-timeout==5.0.1 # via # aiohttp # redis @@ -57,7 +57,7 @@ azure-core==1.32.0 # azure-storage-blob azure-identity==1.19.0 # via feast (setup.py) -azure-storage-blob==12.23.1 +azure-storage-blob==12.24.0 # via feast (setup.py) babel==2.16.0 # via @@ -65,7 +65,7 @@ babel==2.16.0 # sphinx beautifulsoup4==4.12.3 # via nbconvert -bigtree==0.22.1 +bigtree==0.22.3 # via feast (setup.py) bleach==6.2.0 # via nbconvert @@ -128,7 +128,7 @@ comm==0.2.2 # ipywidgets couchbase==4.3.2 # via feast (setup.py) -coverage[toml]==7.6.4 +coverage[toml]==7.6.7 # via pytest-cov cryptography==42.0.8 # via @@ -146,15 +146,15 @@ cryptography==42.0.8 # types-redis cython==3.0.11 # via thriftpy2 -dask[dataframe]==2024.10.0 +dask[dataframe]==2024.11.2 # via # feast (setup.py) # dask-expr -dask-expr==1.1.16 +dask-expr==1.1.19 # via dask -db-dtypes==1.3.0 +db-dtypes==1.3.1 # via google-cloud-bigquery -debugpy==1.8.7 +debugpy==1.8.8 # via ipykernel decorator==5.1.1 # via ipython @@ -176,7 +176,7 @@ duckdb==1.1.3 # via ibis-framework elastic-transport==8.15.1 # via elasticsearch -elasticsearch==8.15.1 +elasticsearch==8.16.0 # via feast (setup.py) entrypoints==0.4 # via altair @@ -193,7 +193,7 @@ executing==2.1.0 # via stack-data faiss-cpu==1.9.0 # via feast (setup.py) -fastapi==0.115.4 +fastapi==0.115.5 # via feast (setup.py) fastjsonschema==2.20.0 # via nbformat @@ -213,7 +213,7 @@ fsspec==2024.9.0 # dask geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.22.0 +google-api-core[grpc]==2.23.0 # via # feast (setup.py) # google-cloud-bigquery @@ -232,11 +232,11 @@ google-auth==2.36.0 # google-cloud-datastore # google-cloud-storage # kubernetes -google-cloud-bigquery[pandas]==3.26.0 +google-cloud-bigquery[pandas]==3.27.0 # via feast (setup.py) google-cloud-bigquery-storage==2.27.0 # via feast (setup.py) -google-cloud-bigtable==2.26.0 +google-cloud-bigtable==2.27.0 # via feast (setup.py) google-cloud-core==2.4.1 # via @@ -256,7 +256,7 @@ google-resumable-media==2.7.2 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.65.0 +googleapis-common-protos[grpc]==1.66.0 # via # feast (setup.py) # google-api-core @@ -309,7 +309,7 @@ hiredis==2.4.0 # via feast (setup.py) hpack==4.0.0 # via h2 -httpcore==1.0.6 +httpcore==1.0.7 # via httpx httptools==0.6.4 # via uvicorn @@ -327,7 +327,7 @@ ibis-framework[duckdb]==9.5.0 # ibis-substrait ibis-substrait==4.0.1 # via feast (setup.py) -identify==2.6.1 +identify==2.6.2 # via pre-commit idna==3.10 # via @@ -358,7 +358,7 @@ isodate==0.7.2 # via azure-storage-blob isoduration==20.11.0 # via jsonschema -jedi==0.19.1 +jedi==0.19.2 # via ipython jinja2==3.1.4 # via @@ -375,7 +375,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.25 +json5==0.9.28 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -524,7 +524,7 @@ oauthlib==3.2.2 # via requests-oauthlib overrides==7.7.0 # via jupyter-server -packaging==24.1 +packaging==24.2 # via # build # dask @@ -597,7 +597,9 @@ prometheus-client==0.21.0 prompt-toolkit==3.0.48 # via ipython propcache==0.2.0 - # via yarl + # via + # aiohttp + # yarl proto-plus==1.25.0 # via # google-api-core @@ -630,7 +632,7 @@ psycopg[binary, pool]==3.2.3 # via feast (setup.py) psycopg-binary==3.2.3 # via psycopg -psycopg-pool==3.2.3 +psycopg-pool==3.2.4 # via psycopg ptyprocess==0.7.0 # via @@ -836,7 +838,7 @@ ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.7.2 +ruff==0.7.4 # via feast (setup.py) s3transfer==0.10.3 # via boto3 @@ -844,7 +846,7 @@ scipy==1.14.1 # via great-expectations send2trash==1.8.3 # via jupyter-server -setuptools==75.3.0 +setuptools==75.5.0 # via # grpcio-tools # jupyterlab @@ -921,7 +923,7 @@ tinycss2==1.4.0 # via nbconvert toml==0.10.2 # via feast (setup.py) -tomli==2.0.2 +tomli==2.1.0 # via # build # coverage @@ -993,7 +995,7 @@ types-redis==4.6.0.20241004 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==75.3.0.20241105 +types-setuptools==75.4.0.20241115 # via # feast (setup.py) # types-cffi @@ -1064,7 +1066,7 @@ watchfiles==0.24.0 # via uvicorn wcwidth==0.2.13 # via prompt-toolkit -webcolors==24.8.0 +webcolors==24.11.1 # via jsonschema webencodings==0.5.1 # via @@ -1074,11 +1076,11 @@ websocket-client==1.8.0 # via # jupyter-server # kubernetes -websockets==13.1 +websockets==14.1 # via uvicorn -werkzeug==3.1.2 +werkzeug==3.1.3 # via moto -wheel==0.44.0 +wheel==0.45.0 # via # pip-tools # singlestoredb @@ -1092,5 +1094,5 @@ xmltodict==0.14.2 # via moto yarl==1.17.1 # via aiohttp -zipp==3.20.2 +zipp==3.21.0 # via importlib-metadata diff --git a/sdk/python/requirements/py3.10-requirements.txt b/sdk/python/requirements/py3.10-requirements.txt index c64480da5d..5a96731af3 100644 --- a/sdk/python/requirements/py3.10-requirements.txt +++ b/sdk/python/requirements/py3.10-requirements.txt @@ -10,7 +10,7 @@ attrs==24.2.0 # via # jsonschema # referencing -bigtree==0.22.1 +bigtree==0.22.3 # via feast (setup.py) certifi==2024.8.30 # via requests @@ -25,17 +25,17 @@ cloudpickle==3.1.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask[dataframe]==2024.10.0 +dask[dataframe]==2024.11.2 # via # feast (setup.py) # dask-expr -dask-expr==1.1.16 +dask-expr==1.1.19 # via dask dill==0.3.9 # via feast (setup.py) exceptiongroup==1.2.2 # via anyio -fastapi==0.115.4 +fastapi==0.115.5 # via feast (setup.py) fsspec==2024.10.0 # via dask @@ -74,7 +74,7 @@ numpy==1.26.4 # feast (setup.py) # dask # pandas -packaging==24.1 +packaging==24.2 # via # dask # gunicorn @@ -140,7 +140,7 @@ tenacity==8.5.0 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) -tomli==2.0.2 +tomli==2.1.0 # via mypy toolz==1.0.0 # via @@ -174,7 +174,7 @@ uvloop==0.21.0 # via uvicorn watchfiles==0.24.0 # via uvicorn -websockets==13.1 +websockets==14.1 # via uvicorn -zipp==3.20.2 +zipp==3.21.0 # via importlib-metadata diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index 60706a8909..e77d6f19bf 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -4,7 +4,7 @@ aiobotocore==2.15.2 # via feast (setup.py) aiohappyeyeballs==2.4.3 # via aiohttp -aiohttp==3.10.10 +aiohttp==3.11.2 # via aiobotocore aioitertools==0.12.0 # via aiobotocore @@ -55,7 +55,7 @@ azure-core==1.32.0 # azure-storage-blob azure-identity==1.19.0 # via feast (setup.py) -azure-storage-blob==12.23.1 +azure-storage-blob==12.24.0 # via feast (setup.py) babel==2.16.0 # via @@ -63,7 +63,7 @@ babel==2.16.0 # sphinx beautifulsoup4==4.12.3 # via nbconvert -bigtree==0.22.1 +bigtree==0.22.3 # via feast (setup.py) bleach==6.2.0 # via nbconvert @@ -126,7 +126,7 @@ comm==0.2.2 # ipywidgets couchbase==4.3.2 # via feast (setup.py) -coverage[toml]==7.6.4 +coverage[toml]==7.6.7 # via pytest-cov cryptography==42.0.8 # via @@ -144,15 +144,15 @@ cryptography==42.0.8 # types-redis cython==3.0.11 # via thriftpy2 -dask[dataframe]==2024.10.0 +dask[dataframe]==2024.11.2 # via # feast (setup.py) # dask-expr -dask-expr==1.1.16 +dask-expr==1.1.19 # via dask -db-dtypes==1.3.0 +db-dtypes==1.3.1 # via google-cloud-bigquery -debugpy==1.8.7 +debugpy==1.8.8 # via ipykernel decorator==5.1.1 # via ipython @@ -174,7 +174,7 @@ duckdb==1.1.3 # via ibis-framework elastic-transport==8.15.1 # via elasticsearch -elasticsearch==8.15.1 +elasticsearch==8.16.0 # via feast (setup.py) entrypoints==0.4 # via altair @@ -186,7 +186,7 @@ executing==2.1.0 # via stack-data faiss-cpu==1.9.0 # via feast (setup.py) -fastapi==0.115.4 +fastapi==0.115.5 # via feast (setup.py) fastjsonschema==2.20.0 # via nbformat @@ -206,7 +206,7 @@ fsspec==2024.9.0 # dask geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.22.0 +google-api-core[grpc]==2.23.0 # via # feast (setup.py) # google-cloud-bigquery @@ -225,11 +225,11 @@ google-auth==2.36.0 # google-cloud-datastore # google-cloud-storage # kubernetes -google-cloud-bigquery[pandas]==3.26.0 +google-cloud-bigquery[pandas]==3.27.0 # via feast (setup.py) google-cloud-bigquery-storage==2.27.0 # via feast (setup.py) -google-cloud-bigtable==2.26.0 +google-cloud-bigtable==2.27.0 # via feast (setup.py) google-cloud-core==2.4.1 # via @@ -249,7 +249,7 @@ google-resumable-media==2.7.2 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.65.0 +googleapis-common-protos[grpc]==1.66.0 # via # feast (setup.py) # google-api-core @@ -302,7 +302,7 @@ hiredis==2.4.0 # via feast (setup.py) hpack==4.0.0 # via h2 -httpcore==1.0.6 +httpcore==1.0.7 # via httpx httptools==0.6.4 # via uvicorn @@ -320,7 +320,7 @@ ibis-framework[duckdb]==9.5.0 # ibis-substrait ibis-substrait==4.0.1 # via feast (setup.py) -identify==2.6.1 +identify==2.6.2 # via pre-commit idna==3.10 # via @@ -349,7 +349,7 @@ isodate==0.7.2 # via azure-storage-blob isoduration==20.11.0 # via jsonschema -jedi==0.19.1 +jedi==0.19.2 # via ipython jinja2==3.1.4 # via @@ -366,7 +366,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.25 +json5==0.9.28 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -515,7 +515,7 @@ oauthlib==3.2.2 # via requests-oauthlib overrides==7.7.0 # via jupyter-server -packaging==24.1 +packaging==24.2 # via # build # dask @@ -588,7 +588,9 @@ prometheus-client==0.21.0 prompt-toolkit==3.0.48 # via ipython propcache==0.2.0 - # via yarl + # via + # aiohttp + # yarl proto-plus==1.25.0 # via # google-api-core @@ -621,7 +623,7 @@ psycopg[binary, pool]==3.2.3 # via feast (setup.py) psycopg-binary==3.2.3 # via psycopg -psycopg-pool==3.2.3 +psycopg-pool==3.2.4 # via psycopg ptyprocess==0.7.0 # via @@ -827,7 +829,7 @@ ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.7.2 +ruff==0.7.4 # via feast (setup.py) s3transfer==0.10.3 # via boto3 @@ -835,7 +837,7 @@ scipy==1.14.1 # via great-expectations send2trash==1.8.3 # via jupyter-server -setuptools==75.3.0 +setuptools==75.5.0 # via # grpcio-tools # jupyterlab @@ -974,7 +976,7 @@ types-redis==4.6.0.20241004 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==75.3.0.20241105 +types-setuptools==75.4.0.20241115 # via # feast (setup.py) # types-cffi @@ -1040,7 +1042,7 @@ watchfiles==0.24.0 # via uvicorn wcwidth==0.2.13 # via prompt-toolkit -webcolors==24.8.0 +webcolors==24.11.1 # via jsonschema webencodings==0.5.1 # via @@ -1050,11 +1052,11 @@ websocket-client==1.8.0 # via # jupyter-server # kubernetes -websockets==13.1 +websockets==14.1 # via uvicorn -werkzeug==3.1.2 +werkzeug==3.1.3 # via moto -wheel==0.44.0 +wheel==0.45.0 # via # pip-tools # singlestoredb @@ -1068,5 +1070,5 @@ xmltodict==0.14.2 # via moto yarl==1.17.1 # via aiohttp -zipp==3.20.2 +zipp==3.21.0 # via importlib-metadata diff --git a/sdk/python/requirements/py3.11-requirements.txt b/sdk/python/requirements/py3.11-requirements.txt index 7d8ee79e05..8753b77a22 100644 --- a/sdk/python/requirements/py3.11-requirements.txt +++ b/sdk/python/requirements/py3.11-requirements.txt @@ -10,7 +10,7 @@ attrs==24.2.0 # via # jsonschema # referencing -bigtree==0.22.1 +bigtree==0.22.3 # via feast (setup.py) certifi==2024.8.30 # via requests @@ -25,15 +25,15 @@ cloudpickle==3.1.0 # via dask colorama==0.4.6 # via feast (setup.py) -dask[dataframe]==2024.10.0 +dask[dataframe]==2024.11.2 # via # feast (setup.py) # dask-expr -dask-expr==1.1.16 +dask-expr==1.1.19 # via dask dill==0.3.9 # via feast (setup.py) -fastapi==0.115.4 +fastapi==0.115.5 # via feast (setup.py) fsspec==2024.10.0 # via dask @@ -72,7 +72,7 @@ numpy==1.26.4 # feast (setup.py) # dask # pandas -packaging==24.1 +packaging==24.2 # via # dask # gunicorn @@ -168,7 +168,7 @@ uvloop==0.21.0 # via uvicorn watchfiles==0.24.0 # via uvicorn -websockets==13.1 +websockets==14.1 # via uvicorn -zipp==3.20.2 +zipp==3.21.0 # via importlib-metadata diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index ceae3980f6..db8ed20143 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -4,7 +4,7 @@ aiobotocore==2.15.2 # via feast (setup.py) aiohappyeyeballs==2.4.3 # via aiohttp -aiohttp==3.10.10 +aiohttp==3.11.2 # via aiobotocore aioitertools==0.12.0 # via aiobotocore @@ -40,7 +40,7 @@ async-lru==2.0.4 # via jupyterlab async-property==0.2.2 # via python-keycloak -async-timeout==4.0.3 +async-timeout==5.0.1 # via # aiohttp # redis @@ -57,7 +57,7 @@ azure-core==1.32.0 # azure-storage-blob azure-identity==1.19.0 # via feast (setup.py) -azure-storage-blob==12.23.1 +azure-storage-blob==12.24.0 # via feast (setup.py) babel==2.16.0 # via @@ -67,7 +67,7 @@ beautifulsoup4==4.12.3 # via nbconvert bidict==0.23.1 # via ibis-framework -bigtree==0.22.1 +bigtree==0.22.3 # via feast (setup.py) bleach==6.2.0 # via nbconvert @@ -130,7 +130,7 @@ comm==0.2.2 # ipywidgets couchbase==4.3.2 # via feast (setup.py) -coverage[toml]==7.6.4 +coverage[toml]==7.6.7 # via pytest-cov cryptography==42.0.8 # via @@ -154,9 +154,9 @@ dask[dataframe]==2024.8.0 # dask-expr dask-expr==1.1.10 # via dask -db-dtypes==1.3.0 +db-dtypes==1.3.1 # via google-cloud-bigquery -debugpy==1.8.7 +debugpy==1.8.8 # via ipykernel decorator==5.1.1 # via ipython @@ -178,7 +178,7 @@ duckdb==0.10.3 # via ibis-framework elastic-transport==8.15.1 # via elasticsearch -elasticsearch==8.15.1 +elasticsearch==8.16.0 # via feast (setup.py) entrypoints==0.4 # via altair @@ -195,7 +195,7 @@ executing==2.1.0 # via stack-data faiss-cpu==1.9.0 # via feast (setup.py) -fastapi==0.115.4 +fastapi==0.115.5 # via feast (setup.py) fastjsonschema==2.20.0 # via nbformat @@ -215,7 +215,7 @@ fsspec==2024.9.0 # dask geomet==0.2.1.post1 # via cassandra-driver -google-api-core[grpc]==2.22.0 +google-api-core[grpc]==2.23.0 # via # feast (setup.py) # google-cloud-bigquery @@ -234,11 +234,11 @@ google-auth==2.36.0 # google-cloud-datastore # google-cloud-storage # kubernetes -google-cloud-bigquery[pandas]==3.26.0 +google-cloud-bigquery[pandas]==3.27.0 # via feast (setup.py) google-cloud-bigquery-storage==2.27.0 # via feast (setup.py) -google-cloud-bigtable==2.26.0 +google-cloud-bigtable==2.27.0 # via feast (setup.py) google-cloud-core==2.4.1 # via @@ -258,7 +258,7 @@ google-resumable-media==2.7.2 # via # google-cloud-bigquery # google-cloud-storage -googleapis-common-protos[grpc]==1.65.0 +googleapis-common-protos[grpc]==1.66.0 # via # feast (setup.py) # google-api-core @@ -311,7 +311,7 @@ hiredis==2.4.0 # via feast (setup.py) hpack==4.0.0 # via h2 -httpcore==1.0.6 +httpcore==1.0.7 # via httpx httptools==0.6.4 # via uvicorn @@ -329,7 +329,7 @@ ibis-framework[duckdb]==9.0.0 # ibis-substrait ibis-substrait==4.0.1 # via feast (setup.py) -identify==2.6.1 +identify==2.6.2 # via pre-commit idna==3.10 # via @@ -367,7 +367,7 @@ isodate==0.7.2 # via azure-storage-blob isoduration==20.11.0 # via jsonschema -jedi==0.19.1 +jedi==0.19.2 # via ipython jinja2==3.1.4 # via @@ -384,7 +384,7 @@ jmespath==1.0.1 # via # boto3 # botocore -json5==0.9.25 +json5==0.9.28 # via jupyterlab-server jsonpatch==1.33 # via great-expectations @@ -533,7 +533,7 @@ oauthlib==3.2.2 # via requests-oauthlib overrides==7.7.0 # via jupyter-server -packaging==24.1 +packaging==24.2 # via # build # dask @@ -605,7 +605,9 @@ prometheus-client==0.21.0 prompt-toolkit==3.0.48 # via ipython propcache==0.2.0 - # via yarl + # via + # aiohttp + # yarl proto-plus==1.25.0 # via # google-api-core @@ -638,7 +640,7 @@ psycopg[binary, pool]==3.1.18 # via feast (setup.py) psycopg-binary==3.1.18 # via psycopg -psycopg-pool==3.2.3 +psycopg-pool==3.2.4 # via psycopg ptyprocess==0.7.0 # via @@ -844,7 +846,7 @@ ruamel-yaml==0.17.40 # via great-expectations ruamel-yaml-clib==0.2.12 # via ruamel-yaml -ruff==0.7.2 +ruff==0.7.4 # via feast (setup.py) s3transfer==0.10.3 # via boto3 @@ -852,7 +854,7 @@ scipy==1.13.1 # via great-expectations send2trash==1.8.3 # via jupyter-server -setuptools==75.3.0 +setuptools==75.5.0 # via # grpcio-tools # jupyterlab @@ -929,7 +931,7 @@ tinycss2==1.4.0 # via nbconvert toml==0.10.2 # via feast (setup.py) -tomli==2.0.2 +tomli==2.1.0 # via # build # coverage @@ -1001,7 +1003,7 @@ types-redis==4.6.0.20241004 # via feast (setup.py) types-requests==2.30.0.0 # via feast (setup.py) -types-setuptools==75.3.0.20241105 +types-setuptools==75.4.0.20241115 # via # feast (setup.py) # types-cffi @@ -1075,7 +1077,7 @@ watchfiles==0.24.0 # via uvicorn wcwidth==0.2.13 # via prompt-toolkit -webcolors==24.8.0 +webcolors==24.11.1 # via jsonschema webencodings==0.5.1 # via @@ -1085,11 +1087,11 @@ websocket-client==1.8.0 # via # jupyter-server # kubernetes -websockets==13.1 +websockets==14.1 # via uvicorn -werkzeug==3.1.2 +werkzeug==3.1.3 # via moto -wheel==0.44.0 +wheel==0.45.0 # via # pip-tools # singlestoredb @@ -1103,5 +1105,5 @@ xmltodict==0.14.2 # via moto yarl==1.17.1 # via aiohttp -zipp==3.20.2 +zipp==3.21.0 # via importlib-metadata diff --git a/sdk/python/requirements/py3.9-requirements.txt b/sdk/python/requirements/py3.9-requirements.txt index 2e3ffa9038..960ac5293c 100644 --- a/sdk/python/requirements/py3.9-requirements.txt +++ b/sdk/python/requirements/py3.9-requirements.txt @@ -10,7 +10,7 @@ attrs==24.2.0 # via # jsonschema # referencing -bigtree==0.22.1 +bigtree==0.22.3 # via feast (setup.py) certifi==2024.8.30 # via requests @@ -35,7 +35,7 @@ dill==0.3.9 # via feast (setup.py) exceptiongroup==1.2.2 # via anyio -fastapi==0.115.4 +fastapi==0.115.5 # via feast (setup.py) fsspec==2024.10.0 # via dask @@ -76,7 +76,7 @@ numpy==1.26.4 # feast (setup.py) # dask # pandas -packaging==24.1 +packaging==24.2 # via # dask # gunicorn @@ -142,7 +142,7 @@ tenacity==8.5.0 # via feast (setup.py) toml==0.10.2 # via feast (setup.py) -tomli==2.0.2 +tomli==2.1.0 # via mypy toolz==1.0.0 # via @@ -177,7 +177,7 @@ uvloop==0.21.0 # via uvicorn watchfiles==0.24.0 # via uvicorn -websockets==13.1 +websockets==14.1 # via uvicorn -zipp==3.20.2 +zipp==3.21.0 # via importlib-metadata From 6f7de5a43d9f33d01f4fe16c77896fe8908eff2a Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Fri, 15 Nov 2024 13:22:40 -0500 Subject: [PATCH 07/10] adding updated builds Signed-off-by: Francisco Javier Arceo --- sdk/python/requirements/py3.10-ci-requirements.txt | 2 +- sdk/python/requirements/py3.11-ci-requirements.txt | 2 +- sdk/python/requirements/py3.9-ci-requirements.txt | 2 +- .../feature_repos/universal/online_store/milvus.py | 6 +++--- 4 files changed, 6 insertions(+), 6 deletions(-) diff --git a/sdk/python/requirements/py3.10-ci-requirements.txt b/sdk/python/requirements/py3.10-ci-requirements.txt index c2b5225207..04cd7ac919 100644 --- a/sdk/python/requirements/py3.10-ci-requirements.txt +++ b/sdk/python/requirements/py3.10-ci-requirements.txt @@ -915,7 +915,7 @@ terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals -testcontainers==4.4.0 +testcontainers==4.8.2 # via feast (setup.py) thriftpy2==0.5.2 # via happybase diff --git a/sdk/python/requirements/py3.11-ci-requirements.txt b/sdk/python/requirements/py3.11-ci-requirements.txt index e77d6f19bf..f1d1548f49 100644 --- a/sdk/python/requirements/py3.11-ci-requirements.txt +++ b/sdk/python/requirements/py3.11-ci-requirements.txt @@ -906,7 +906,7 @@ terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals -testcontainers==4.4.0 +testcontainers==4.8.2 # via feast (setup.py) thriftpy2==0.5.2 # via happybase diff --git a/sdk/python/requirements/py3.9-ci-requirements.txt b/sdk/python/requirements/py3.9-ci-requirements.txt index db8ed20143..7507e21407 100644 --- a/sdk/python/requirements/py3.9-ci-requirements.txt +++ b/sdk/python/requirements/py3.9-ci-requirements.txt @@ -923,7 +923,7 @@ terminado==0.18.1 # via # jupyter-server # jupyter-server-terminals -testcontainers==4.4.0 +testcontainers==4.8.2 # via feast (setup.py) thriftpy2==0.5.2 # via happybase diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py b/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py index 3ccf2b92d6..4d7683727e 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py @@ -1,6 +1,6 @@ from typing import Any, Dict -from testcontainers.core.container import DockerContainer +from testcontainers.milvus import MilvusContainer from testcontainers.core.waiting_utils import wait_for_logs from tests.integration.feature_repos.universal.online_store_creator import ( @@ -11,7 +11,7 @@ class MilvusOnlineStoreCreator(OnlineStoreCreator): def __init__(self, project_name: str, **kwargs): super().__init__(project_name) - self.container = DockerContainer("milvusdb/milvus:v2.4.4").with_exposed_ports( + self.container = MilvusContainer("milvusdb/milvus:v2.4.4").with_exposed_ports( 19530 ) @@ -36,7 +36,7 @@ def create_online_store(self) -> Dict[str, Any]: "port": int(port), "index_type": "IVF_FLAT", "metric_type": "L2", - "embedding_dim": 128, # Adjust based on your embedding dimension + "embedding_dim": 128, "vector_enabled": True, } From b2a536447686d7cb1b9140a5d44df79d245493b6 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Fri, 15 Nov 2024 13:23:11 -0500 Subject: [PATCH 08/10] adding updated testcontainers package Signed-off-by: Francisco Javier Arceo --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 26347c7893..2ac009a65f 100644 --- a/setup.py +++ b/setup.py @@ -181,7 +181,7 @@ "pytest-mock==1.10.4", "pytest-env", "Sphinx>4.0.0,<7", - "testcontainers==4.4.0", + "testcontainers==4.8.2", "python-keycloak==4.2.2", "pre-commit<3.3.2", "assertpy==1.1", From 3eab8d9988fb0a794ee2e53f6662c05edbef2899 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Fri, 15 Nov 2024 13:41:23 -0500 Subject: [PATCH 09/10] changed import Signed-off-by: Francisco Javier Arceo --- .../integration/feature_repos/universal/online_store/milvus.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py b/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py index 4d7683727e..43fced476b 100644 --- a/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py +++ b/sdk/python/tests/integration/feature_repos/universal/online_store/milvus.py @@ -1,7 +1,7 @@ from typing import Any, Dict -from testcontainers.milvus import MilvusContainer from testcontainers.core.waiting_utils import wait_for_logs +from testcontainers.milvus import MilvusContainer from tests.integration.feature_repos.universal.online_store_creator import ( OnlineStoreCreator, From a714f858b458c0605c65566358d3ad7471b9a9f1 Mon Sep 17 00:00:00 2001 From: Francisco Javier Arceo Date: Tue, 19 Nov 2024 21:55:29 -0500 Subject: [PATCH 10/10] adding repo config Signed-off-by: Francisco Javier Arceo --- .../adding-support-for-a-new-online-store.md | 2 +- .../milvus_online_store/milvus.py | 37 +++++++------------ sdk/python/feast/repo_config.py | 1 + 3 files changed, 15 insertions(+), 25 deletions(-) diff --git a/docs/how-to-guides/customizing-feast/adding-support-for-a-new-online-store.md b/docs/how-to-guides/customizing-feast/adding-support-for-a-new-online-store.md index 5e26f133ce..ee75aa6b74 100644 --- a/docs/how-to-guides/customizing-feast/adding-support-for-a-new-online-store.md +++ b/docs/how-to-guides/customizing-feast/adding-support-for-a-new-online-store.md @@ -25,7 +25,7 @@ OnlineStore class names must end with the OnlineStore suffix! ### Contrib online stores -New online stores go in `sdk/python/feast/infra/online_stores/contrib/`. +New online stores go in `sdk/python/feast/infra/online_stores/`. #### What is a contrib plugin? diff --git a/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py index 757b6469a0..cab30d2716 100644 --- a/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py +++ b/sdk/python/feast/infra/online_stores/milvus_online_store/milvus.py @@ -31,30 +31,19 @@ class MilvusOnlineStoreConfig(FeastConfigBaseModel, VectorStoreConfig): - """Online store config for Milvus vector database""" - - type: Literal["milvus", "feast.infra.online_stores.milvus.MilvusOnlineStore"] = ( - "milvus" - ) - """Online store type selector""" - - host: StrictStr = "localhost" - """Hostname for Milvus server""" - - port: int = 19530 - """Port for Milvus server""" - - index_type: str = "IVF_FLAT" - """Index type for Milvus collection""" - - metric_type: str = "L2" - """Distance metric type""" + """ + Configuration for the Milvus online store. + NOTE: The class *must* end with the `OnlineStoreConfig` suffix. + """ - embedding_dim: int = 128 - """Dimension of the embedding vectors""" + type: Literal["milvus"] = "milvus" - vector_enabled: bool = True - """Flag to enable vector search""" + host: Optional[StrictStr] = "localhost" + port: Optional[int] = 19530 + index_type: Optional[str] = "IVF_FLAT" + metric_type: Optional[str] = "L2" + embedding_dim: Optional[int] = 128 + vector_enabled: Optional[bool] = True class MilvusOnlineStore(OnlineStore): @@ -69,7 +58,7 @@ class MilvusOnlineStore(OnlineStore): def _connect(self, config: RepoConfig): connections.connect( - alias="default", + alias="feast", host=config.online_store.host, port=str(config.online_store.port), ) @@ -350,7 +339,7 @@ def __init__(self, host: str, port: int, name: str): self._connect() def _connect(self): - connections.connect(alias="default", host=self.host, port=str(self.port)) + return connections.connect(alias="default", host=self.host, port=str(self.port)) def to_infra_object_proto(self) -> InfraObjectProto: # Implement serialization if needed diff --git a/sdk/python/feast/repo_config.py b/sdk/python/feast/repo_config.py index 185a8723ad..39600c87bc 100644 --- a/sdk/python/feast/repo_config.py +++ b/sdk/python/feast/repo_config.py @@ -81,6 +81,7 @@ "singlestore": "feast.infra.online_stores.singlestore_online_store.singlestore.SingleStoreOnlineStore", "qdrant": "feast.infra.online_stores.cqdrant.QdrantOnlineStore", "couchbase": "feast.infra.online_stores.couchbase_online_store.couchbase.CouchbaseOnlineStore", + "milvus": "feast.infra.online_stores.milvus_online_store.milvus.MilvusOnlineStore", **LEGACY_ONLINE_STORE_CLASS_FOR_TYPE, }