From ae8523b4a5fcad87db422cb70db13111cff7f7c5 Mon Sep 17 00:00:00 2001 From: Gabriel Gore Date: Wed, 21 Aug 2024 16:23:42 -0400 Subject: [PATCH 1/9] gen-asyncapi docs invoke placeholder task --- tasks.py | 25 +++++++++++++++++++++++++ 1 file changed, 25 insertions(+) diff --git a/tasks.py b/tasks.py index c8d0dcfe..613ffcd1 100644 --- a/tasks.py +++ b/tasks.py @@ -24,6 +24,12 @@ PYPROJECT_TOML: Final[pathlib.Path] = PROJECT_ROOT / "pyproject.toml" DOCKERFILE_PATH: Final[str] = "./Dockerfile" +ASYNCAPI_YAML: Final = PROJECT_ROOT / "asyncapi.yaml" +ASNYCAPI_JSON: Final = PROJECT_ROOT / "asyncapi.json" + +# TODO: need a module level constant for the runner app to generate docs +AGENT_APP_PATH_ID: Final = "great_expectations_cloud.agent.agent:app" + @functools.lru_cache(maxsize=8) def _get_pyproject_tool_dict( @@ -78,6 +84,25 @@ def lint(ctx: Context, check: bool = False, unsafe_fixes: bool = False) -> None: ctx.run(" ".join(cmds), echo=True, pty=True) +@invoke.task(aliases=["sync-asyncapi"]) +def gen_asyncapi(ctx: Context) -> None: + """ + Generate asyncapi docs + TODO: this will not work until AGENT_APP_PATH_ID points to a FastStream app instance + https://faststream.airt.ai/latest/faststream/#running-the-application + """ + core_cmds = ["faststream", "docs", "gen"] + yaml_cmds = [*core_cmds, "--yaml", AGENT_APP_PATH_ID] + ctx.run(" ".join(yaml_cmds), echo=True, pty=True) + assert ASYNCAPI_YAML.exists(), f"{ASYNCAPI_YAML} not found" + print(f"Updated {ASYNCAPI_YAML}") + + json_cmds = [*core_cmds, AGENT_APP_PATH_ID] + ctx.run(" ".join(json_cmds), echo=True, pty=True) + assert ASNYCAPI_JSON.exists(), f"{ASNYCAPI_JSON} not found" + print(f"Updated {ASNYCAPI_JSON}") + + @invoke.task( aliases=("build",), help={ From c5cf21e95e4ff52d663f7d0f40da397fcdb0a22d Mon Sep 17 00:00:00 2001 From: Gabriel Gore Date: Wed, 21 Aug 2024 17:22:10 -0400 Subject: [PATCH 2/9] gen-asyncapi docs --- asyncapi.json | 21 +++++++++++++++++++++ asyncapi.yaml | 15 +++++++++++++++ great_expectations_cloud/agent/agent.py | 14 ++++++++------ 3 files changed, 44 insertions(+), 6 deletions(-) create mode 100644 asyncapi.json create mode 100644 asyncapi.yaml diff --git a/asyncapi.json b/asyncapi.json new file mode 100644 index 00000000..9cb9fa40 --- /dev/null +++ b/asyncapi.json @@ -0,0 +1,21 @@ +{ + "asyncapi": "2.6.0", + "defaultContentType": "application/json", + "info": { + "title": "FastStream", + "version": "0.1.0", + "description": "" + }, + "servers": { + "development": { + "url": "amqps://org-agent-a7ba9504-a02f-40c8-8c46-145b4ff5765b:B-8Eflf%7Bd5y_%23k_-Lcs%5CVYcN8Osf,!%40l%3CB=dE9j515Yj%3Ec%5Bb4POG2%5D*_ja%60%5BV%5BZb@mq.greatexpectations.io:5671/", + "protocol": "amqps", + "protocolVersion": "0.9.1" + } + }, + "channels": {}, + "components": { + "messages": {}, + "schemas": {} + } +} diff --git a/asyncapi.yaml b/asyncapi.yaml new file mode 100644 index 00000000..c840a334 --- /dev/null +++ b/asyncapi.yaml @@ -0,0 +1,15 @@ +asyncapi: 2.6.0 +defaultContentType: application/json +info: + title: FastStream + version: 0.1.0 + description: "" +servers: + development: + url: amqps://org-agent-a7ba9504-a02f-40c8-8c46-145b4ff5765b:XI,IT)U5%5Bd%5C%23%22$%40%3F0SwhhdF0gp%60z(1-WHE%3Fu%230tTm6N%3C%5DQLjg;*v6%22%7DU%7D4%5D%7CM%5BjK@mq.greatexpectations.io:5671/ + protocol: amqps + protocolVersion: 0.9.1 +channels: {} +components: + messages: {} + schemas: {} diff --git a/great_expectations_cloud/agent/agent.py b/great_expectations_cloud/agent/agent.py index eaabb8b5..06e11c86 100644 --- a/great_expectations_cloud/agent/agent.py +++ b/great_expectations_cloud/agent/agent.py @@ -191,7 +191,9 @@ class GXAgent: _PYPI_GX_AGENT_PACKAGE_NAME = "great_expectations_cloud" _PYPI_GREAT_EXPECTATIONS_PACKAGE_NAME = "great_expectations" - def __init__(self: Self): + def __init__(self: Self, app: FastStream, broker: RabbitBroker) -> None: + self.app = app + self.broker = broker agent_version: str = self.get_current_gx_agent_version() great_expectations_version: str = self._get_current_great_expectations_version() print(f"GX Agent version: {agent_version}") @@ -239,15 +241,11 @@ def run(self) -> None: async def _listen(self) -> None: """Manage connection lifecycle.""" try: - broker = RabbitBroker( - url=str(self._config.connection_string), - ) - app = FastStream(broker) queue = RabbitQueue(name=self._config.queue, durable=True, passive=True) print("Queue is valid.") # FastStream declares default exchange if not provided - @broker.subscriber(queue, retry=MAX_DELIVERY) + @self.broker.subscriber(queue, retry=MAX_DELIVERY) async def handle_me( msg: dict[str, Any], correlation_id: str = Context("message.correlation_id") ) -> None: @@ -542,3 +540,7 @@ def _update_headers_agent_patch( # TODO: this is relying on a private implementation detail # use a public API once it is available http._update_headers = _update_headers_agent_patch + + +broker = RabbitBroker(str(GXAgent._get_config().connection_string)) +app: Final[FastStream] = FastStream(broker) From eced3ddcca57763ad46aa7dcdffbd2f97f4bf725 Mon Sep 17 00:00:00 2001 From: Gabriel Gore Date: Thu, 22 Aug 2024 10:02:59 -0400 Subject: [PATCH 3/9] temporary global handler --- great_expectations_cloud/agent/agent.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/great_expectations_cloud/agent/agent.py b/great_expectations_cloud/agent/agent.py index 06e11c86..9614acc3 100644 --- a/great_expectations_cloud/agent/agent.py +++ b/great_expectations_cloud/agent/agent.py @@ -45,6 +45,7 @@ ) from great_expectations_cloud.agent.models import ( AgentBaseExtraForbid, + EventMessage, JobCompleted, JobStarted, JobStatus, @@ -544,3 +545,10 @@ def _update_headers_agent_patch( broker = RabbitBroker(str(GXAgent._get_config().connection_string)) app: Final[FastStream] = FastStream(broker) +queue = RabbitQueue(name=GXAgent._get_config().queue, durable=True, passive=True) + + +# FastStream declares default exchange if not provided +@broker.subscriber(queue, retry=MAX_DELIVERY) +async def handle_event(msg: EventMessage): + print(msg) From 9fbf35dca50ff0c733352bd0965a51342d2e71fa Mon Sep 17 00:00:00 2001 From: Gabriel Gore Date: Thu, 22 Aug 2024 10:04:01 -0400 Subject: [PATCH 4/9] use framework event parsing --- great_expectations_cloud/agent/agent.py | 26 ++++++++++++------------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/great_expectations_cloud/agent/agent.py b/great_expectations_cloud/agent/agent.py index 9614acc3..5d73006d 100644 --- a/great_expectations_cloud/agent/agent.py +++ b/great_expectations_cloud/agent/agent.py @@ -12,10 +12,7 @@ import aiormq import orjson -from faststream import ( - Context, - FastStream, -) +from faststream import Context, FastStream from faststream.rabbit import RabbitBroker, RabbitQueue from great_expectations.core.http import create_session from great_expectations.data_context.cloud_constants import CLOUD_DEFAULT_BASE_URL @@ -109,14 +106,12 @@ class Config: json_loads = orjson_loads -async def handler(msg: dict[str, Any], gx_agent: GXAgent, correlation_id: str) -> None: # noqa: +async def handler(msg: EventMessage, gx_agent: GXAgent) -> None: # noqa: print(f"Received: {msg}") print(f"GX Agent: {gx_agent}") - - event = EventHandler.parse_event_from_dict(msg) event_context = EventContext( - event=event, - correlation_id=correlation_id, # msg.correlation_id, + event=msg.event, + correlation_id=msg.correlation_id, ) organization_id = None try: @@ -175,7 +170,7 @@ async def handler(msg: dict[str, Any], gx_agent: GXAgent, correlation_id: str) - extra={ "event_type": event_context.event.type, "correlation_id": event_context.correlation_id, - "event": event.dict(), + "event": msg.event.dict(), }, ) @@ -192,9 +187,12 @@ class GXAgent: _PYPI_GX_AGENT_PACKAGE_NAME = "great_expectations_cloud" _PYPI_GREAT_EXPECTATIONS_PACKAGE_NAME = "great_expectations" - def __init__(self: Self, app: FastStream, broker: RabbitBroker) -> None: + app: FastStream + broker: RabbitBroker + + def __init__(self: Self, app: FastStream) -> None: self.app = app - self.broker = broker + self.broker = app.broker # type: ignore[assignment] agent_version: str = self.get_current_gx_agent_version() great_expectations_version: str = self._get_current_great_expectations_version() print(f"GX Agent version: {agent_version}") @@ -248,11 +246,11 @@ async def _listen(self) -> None: # FastStream declares default exchange if not provided @self.broker.subscriber(queue, retry=MAX_DELIVERY) async def handle_me( - msg: dict[str, Any], correlation_id: str = Context("message.correlation_id") + msg: EventMessage, correlation_id: str = Context("message.correlation_id") ) -> None: print(f"Received: {msg}") print(f"Correlation ID: {correlation_id}") - await handler(msg, gx_agent=self, correlation_id=correlation_id) + await handler(msg, gx_agent=self) print("FastStream is ready.") await app.run() From 1fe09012e55f5221a215915905d291ae937d170a Mon Sep 17 00:00:00 2001 From: Gabriel Gore Date: Thu, 22 Aug 2024 14:41:16 -0400 Subject: [PATCH 5/9] ban pydantic v1 --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index a068d910..72dc6d20 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -193,6 +193,7 @@ max-complexity = 10 [tool.ruff.lint.flake8-tidy-imports.banned-api] "os.environ".msg = """Please do not use os.environ, instead use a pydantic.BaseSettings model""" +"pydantic.v1".msg = "Do not use pydantic v1." "great_expectations.compatibility.pydantic".msg = "Import pydantic directly." "great_expectations.compatibility.sqlalchemy".msg = "Import sqlalchemy directly." "great_expectations.compatibility".msg = "Favor specific version imports over compatibility module." From 8ff4f45cf857271cbea804423c7eff0646960345 Mon Sep 17 00:00:00 2001 From: Gabriel Gore Date: Thu, 22 Aug 2024 14:53:16 -0400 Subject: [PATCH 6/9] pydantic-settings --- poetry.lock | 109 ++++++++++++++++++------------------------------- pyproject.toml | 1 + 2 files changed, 41 insertions(+), 69 deletions(-) diff --git a/poetry.lock b/poetry.lock index db087293..705d948d 100644 --- a/poetry.lock +++ b/poetry.lock @@ -2461,7 +2461,11 @@ files = [ ] [package.dependencies] -numpy = {version = ">=1.20.3", markers = "python_version < \"3.10\""} +numpy = [ + {version = ">=1.20.3", markers = "python_version < \"3.10\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, +] python-dateutil = ">=2.8.2" pytz = ">=2020.1" tzdata = ">=2022.1" @@ -2489,73 +2493,6 @@ sql-other = ["SQLAlchemy (>=1.4.16)"] test = ["hypothesis (>=6.34.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] xml = ["lxml (>=4.6.3)"] -[[package]] -name = "pandas" -version = "2.1.4" -description = "Powerful data structures for data analysis, time series, and statistics" -optional = false -python-versions = ">=3.9" -files = [ - {file = "pandas-2.1.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:bdec823dc6ec53f7a6339a0e34c68b144a7a1fd28d80c260534c39c62c5bf8c9"}, - {file = "pandas-2.1.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:294d96cfaf28d688f30c918a765ea2ae2e0e71d3536754f4b6de0ea4a496d034"}, - {file = "pandas-2.1.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6b728fb8deba8905b319f96447a27033969f3ea1fea09d07d296c9030ab2ed1d"}, - {file = "pandas-2.1.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:00028e6737c594feac3c2df15636d73ace46b8314d236100b57ed7e4b9ebe8d9"}, - {file = "pandas-2.1.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:426dc0f1b187523c4db06f96fb5c8d1a845e259c99bda74f7de97bd8a3bb3139"}, - {file = "pandas-2.1.4-cp310-cp310-win_amd64.whl", hash = "sha256:f237e6ca6421265643608813ce9793610ad09b40154a3344a088159590469e46"}, - {file = "pandas-2.1.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b7d852d16c270e4331f6f59b3e9aa23f935f5c4b0ed2d0bc77637a8890a5d092"}, - {file = "pandas-2.1.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:bd7d5f2f54f78164b3d7a40f33bf79a74cdee72c31affec86bfcabe7e0789821"}, - {file = "pandas-2.1.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0aa6e92e639da0d6e2017d9ccff563222f4eb31e4b2c3cf32a2a392fc3103c0d"}, - {file = "pandas-2.1.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d797591b6846b9db79e65dc2d0d48e61f7db8d10b2a9480b4e3faaddc421a171"}, - {file = "pandas-2.1.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:d2d3e7b00f703aea3945995ee63375c61b2e6aa5aa7871c5d622870e5e137623"}, - {file = "pandas-2.1.4-cp311-cp311-win_amd64.whl", hash = "sha256:dc9bf7ade01143cddc0074aa6995edd05323974e6e40d9dbde081021ded8510e"}, - {file = "pandas-2.1.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:482d5076e1791777e1571f2e2d789e940dedd927325cc3cb6d0800c6304082f6"}, - {file = "pandas-2.1.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:8a706cfe7955c4ca59af8c7a0517370eafbd98593155b48f10f9811da440248b"}, - {file = "pandas-2.1.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b0513a132a15977b4a5b89aabd304647919bc2169eac4c8536afb29c07c23540"}, - {file = "pandas-2.1.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e9f17f2b6fc076b2a0078862547595d66244db0f41bf79fc5f64a5c4d635bead"}, - {file = "pandas-2.1.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:45d63d2a9b1b37fa6c84a68ba2422dc9ed018bdaa668c7f47566a01188ceeec1"}, - {file = "pandas-2.1.4-cp312-cp312-win_amd64.whl", hash = "sha256:f69b0c9bb174a2342818d3e2778584e18c740d56857fc5cdb944ec8bbe4082cf"}, - {file = "pandas-2.1.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:3f06bda01a143020bad20f7a85dd5f4a1600112145f126bc9e3e42077c24ef34"}, - {file = "pandas-2.1.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ab5796839eb1fd62a39eec2916d3e979ec3130509930fea17fe6f81e18108f6a"}, - {file = "pandas-2.1.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edbaf9e8d3a63a9276d707b4d25930a262341bca9874fcb22eff5e3da5394732"}, - {file = "pandas-2.1.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1ebfd771110b50055712b3b711b51bee5d50135429364d0498e1213a7adc2be8"}, - {file = "pandas-2.1.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:8ea107e0be2aba1da619cc6ba3f999b2bfc9669a83554b1904ce3dd9507f0860"}, - {file = "pandas-2.1.4-cp39-cp39-win_amd64.whl", hash = "sha256:d65148b14788b3758daf57bf42725caa536575da2b64df9964c563b015230984"}, - {file = "pandas-2.1.4.tar.gz", hash = "sha256:fcb68203c833cc735321512e13861358079a96c174a61f5116a1de89c58c0ef7"}, -] - -[package.dependencies] -numpy = [ - {version = ">=1.22.4,<2", markers = "python_version < \"3.11\""}, - {version = ">=1.23.2,<2", markers = "python_version == \"3.11\""}, -] -python-dateutil = ">=2.8.2" -pytz = ">=2020.1" -tzdata = ">=2022.1" - -[package.extras] -all = ["PyQt5 (>=5.15.6)", "SQLAlchemy (>=1.4.36)", "beautifulsoup4 (>=4.11.1)", "bottleneck (>=1.3.4)", "dataframe-api-compat (>=0.1.7)", "fastparquet (>=0.8.1)", "fsspec (>=2022.05.0)", "gcsfs (>=2022.05.0)", "html5lib (>=1.1)", "hypothesis (>=6.46.1)", "jinja2 (>=3.1.2)", "lxml (>=4.8.0)", "matplotlib (>=3.6.1)", "numba (>=0.55.2)", "numexpr (>=2.8.0)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pandas-gbq (>=0.17.5)", "psycopg2 (>=2.9.3)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.5)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)", "pyxlsb (>=1.0.9)", "qtpy (>=2.2.0)", "s3fs (>=2022.05.0)", "scipy (>=1.8.1)", "tables (>=3.7.0)", "tabulate (>=0.8.10)", "xarray (>=2022.03.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)", "zstandard (>=0.17.0)"] -aws = ["s3fs (>=2022.05.0)"] -clipboard = ["PyQt5 (>=5.15.6)", "qtpy (>=2.2.0)"] -compression = ["zstandard (>=0.17.0)"] -computation = ["scipy (>=1.8.1)", "xarray (>=2022.03.0)"] -consortium-standard = ["dataframe-api-compat (>=0.1.7)"] -excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.10)", "pyxlsb (>=1.0.9)", "xlrd (>=2.0.1)", "xlsxwriter (>=3.0.3)"] -feather = ["pyarrow (>=7.0.0)"] -fss = ["fsspec (>=2022.05.0)"] -gcp = ["gcsfs (>=2022.05.0)", "pandas-gbq (>=0.17.5)"] -hdf5 = ["tables (>=3.7.0)"] -html = ["beautifulsoup4 (>=4.11.1)", "html5lib (>=1.1)", "lxml (>=4.8.0)"] -mysql = ["SQLAlchemy (>=1.4.36)", "pymysql (>=1.0.2)"] -output-formatting = ["jinja2 (>=3.1.2)", "tabulate (>=0.8.10)"] -parquet = ["pyarrow (>=7.0.0)"] -performance = ["bottleneck (>=1.3.4)", "numba (>=0.55.2)", "numexpr (>=2.8.0)"] -plot = ["matplotlib (>=3.6.1)"] -postgresql = ["SQLAlchemy (>=1.4.36)", "psycopg2 (>=2.9.3)"] -spss = ["pyreadstat (>=1.1.5)"] -sql-other = ["SQLAlchemy (>=1.4.36)"] -test = ["hypothesis (>=6.46.1)", "pytest (>=7.3.2)", "pytest-xdist (>=2.2.0)"] -xml = ["lxml (>=4.8.0)"] - [[package]] name = "pandocfilters" version = "1.5.1" @@ -3036,6 +2973,26 @@ files = [ [package.dependencies] typing-extensions = ">=4.6.0,<4.7.0 || >4.7.0" +[[package]] +name = "pydantic-settings" +version = "2.4.0" +description = "Settings management using Pydantic" +optional = false +python-versions = ">=3.8" +files = [ + {file = "pydantic_settings-2.4.0-py3-none-any.whl", hash = "sha256:bb6849dc067f1687574c12a639e231f3a6feeed0a12d710c1382045c5db1c315"}, + {file = "pydantic_settings-2.4.0.tar.gz", hash = "sha256:ed81c3a0f46392b4d7c0a565c05884e6e54b3456e6f0fe4d8814981172dc9a88"}, +] + +[package.dependencies] +pydantic = ">=2.7.0" +python-dotenv = ">=0.21.0" + +[package.extras] +azure-key-vault = ["azure-identity (>=1.16.0)", "azure-keyvault-secrets (>=4.8.0)"] +toml = ["tomli (>=2.0.1)"] +yaml = ["pyyaml (>=6.0.1)"] + [[package]] name = "pyfakefs" version = "5.6.0" @@ -3197,6 +3154,20 @@ files = [ [package.dependencies] six = ">=1.5" +[[package]] +name = "python-dotenv" +version = "1.0.1" +description = "Read key-value pairs from a .env file and set them as environment variables" +optional = false +python-versions = ">=3.8" +files = [ + {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"}, + {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"}, +] + +[package.extras] +cli = ["click (>=5.0)"] + [[package]] name = "python-json-logger" version = "2.0.7" @@ -4557,4 +4528,4 @@ snowflake = ["snowflake-connector-python", "snowflake-sqlalchemy"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "6e3fdedff3b0afb13bda58ce1f3dcd1caef278b42520158eee3b4d981a322f9a" +content-hash = "23eefe21f7d9309b82f770b32bc680df0f8514bee210906636e6daab3777d585" diff --git a/pyproject.toml b/pyproject.toml index 72dc6d20..c1d7f7df 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -25,6 +25,7 @@ include = [ python = ">=3.8,<3.12" great-expectations = "^0.18.19" pydantic = ">=2.8.1,<3" +pydantic-settings = "^2.4.0" pika = "^1.3.1" # needed for metrics serialization orjson = "^3.9.7, !=3.9.10" # TODO: remove inequality once dep resolution issue is resolved From c5afcf921d947b099bf20dcb7e87f28dc7a16d5b Mon Sep 17 00:00:00 2001 From: Gabriel Gore Date: Thu, 22 Aug 2024 14:54:59 -0400 Subject: [PATCH 7/9] move to pydantic v2 models --- .../agent/actions/agent_action.py | 2 +- great_expectations_cloud/agent/agent.py | 7 +++---- great_expectations_cloud/agent/config.py | 3 ++- great_expectations_cloud/agent/event_handler.py | 11 ++++++----- great_expectations_cloud/agent/models.py | 14 +++++++------- pyproject.toml | 1 + 6 files changed, 20 insertions(+), 18 deletions(-) diff --git a/great_expectations_cloud/agent/actions/agent_action.py b/great_expectations_cloud/agent/actions/agent_action.py index eb09141b..0fb7a54b 100644 --- a/great_expectations_cloud/agent/actions/agent_action.py +++ b/great_expectations_cloud/agent/actions/agent_action.py @@ -5,7 +5,7 @@ from typing import TYPE_CHECKING, Generic, Optional, Sequence, TypeVar from uuid import UUID -from pydantic.v1 import BaseModel +from pydantic import BaseModel from great_expectations_cloud.agent.models import CreatedResource, Event diff --git a/great_expectations_cloud/agent/agent.py b/great_expectations_cloud/agent/agent.py index 5d73006d..3acf7f0a 100644 --- a/great_expectations_cloud/agent/agent.py +++ b/great_expectations_cloud/agent/agent.py @@ -19,8 +19,7 @@ from great_expectations.data_context.data_context.context_factory import get_context from packaging.version import Version from pika.exceptions import AuthenticationError, ProbableAuthenticationError -from pydantic import v1 as pydantic_v1 -from pydantic.v1 import AmqpDsn, AnyUrl +from pydantic import AmqpDsn, AnyUrl, ValidationError from tenacity import after_log, retry, retry_if_exception_type, stop_after_attempt, wait_exponential from great_expectations_cloud.agent.config import ( @@ -370,7 +369,7 @@ def _get_config(cls) -> GXAgentConfig: try: env_vars = GxAgentEnvVars() - except pydantic_v1.ValidationError as validation_err: + except ValidationError as validation_err: raise GXAgentConfigError( generate_config_validation_error_text(validation_err) ) from validation_err @@ -402,7 +401,7 @@ def _get_config(cls) -> GXAgentConfig: gx_cloud_organization_id=env_vars.gx_cloud_organization_id, gx_cloud_access_token=env_vars.gx_cloud_access_token, ) - except pydantic_v1.ValidationError as validation_err: + except ValidationError as validation_err: raise GXAgentConfigError( generate_config_validation_error_text(validation_err) ) from validation_err diff --git a/great_expectations_cloud/agent/config.py b/great_expectations_cloud/agent/config.py index e4d43464..410533cb 100644 --- a/great_expectations_cloud/agent/config.py +++ b/great_expectations_cloud/agent/config.py @@ -1,7 +1,8 @@ from __future__ import annotations from great_expectations.data_context.cloud_constants import CLOUD_DEFAULT_BASE_URL -from pydantic.v1 import AnyUrl, BaseSettings, ValidationError +from pydantic import AnyUrl, ValidationError +from pydantic_settings import BaseSettings class GxAgentEnvVars(BaseSettings): diff --git a/great_expectations_cloud/agent/event_handler.py b/great_expectations_cloud/agent/event_handler.py index d3b8cf4c..fcd1af9f 100644 --- a/great_expectations_cloud/agent/event_handler.py +++ b/great_expectations_cloud/agent/event_handler.py @@ -10,12 +10,13 @@ import great_expectations as gx from packaging.version import Version from packaging.version import parse as parse_version -from pydantic import v1 as pydantic_v1 +from pydantic import ValidationError from great_expectations_cloud.agent.actions.unknown import UnknownEventAction from great_expectations_cloud.agent.exceptions import GXAgentError from great_expectations_cloud.agent.models import ( Event, + EventTA, UnknownEvent, ) @@ -103,8 +104,8 @@ def handle_event( # Refactor opportunity @classmethod def parse_event_from(cls, msg_body: bytes) -> Event: try: - event: Event = pydantic_v1.parse_raw_as(Event, msg_body) # type: ignore[arg-type] # FIXME - except (pydantic_v1.ValidationError, JSONDecodeError): + event: Event = EventTA.validate_json(msg_body) + except (ValidationError, JSONDecodeError): # Log as bytes LOGGER.exception("Unable to parse event type", extra={"msg_body": f"{msg_body!r}"}) return UnknownEvent() @@ -114,8 +115,8 @@ def parse_event_from(cls, msg_body: bytes) -> Event: @classmethod def parse_event_from_dict(cls, msg_body: dict[str, Any]) -> Event: try: - event: Event = pydantic_v1.parse_obj_as(Event, msg_body) # type: ignore[arg-type] # FIXME - except pydantic_v1.ValidationError: + event: Event = EventTA.validate_python(msg_body) + except ValidationError: # Log as dict LOGGER.exception("Unable to parse event type", extra={"msg_body": msg_body}) return UnknownEvent() diff --git a/great_expectations_cloud/agent/models.py b/great_expectations_cloud/agent/models.py index e54139bd..3881d87c 100644 --- a/great_expectations_cloud/agent/models.py +++ b/great_expectations_cloud/agent/models.py @@ -5,22 +5,20 @@ from uuid import UUID from great_expectations.experimental.metric_repository.metrics import MetricTypes -from pydantic.v1 import BaseModel, Extra, Field +from pydantic import BaseModel, Field, TypeAdapter from typing_extensions import Annotated from great_expectations_cloud.agent.exceptions import GXCoreError class AgentBaseExtraForbid(BaseModel): - class Config: - # 2024-03-04: ZEL-501 Strictly enforce models for handling outdated APIs - extra: str = Extra.forbid + # 2024-03-04: ZEL-501 Strictly enforce models for handling outdated APIs + model_config = {"extra": "forbid"} class AgentBaseExtraIgnore(BaseModel): - class Config: - # Extra fields on Events are not strictly enforced - extra: str = Extra.ignore + # Extra fields on Events are not strictly enforced + model_config = {"extra": "ignore"} class EventBase(AgentBaseExtraIgnore): @@ -109,6 +107,8 @@ class UnknownEvent(AgentBaseExtraForbid): Field(discriminator="type"), ] +EventTA: TypeAdapter[Event] = TypeAdapter(Event) + class EventMessage(AgentBaseExtraForbid): event: Event diff --git a/pyproject.toml b/pyproject.toml index c1d7f7df..9bc4e570 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -177,6 +177,7 @@ runtime-evaluated-base-classes = [ # NOTE: ruff is unable to detect that these are subclasses of pydantic.BaseModel "pydantic.BaseModel", "pydantic.v1.BaseModel", + "pydantic_settings.BaseSettings", "great_expectations_cloud.agent.models.AgentBaseExtraForbid", "great_expectations_cloud.agent.models.AgentBaseExtraIgnore", "great_expectations_cloud.agent.models.EventBase", From 2a552f70bed3cac1f3499c287d5a2a8783c7493b Mon Sep 17 00:00:00 2001 From: Gabriel Gore Date: Thu, 22 Aug 2024 14:57:35 -0400 Subject: [PATCH 8/9] docs working! --- asyncapi.json | 522 +++++++++++++++++++++++++++++++++++++++++++++++++- asyncapi.yaml | 348 ++++++++++++++++++++++++++++++++- 2 files changed, 862 insertions(+), 8 deletions(-) diff --git a/asyncapi.json b/asyncapi.json index 9cb9fa40..7ea56349 100644 --- a/asyncapi.json +++ b/asyncapi.json @@ -8,14 +8,528 @@ }, "servers": { "development": { - "url": "amqps://org-agent-a7ba9504-a02f-40c8-8c46-145b4ff5765b:B-8Eflf%7Bd5y_%23k_-Lcs%5CVYcN8Osf,!%40l%3CB=dE9j515Yj%3Ec%5Bb4POG2%5D*_ja%60%5BV%5BZb@mq.greatexpectations.io:5671/", + "url": "amqps://org-agent-a7ba9504-a02f-40c8-8c46-145b4ff5765b:$E%25Q.gY3)wDP%2FSeAKFuovvo%3C%3CCcaf%5Bm%23iJF%7CFg3D4pWp(j1!%3E0'Bu5pVCE6.g3.%5D@mq.greatexpectations.io:5671/", "protocol": "amqps", "protocolVersion": "0.9.1" } }, - "channels": {}, + "channels": { + "q-a7ba9504-a02f-40c8-8c46-145b4ff5765b:_:HandleEvent": { + "servers": [ + "development" + ], + "bindings": { + "amqp": { + "is": "routingKey", + "bindingVersion": "0.2.0", + "queue": { + "name": "q-a7ba9504-a02f-40c8-8c46-145b4ff5765b", + "durable": true, + "exclusive": false, + "autoDelete": false, + "vhost": "/" + }, + "exchange": { + "type": "default", + "vhost": "/" + } + } + }, + "subscribe": { + "bindings": { + "amqp": { + "cc": "q-a7ba9504-a02f-40c8-8c46-145b4ff5765b", + "ack": true, + "bindingVersion": "0.2.0" + } + }, + "message": { + "$ref": "#/components/messages/q-a7ba9504-a02f-40c8-8c46-145b4ff5765b:_:HandleEvent:Message" + } + } + } + }, "components": { - "messages": {}, - "schemas": {} + "messages": { + "q-a7ba9504-a02f-40c8-8c46-145b4ff5765b:_:HandleEvent:Message": { + "title": "q-a7ba9504-a02f-40c8-8c46-145b4ff5765b:_:HandleEvent:Message", + "correlationId": { + "location": "$message.header#/correlation_id" + }, + "payload": { + "$ref": "#/components/schemas/EventMessage" + } + } + }, + "schemas": { + "DraftDatasourceConfigEvent": { + "properties": { + "type": { + "const": "test_datasource_config", + "default": "test_datasource_config", + "enum": [ + "test_datasource_config" + ], + "title": "Type", + "type": "string" + }, + "organization_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Organization Id" + }, + "config_id": { + "format": "uuid", + "title": "Config Id", + "type": "string" + } + }, + "required": [ + "config_id" + ], + "title": "DraftDatasourceConfigEvent", + "type": "object" + }, + "ListTableNamesEvent": { + "properties": { + "type": { + "const": "list_table_names_request.received", + "default": "list_table_names_request.received", + "enum": [ + "list_table_names_request.received" + ], + "title": "Type", + "type": "string" + }, + "organization_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Organization Id" + }, + "datasource_name": { + "title": "Datasource Name", + "type": "string" + } + }, + "required": [ + "datasource_name" + ], + "title": "ListTableNamesEvent", + "type": "object" + }, + "MetricTypes": { + "description": "Represents Metric types in OSS that are used for ColumnDescriptiveMetrics and MetricRepository.\n\nMore Metric types will be added in the future.", + "enum": [ + "table.columns", + "table.row_count", + "table.column_types", + "column.min", + "column.max", + "column.median", + "column.mean", + "column_values.null.count" + ], + "title": "MetricTypes", + "type": "string" + }, + "RunCheckpointEvent": { + "properties": { + "type": { + "const": "run_checkpoint_request", + "default": "run_checkpoint_request", + "enum": [ + "run_checkpoint_request" + ], + "title": "Type", + "type": "string" + }, + "organization_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Organization Id" + }, + "datasource_names_to_asset_names": { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": "array", + "uniqueItems": true + }, + "title": "Datasource Names To Asset Names", + "type": "object" + }, + "checkpoint_id": { + "format": "uuid", + "title": "Checkpoint Id", + "type": "string" + }, + "splitter_options": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Splitter Options" + } + }, + "required": [ + "datasource_names_to_asset_names", + "checkpoint_id" + ], + "title": "RunCheckpointEvent", + "type": "object" + }, + "RunColumnDescriptiveMetricsEvent": { + "properties": { + "type": { + "const": "column_descriptive_metrics_request.received", + "default": "column_descriptive_metrics_request.received", + "enum": [ + "column_descriptive_metrics_request.received" + ], + "title": "Type", + "type": "string" + }, + "organization_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Organization Id" + }, + "datasource_name": { + "title": "Datasource Name", + "type": "string" + }, + "data_asset_name": { + "title": "Data Asset Name", + "type": "string" + } + }, + "required": [ + "datasource_name", + "data_asset_name" + ], + "title": "RunColumnDescriptiveMetricsEvent", + "type": "object" + }, + "RunMetricsListEvent": { + "properties": { + "type": { + "const": "metrics_list_request.received", + "default": "metrics_list_request.received", + "enum": [ + "metrics_list_request.received" + ], + "title": "Type", + "type": "string" + }, + "organization_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Organization Id" + }, + "datasource_name": { + "title": "Datasource Name", + "type": "string" + }, + "data_asset_name": { + "title": "Data Asset Name", + "type": "string" + }, + "metric_names": { + "items": { + "$ref": "#/components/schemas/MetricTypes" + }, + "title": "Metric Names", + "type": "array" + } + }, + "required": [ + "datasource_name", + "data_asset_name", + "metric_names" + ], + "title": "RunMetricsListEvent", + "type": "object" + }, + "RunMissingnessDataAssistantEvent": { + "properties": { + "type": { + "const": "missingness_data_assistant_request.received", + "default": "missingness_data_assistant_request.received", + "enum": [ + "missingness_data_assistant_request.received" + ], + "title": "Type", + "type": "string" + }, + "organization_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Organization Id" + }, + "datasource_name": { + "title": "Datasource Name", + "type": "string" + }, + "data_asset_name": { + "title": "Data Asset Name", + "type": "string" + }, + "expectation_suite_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expectation Suite Name" + } + }, + "required": [ + "datasource_name", + "data_asset_name" + ], + "title": "RunMissingnessDataAssistantEvent", + "type": "object" + }, + "RunOnboardingDataAssistantEvent": { + "properties": { + "type": { + "const": "onboarding_data_assistant_request.received", + "default": "onboarding_data_assistant_request.received", + "enum": [ + "onboarding_data_assistant_request.received" + ], + "title": "Type", + "type": "string" + }, + "organization_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Organization Id" + }, + "datasource_name": { + "title": "Datasource Name", + "type": "string" + }, + "data_asset_name": { + "title": "Data Asset Name", + "type": "string" + }, + "expectation_suite_name": { + "anyOf": [ + { + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Expectation Suite Name" + } + }, + "required": [ + "datasource_name", + "data_asset_name" + ], + "title": "RunOnboardingDataAssistantEvent", + "type": "object" + }, + "RunScheduledCheckpointEvent": { + "properties": { + "type": { + "const": "run_scheduled_checkpoint.received", + "default": "run_scheduled_checkpoint.received", + "enum": [ + "run_scheduled_checkpoint.received" + ], + "title": "Type", + "type": "string" + }, + "organization_id": { + "anyOf": [ + { + "format": "uuid", + "type": "string" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Organization Id" + }, + "schedule_id": { + "format": "uuid", + "title": "Schedule Id", + "type": "string" + }, + "datasource_names_to_asset_names": { + "additionalProperties": { + "items": { + "type": "string" + }, + "type": "array", + "uniqueItems": true + }, + "title": "Datasource Names To Asset Names", + "type": "object" + }, + "checkpoint_id": { + "format": "uuid", + "title": "Checkpoint Id", + "type": "string" + }, + "splitter_options": { + "anyOf": [ + { + "type": "object" + }, + { + "type": "null" + } + ], + "default": null, + "title": "Splitter Options" + } + }, + "required": [ + "schedule_id", + "datasource_names_to_asset_names", + "checkpoint_id" + ], + "title": "RunScheduledCheckpointEvent", + "type": "object" + }, + "UnknownEvent": { + "additionalProperties": false, + "properties": { + "type": { + "const": "unknown_event", + "default": "unknown_event", + "enum": [ + "unknown_event" + ], + "title": "Type", + "type": "string" + } + }, + "title": "UnknownEvent", + "type": "object" + }, + "EventMessage": { + "additionalProperties": false, + "properties": { + "event": { + "discriminator": "type", + "oneOf": [ + { + "$ref": "#/components/schemas/RunOnboardingDataAssistantEvent" + }, + { + "$ref": "#/components/schemas/RunMissingnessDataAssistantEvent" + }, + { + "$ref": "#/components/schemas/RunCheckpointEvent" + }, + { + "$ref": "#/components/schemas/RunScheduledCheckpointEvent" + }, + { + "$ref": "#/components/schemas/RunColumnDescriptiveMetricsEvent" + }, + { + "$ref": "#/components/schemas/RunMetricsListEvent" + }, + { + "$ref": "#/components/schemas/DraftDatasourceConfigEvent" + }, + { + "$ref": "#/components/schemas/ListTableNamesEvent" + }, + { + "$ref": "#/components/schemas/UnknownEvent" + } + ], + "title": "Event" + }, + "correlation_id": { + "title": "Correlation Id", + "type": "string" + } + }, + "required": [ + "event", + "correlation_id" + ], + "title": "EventMessage", + "type": "object" + } + } } } diff --git a/asyncapi.yaml b/asyncapi.yaml index c840a334..1f35553f 100644 --- a/asyncapi.yaml +++ b/asyncapi.yaml @@ -6,10 +6,350 @@ info: description: "" servers: development: - url: amqps://org-agent-a7ba9504-a02f-40c8-8c46-145b4ff5765b:XI,IT)U5%5Bd%5C%23%22$%40%3F0SwhhdF0gp%60z(1-WHE%3Fu%230tTm6N%3C%5DQLjg;*v6%22%7DU%7D4%5D%7CM%5BjK@mq.greatexpectations.io:5671/ + url: amqps://org-agent-a7ba9504-a02f-40c8-8c46-145b4ff5765b:d%23,0foO~&,7+nh%3EVgH%5DIY.m9%3Ayi%3FX0Ydn%7DR%40d8I(GK%60%3FzfFcen,-%7D33o%2FuE*~SD'@mq.greatexpectations.io:5671/ protocol: amqps protocolVersion: 0.9.1 -channels: {} +channels: + q-a7ba9504-a02f-40c8-8c46-145b4ff5765b:_:HandleEvent: + servers: + - development + bindings: + amqp: + is: routingKey + bindingVersion: 0.2.0 + queue: + name: q-a7ba9504-a02f-40c8-8c46-145b4ff5765b + durable: true + exclusive: false + autoDelete: false + vhost: / + exchange: + type: default + vhost: / + subscribe: + bindings: + amqp: + cc: q-a7ba9504-a02f-40c8-8c46-145b4ff5765b + ack: true + bindingVersion: 0.2.0 + message: + $ref: "#/components/messages/q-a7ba9504-a02f-40c8-8c46-145b4ff5765b:_:HandleEvent:Message" components: - messages: {} - schemas: {} + messages: + q-a7ba9504-a02f-40c8-8c46-145b4ff5765b:_:HandleEvent:Message: + title: q-a7ba9504-a02f-40c8-8c46-145b4ff5765b:_:HandleEvent:Message + correlationId: + location: $message.header#/correlation_id + payload: + $ref: "#/components/schemas/EventMessage" + schemas: + DraftDatasourceConfigEvent: + properties: + type: + const: test_datasource_config + default: test_datasource_config + enum: + - test_datasource_config + title: Type + type: string + organization_id: + anyOf: + - format: uuid + type: string + - type: "null" + default: null + title: Organization Id + config_id: + format: uuid + title: Config Id + type: string + required: + - config_id + title: DraftDatasourceConfigEvent + type: object + ListTableNamesEvent: + properties: + type: + const: list_table_names_request.received + default: list_table_names_request.received + enum: + - list_table_names_request.received + title: Type + type: string + organization_id: + anyOf: + - format: uuid + type: string + - type: "null" + default: null + title: Organization Id + datasource_name: + title: Datasource Name + type: string + required: + - datasource_name + title: ListTableNamesEvent + type: object + MetricTypes: + description: + "Represents Metric types in OSS that are used for ColumnDescriptiveMetrics + and MetricRepository. + + + More Metric types will be added in the future." + enum: + - table.columns + - table.row_count + - table.column_types + - column.min + - column.max + - column.median + - column.mean + - column_values.null.count + title: MetricTypes + type: string + RunCheckpointEvent: + properties: + type: + const: run_checkpoint_request + default: run_checkpoint_request + enum: + - run_checkpoint_request + title: Type + type: string + organization_id: + anyOf: + - format: uuid + type: string + - type: "null" + default: null + title: Organization Id + datasource_names_to_asset_names: + additionalProperties: + items: + type: string + type: array + uniqueItems: true + title: Datasource Names To Asset Names + type: object + checkpoint_id: + format: uuid + title: Checkpoint Id + type: string + splitter_options: + anyOf: + - type: object + - type: "null" + default: null + title: Splitter Options + required: + - datasource_names_to_asset_names + - checkpoint_id + title: RunCheckpointEvent + type: object + RunColumnDescriptiveMetricsEvent: + properties: + type: + const: column_descriptive_metrics_request.received + default: column_descriptive_metrics_request.received + enum: + - column_descriptive_metrics_request.received + title: Type + type: string + organization_id: + anyOf: + - format: uuid + type: string + - type: "null" + default: null + title: Organization Id + datasource_name: + title: Datasource Name + type: string + data_asset_name: + title: Data Asset Name + type: string + required: + - datasource_name + - data_asset_name + title: RunColumnDescriptiveMetricsEvent + type: object + RunMetricsListEvent: + properties: + type: + const: metrics_list_request.received + default: metrics_list_request.received + enum: + - metrics_list_request.received + title: Type + type: string + organization_id: + anyOf: + - format: uuid + type: string + - type: "null" + default: null + title: Organization Id + datasource_name: + title: Datasource Name + type: string + data_asset_name: + title: Data Asset Name + type: string + metric_names: + items: + $ref: "#/components/schemas/MetricTypes" + title: Metric Names + type: array + required: + - datasource_name + - data_asset_name + - metric_names + title: RunMetricsListEvent + type: object + RunMissingnessDataAssistantEvent: + properties: + type: + const: missingness_data_assistant_request.received + default: missingness_data_assistant_request.received + enum: + - missingness_data_assistant_request.received + title: Type + type: string + organization_id: + anyOf: + - format: uuid + type: string + - type: "null" + default: null + title: Organization Id + datasource_name: + title: Datasource Name + type: string + data_asset_name: + title: Data Asset Name + type: string + expectation_suite_name: + anyOf: + - type: string + - type: "null" + default: null + title: Expectation Suite Name + required: + - datasource_name + - data_asset_name + title: RunMissingnessDataAssistantEvent + type: object + RunOnboardingDataAssistantEvent: + properties: + type: + const: onboarding_data_assistant_request.received + default: onboarding_data_assistant_request.received + enum: + - onboarding_data_assistant_request.received + title: Type + type: string + organization_id: + anyOf: + - format: uuid + type: string + - type: "null" + default: null + title: Organization Id + datasource_name: + title: Datasource Name + type: string + data_asset_name: + title: Data Asset Name + type: string + expectation_suite_name: + anyOf: + - type: string + - type: "null" + default: null + title: Expectation Suite Name + required: + - datasource_name + - data_asset_name + title: RunOnboardingDataAssistantEvent + type: object + RunScheduledCheckpointEvent: + properties: + type: + const: run_scheduled_checkpoint.received + default: run_scheduled_checkpoint.received + enum: + - run_scheduled_checkpoint.received + title: Type + type: string + organization_id: + anyOf: + - format: uuid + type: string + - type: "null" + default: null + title: Organization Id + schedule_id: + format: uuid + title: Schedule Id + type: string + datasource_names_to_asset_names: + additionalProperties: + items: + type: string + type: array + uniqueItems: true + title: Datasource Names To Asset Names + type: object + checkpoint_id: + format: uuid + title: Checkpoint Id + type: string + splitter_options: + anyOf: + - type: object + - type: "null" + default: null + title: Splitter Options + required: + - schedule_id + - datasource_names_to_asset_names + - checkpoint_id + title: RunScheduledCheckpointEvent + type: object + UnknownEvent: + additionalProperties: false + properties: + type: + const: unknown_event + default: unknown_event + enum: + - unknown_event + title: Type + type: string + title: UnknownEvent + type: object + EventMessage: + additionalProperties: false + properties: + event: + discriminator: type + oneOf: + - $ref: "#/components/schemas/RunOnboardingDataAssistantEvent" + - $ref: "#/components/schemas/RunMissingnessDataAssistantEvent" + - $ref: "#/components/schemas/RunCheckpointEvent" + - $ref: "#/components/schemas/RunScheduledCheckpointEvent" + - $ref: "#/components/schemas/RunColumnDescriptiveMetricsEvent" + - $ref: "#/components/schemas/RunMetricsListEvent" + - $ref: "#/components/schemas/DraftDatasourceConfigEvent" + - $ref: "#/components/schemas/ListTableNamesEvent" + - $ref: "#/components/schemas/UnknownEvent" + title: Event + correlation_id: + title: Correlation Id + type: string + required: + - event + - correlation_id + title: EventMessage + type: object From 2eaad2419fa6404dc106f56270493f956d1a5bc8 Mon Sep 17 00:00:00 2001 From: Gabriel Gore Date: Thu, 22 Aug 2024 15:38:01 -0400 Subject: [PATCH 9/9] fast-depends version bump --- poetry.lock | 15 ++++++++------- pyproject.toml | 1 + 2 files changed, 9 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index 705d948d..b75aacb2 100644 --- a/poetry.lock +++ b/poetry.lock @@ -856,18 +856,19 @@ tests = ["asttokens (>=2.1.0)", "coverage", "coverage-enable-subprocess", "ipyth [[package]] name = "fast-depends" -version = "2.4.3" +version = "2.4.9" description = "FastDepends - extracted and cleared from HTTP domain logic FastAPI Dependency Injection System. Async and sync are both supported." optional = false python-versions = ">=3.8" files = [ - {file = "fast_depends-2.4.3-py3-none-any.whl", hash = "sha256:4cf483575b2b7b0a3294fd4ca244698f1bf24d995ea665c3ae79b67e79b1384d"}, - {file = "fast_depends-2.4.3.tar.gz", hash = "sha256:be39b3d80bf1ee807ccf3e13a5c12a646605e6b2070df0b3f8807b5efb42d34a"}, + {file = "fast_depends-2.4.9-py3-none-any.whl", hash = "sha256:6e44955523d095148c40aae6ed2ab80e4f6b995157073451e94960c1dd02a957"}, + {file = "fast_depends-2.4.9.tar.gz", hash = "sha256:ca56542061ba8f8dde12021da3c0b82bc883394211941f03df38527b1d1c2c8a"}, ] [package.dependencies] anyio = ">=3.0.0,<5.0.0" pydantic = ">=1.7.4,<1.8 || >1.8,<1.8.1 || >1.8.1,<3.0.0" +typing-extensions = {version = "<4.12.1", markers = "python_version < \"3.9\""} [[package]] name = "fastjsonschema" @@ -4250,13 +4251,13 @@ files = [ [[package]] name = "typing-extensions" -version = "4.12.2" +version = "4.12.0" description = "Backported and Experimental Type Hints for Python 3.8+" optional = false python-versions = ">=3.8" files = [ - {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, - {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, + {file = "typing_extensions-4.12.0-py3-none-any.whl", hash = "sha256:b349c66bea9016ac22978d800cfff206d5f9816951f12a7d0ec5578b0a819594"}, + {file = "typing_extensions-4.12.0.tar.gz", hash = "sha256:8cbcdc8606ebcb0d95453ad7dc5065e6237b6aa230a31e81d0f440c30fed5fd8"}, ] [[package]] @@ -4528,4 +4529,4 @@ snowflake = ["snowflake-connector-python", "snowflake-sqlalchemy"] [metadata] lock-version = "2.0" python-versions = ">=3.8,<3.12" -content-hash = "23eefe21f7d9309b82f770b32bc680df0f8514bee210906636e6daab3777d585" +content-hash = "2d33fd02857d1ba14081042e012a79c59ae8468e41097d55f9a37689b149a7a7" diff --git a/pyproject.toml b/pyproject.toml index 9bc4e570..46b8f51a 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -34,6 +34,7 @@ packaging = ">=21.3,<25.0" tenacity = ">=8.2.3,<10.0.0" sqlalchemy = { version = ">=2.0", optional = false } faststream = { extras = ["rabbit"], version = "^0.5.15", optional = false } +fast-depends = ">=2.4.9" # optional dependencies snowflake-connector-python = { version = ">=3.3.1", optional = true } snowflake-sqlalchemy = { version = ">=1.6", optional = true }