diff --git a/.github/.github/pull_request_template.md b/.github/.github/pull_request_template.md index 8ce224e8..62c0286c 100644 --- a/.github/.github/pull_request_template.md +++ b/.github/.github/pull_request_template.md @@ -1,7 +1,7 @@ -## What type of PR is this? +## What type of PR is this? - [ ] Refactor @@ -13,8 +13,8 @@ ## How is this tested? -- [ ] Unit tests -- [ ] E2E Tests +- [ ] Unit tests +- [ ] E2E Tests - [ ] Manually - [ ] N/A diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index aa9b23ef..961a7638 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -2,4 +2,4 @@ # the repo. Unless a later match takes precedence, these # users will be requested for review when someone opens a # pull request. -* @susodapop @arikfr @yunbodeng-db @andrefurlan-db +* @rcypher-databricks @yunbodeng-db @andrefurlan-db @jackyhu-db @benc-db @kravets-levko diff --git a/.github/workflows/code-quality-checks.yml b/.github/workflows/code-quality-checks.yml index fe47eb15..80ac94a7 100644 --- a/.github/workflows/code-quality-checks.yml +++ b/.github/workflows/code-quality-checks.yml @@ -1,5 +1,5 @@ name: Code Quality Checks -on: +on: push: branches: - main @@ -11,7 +11,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9, "3.10", "3.11"] + python-version: [3.8, 3.9, "3.10", "3.11"] steps: #---------------------------------------------- # check-out repo and set-up python @@ -62,7 +62,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9, "3.10"] + python-version: [3.8, 3.9, "3.10"] steps: #---------------------------------------------- # check-out repo and set-up python @@ -114,7 +114,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - python-version: [3.7, 3.8, 3.9, "3.10"] + python-version: [3.8, 3.9, "3.10"] steps: #---------------------------------------------- # check-out repo and set-up python @@ -157,7 +157,9 @@ jobs: - name: Install library run: poetry install --no-interaction #---------------------------------------------- - # black the code + # mypy the code #---------------------------------------------- - name: Mypy - run: poetry run mypy --install-types --non-interactive src + run: | + mkdir .mypy_cache # Workaround for bad error message "error: --install-types failed (no mypy cache directory)"; see https://github.com/python/mypy/issues/10768#issuecomment-2178450153 + poetry run mypy --install-types --non-interactive src diff --git a/.github/workflows/integration.yml b/.github/workflows/integration.yml new file mode 100644 index 00000000..f28c22a8 --- /dev/null +++ b/.github/workflows/integration.yml @@ -0,0 +1,59 @@ +name: Integration Tests +on: + push: + paths-ignore: + - "**.MD" + - "**.md" + +jobs: + run-e2e-tests: + runs-on: ubuntu-latest + environment: azure-prod + env: + DATABRICKS_SERVER_HOSTNAME: ${{ secrets.DATABRICKS_HOST }} + DATABRICKS_HTTP_PATH: ${{ secrets.TEST_PECO_WAREHOUSE_HTTP_PATH }} + DATABRICKS_TOKEN: ${{ secrets.DATABRICKS_TOKEN }} + DATABRICKS_CATALOG: peco + DATABRICKS_USER: ${{ secrets.TEST_PECO_SP_ID }} + steps: + #---------------------------------------------- + # check-out repo and set-up python + #---------------------------------------------- + - name: Check out repository + uses: actions/checkout@v3 + - name: Set up python + id: setup-python + uses: actions/setup-python@v4 + with: + python-version: "3.10" + #---------------------------------------------- + # ----- install & configure poetry ----- + #---------------------------------------------- + - name: Install Poetry + uses: snok/install-poetry@v1 + with: + virtualenvs-create: true + virtualenvs-in-project: true + installer-parallel: true + + #---------------------------------------------- + # load cached venv if cache exists + #---------------------------------------------- + - name: Load cached venv + id: cached-poetry-dependencies + uses: actions/cache@v2 + with: + path: .venv + key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ github.event.repository.name }}-${{ hashFiles('**/poetry.lock') }} + #---------------------------------------------- + # install dependencies if cache does not exist + #---------------------------------------------- + - name: Install dependencies + run: poetry install --no-interaction --all-extras + #---------------------------------------------- + # run test suite + #---------------------------------------------- + - name: Run e2e tests + run: poetry run python -m pytest tests/e2e + - name: Run SQL Alchemy tests + run: poetry run python -m pytest src/databricks/sqlalchemy/test_local diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 9ea751d0..324575ff 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -61,4 +61,4 @@ jobs: - name: Build and publish to pypi uses: JRubics/poetry-publish@v1.10 with: - pypi_token: ${{ secrets.PROD_PYPI_TOKEN }} \ No newline at end of file + pypi_token: ${{ secrets.PROD_PYPI_TOKEN }} diff --git a/.gitignore b/.gitignore index 56e5642e..2ae38dbc 100644 --- a/.gitignore +++ b/.gitignore @@ -204,4 +204,7 @@ dist/ build/ # vs code stuff -.vscode \ No newline at end of file +.vscode + +# don't commit authentication info to source control +test.env diff --git a/CHANGELOG.md b/CHANGELOG.md index d424c7b3..14db0f8b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,149 @@ # Release History -## 2.5.x (Unreleased) +# 3.4.0 (2024-08-27) + +- Unpin pandas to support v2.2.2 (databricks/databricks-sql-python#416 by @kfollesdal) +- Make OAuth as the default authenticator if no authentication setting is provided (databricks/databricks-sql-python#419 by @jackyhu-db) +- Fix (regression): use SSL options with HTTPS connection pool (databricks/databricks-sql-python#425 by @kravets-levko) + +# 3.3.0 (2024-07-18) + +- Don't retry requests that fail with HTTP code 401 (databricks/databricks-sql-python#408 by @Hodnebo) +- Remove username/password (aka "basic") auth option (databricks/databricks-sql-python#409 by @jackyhu-db) +- Refactor CloudFetch handler to fix numerous issues with it (databricks/databricks-sql-python#405 by @kravets-levko) +- Add option to disable SSL verification for CloudFetch links (databricks/databricks-sql-python#414 by @kravets-levko) + +Databricks-managed passwords reached end of life on July 10, 2024. Therefore, Basic auth support was removed from +the library. See https://docs.databricks.com/en/security/auth-authz/password-deprecation.html + +The existing option `_tls_no_verify=True` of `sql.connect(...)` will now also disable SSL cert verification +(but not the SSL itself) for CloudFetch links. This option should be used as a workaround only, when other ways +to fix SSL certificate errors didn't work. + +# 3.2.0 (2024-06-06) + +- Update proxy authentication (databricks/databricks-sql-python#354 by @amir-haroun) +- Relax `pyarrow` pin (databricks/databricks-sql-python#389 by @dhirschfeld) +- Fix error logging in OAuth manager (databricks/databricks-sql-python#269 by @susodapop) +- SQLAlchemy: enable delta.feature.allowColumnDefaults for all tables (databricks/databricks-sql-python#343 by @dhirschfeld) +- Update `thrift` dependency (databricks/databricks-sql-python#397 by @m1n0) + +# 3.1.2 (2024-04-18) + +- Remove broken cookie code (#379) +- Small typing fixes (#382, #384 thanks @wyattscarpenter) + +# 3.1.1 (2024-03-19) + +- Don't retry requests that fail with code 403 (#373) +- Assume a default retry-after for 429/503 (#371) +- Fix boolean literals (#357) + +# 3.1.0 (2024-02-16) + +- Revert retry-after behavior to be exponential backoff (#349) +- Support Databricks OAuth on Azure (#351) +- Support Databricks OAuth on GCP (#338) + +# 3.0.3 (2024-02-02) + +- Revised docstrings and examples for OAuth (#339) +- Redact the URL query parameters from the urllib3.connectionpool logs (#341) + +# 3.0.2 (2024-01-25) + +- SQLAlchemy dialect now supports table and column comments (thanks @cbornet!) +- Fix: SQLAlchemy dialect now correctly reflects TINYINT types (thanks @TimTheinAtTabs!) +- Fix: `server_hostname` URIs that included `https://` would raise an exception +- Other: pinned to `pandas<=2.1` and `urllib3>=1.26` to avoid runtime errors in dbt-databricks (#330) + +## 3.0.1 (2023-12-01) + +- Other: updated docstring comment about default parameterization approach (#287) +- Other: added tests for reading complex types and revised docstrings and type hints (#293) +- Fix: SQLAlchemy dialect raised DeprecationWarning due to `dbapi` classmethod (#294) +- Fix: SQLAlchemy dialect could not reflect TIMESTAMP_NTZ columns (#296) + +## 3.0.0 (2023-11-17) + +- Remove support for Python 3.7 +- Add support for native parameterized SQL queries. Requires DBR 14.2 and above. See docs/parameters.md for more info. +- Completely rewritten SQLAlchemy dialect + - Adds support for SQLAlchemy >= 2.0 and drops support for SQLAlchemy 1.x + - Full e2e test coverage of all supported features + - Detailed usage notes in `README.sqlalchemy.md` + - Adds support for: + - New types: `TIME`, `TIMESTAMP`, `TIMESTAMP_NTZ`, `TINYINT` + - `Numeric` type scale and precision, like `Numeric(10,2)` + - Reading and writing `PrimaryKeyConstraint` and `ForeignKeyConstraint` + - Reading and writing composite keys + - Reading and writing from views + - Writing `Identity` to tables (i.e. autoincrementing primary keys) + - `LIMIT` and `OFFSET` for paging through results + - Caching metadata calls +- Enable cloud fetch by default. To disable, set `use_cloud_fetch=False` when building `databricks.sql.client`. +- Add integration tests for Databricks UC Volumes ingestion queries +- Retries: + - Add `_retry_max_redirects` config + - Set `_enable_v3_retries=True` and warn if users override it +- Security: bump minimum pyarrow version to 14.0.1 (CVE-2023-47248) + +## 2.9.3 (2023-08-24) + +- Fix: Connections failed when urllib3~=1.0.0 is installed (#206) + +## 2.9.2 (2023-08-17) + +**Note: this release was yanked from Pypi on 13 September 2023 due to compatibility issues with environments where `urllib3<=2.0.0` were installed. The log changes are incorporated into version 2.9.3 and greater.** + +- Other: Add `examples/v3_retries_query_execute.py` (#199) +- Other: suppress log message when `_enable_v3_retries` is not `True` (#199) +- Other: make this connector backwards compatible with `urllib3>=1.0.0` (#197) + +## 2.9.1 (2023-08-11) + +**Note: this release was yanked from Pypi on 13 September 2023 due to compatibility issues with environments where `urllib3<=2.0.0` were installed.** + +- Other: Explicitly pin urllib3 to ^2.0.0 (#191) + +## 2.9.0 (2023-08-10) + +- Replace retry handling with DatabricksRetryPolicy. This is disabled by default. To enable, set `_enable_v3_retries=True` when creating `databricks.sql.client` (#182) +- Other: Fix typo in README quick start example (#186) +- Other: Add autospec to Client mocks and tidy up `make_request` (#188) + +## 2.8.0 (2023-07-21) + +- Add support for Cloud Fetch. Disabled by default. Set `use_cloud_fetch=True` when building `databricks.sql.client` to enable it (#146, #151, #154) +- SQLAlchemy has_table function now honours schema= argument and adds catalog= argument (#174) +- SQLAlchemy set non_native_boolean_check_constraint False as it's not supported by Databricks (#120) +- Fix: Revised SQLAlchemy dialect and examples for compatibility with SQLAlchemy==1.3.x (#173) +- Fix: oauth would fail if expired credentials appeared in ~/.netrc (#122) +- Fix: Python HTTP proxies were broken after switch to urllib3 (#158) +- Other: remove unused import in SQLAlchemy dialect +- Other: Relax pandas dependency constraint to allow ^2.0.0 (#164) +- Other: Connector now logs operation handle guids as hexadecimal instead of bytes (#170) +- Other: test_socket_timeout_user_defined e2e test was broken (#144) + +## 2.7.0 (2023-06-26) + +- Fix: connector raised exception when calling close() on a closed Thrift session +- Improve e2e test development ergonomics +- Redact logged thrift responses by default +- Add support for OAuth on Databricks Azure + +## 2.6.2 (2023-06-14) + +- Fix: Retry GetOperationStatus requests for http errors + +## 2.6.1 (2023-06-08) + +- Fix: http.client would raise a BadStatusLine exception in some cases + +## 2.6.0 (2023-06-07) + +- Add support for HTTP 1.1 connections (connection pools) +- Add a default socket timeout for thrift RPCs ## 2.5.2 (2023-05-08) @@ -12,6 +155,7 @@ - Other: Relax sqlalchemy required version as it was unecessarily strict. ## 2.5.0 (2023-04-14) + - Add support for External Auth providers - Fix: Python HTTP proxies were broken - Other: All Thrift requests that timeout during connection will be automatically retried @@ -33,8 +177,8 @@ ## 2.2.2 (2023-01-03) -- Support custom oauth client id and redirect port -- Fix: Add none check on _oauth_persistence in DatabricksOAuthProvider +- Support custom oauth client id and redirect port +- Fix: Add none check on \_oauth_persistence in DatabricksOAuthProvider ## 2.2.1 (2022-11-29) @@ -66,57 +210,71 @@ Huge thanks to @dbaxa for contributing this change! - Add retry logic for `GetOperationStatus` requests that fail with an `OSError` - Reorganised code to use Poetry for dependency management. + ## 2.0.2 (2022-05-04) + - Better exception handling in automatic connection close ## 2.0.1 (2022-04-21) + - Fixed Pandas dependency in setup.cfg to be >= 1.2.0 ## 2.0.0 (2022-04-19) + - Initial stable release of V2 -- Added better support for complex types, so that in Databricks runtime 10.3+, Arrays, Maps and Structs will get +- Added better support for complex types, so that in Databricks runtime 10.3+, Arrays, Maps and Structs will get deserialized as lists, lists of tuples and dicts, respectively. - Changed the name of the metadata arg to http_headers ## 2.0.b2 (2022-04-04) + - Change import of collections.Iterable to collections.abc.Iterable to make the library compatible with Python 3.10 - Fixed bug with .tables method so that .tables works as expected with Unity-Catalog enabled endpoints ## 2.0.0b1 (2022-03-04) + - Fix packaging issue (dependencies were not being installed properly) - Fetching timestamp results will now return aware instead of naive timestamps - The client will now default to using simplified error messages ## 2.0.0b (2022-02-08) + - Initial beta release of V2. V2 is an internal re-write of large parts of the connector to use Databricks edge features. All public APIs from V1 remain. -- Added Unity Catalog support (pass catalog and / or schema key word args to the .connect method to select initial schema and catalog) +- Added Unity Catalog support (pass catalog and / or schema key word args to the .connect method to select initial schema and catalog) --- **Note**: The code for versions prior to `v2.0.0b` is not contained in this repository. The below entries are included for reference only. --- + ## 1.0.0 (2022-01-20) + - Add operations for retrieving metadata - Add the ability to access columns by name on result rows - Add the ability to provide configuration settings on connect ## 0.9.4 (2022-01-10) + - Improved logging and error messages. ## 0.9.3 (2021-12-08) + - Add retries for 429 and 503 HTTP responses. ## 0.9.2 (2021-12-02) + - (Bug fix) Increased Thrift requirement from 0.10.0 to 0.13.0 as 0.10.0 was in fact incompatible - (Bug fix) Fixed error message after query execution failed -SQLSTATE and Error message were misplaced ## 0.9.1 (2021-09-01) + - Public Preview release, Experimental tag removed - minor updates in internal build/packaging - no functional changes ## 0.9.0 (2021-08-04) + - initial (Experimental) release of pyhive-forked connector - Python DBAPI 2.0 (PEP-0249), thrift based - see docs for more info: https://docs.databricks.com/dev-tools/python-sql-connector.html diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index aea830eb..ce0968d4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -74,7 +74,7 @@ If you set your `user.name` and `user.email` git configs, you can sign your comm This project uses [Poetry](https://python-poetry.org/) for dependency management, tests, and linting. 1. Clone this respository -2. Run `poetry install` +2. Run `poetry install` ### Run tests @@ -107,8 +107,21 @@ End-to-end tests require a Databricks account. Before you can run them, you must export host="" export http_path="" export access_token="" +export catalog="" +export schema="" ``` +Or you can write these into a file called `test.env` in the root of the repository: + +``` +host="****.cloud.databricks.com" +http_path="/sql/1.0/warehouses/***" +access_token="dapi***" +staging_ingestion_user="***@example.com" +``` + +To see logging output from pytest while running tests, set `log_cli = "true"` under `tool.pytest.ini_options` in `pyproject.toml`. You can also set `log_cli_level` to any of the default Python log levels: `DEBUG`, `INFO`, `WARNING`, `ERROR`, `CRITICAL` + There are several e2e test suites available: - `PySQLCoreTestSuite` - `PySQLLargeQueriesSuite` @@ -130,6 +143,11 @@ The `PySQLLargeQueriesSuite` namespace contains long-running query tests and is The `PySQLStagingIngestionTestSuite` namespace requires a cluster running DBR version > 12.x which supports staging ingestion commands. The suites marked `[not documented]` require additional configuration which will be documented at a later time. + +#### SQLAlchemy dialect tests + +See README.tests.md for details. + ### Code formatting This project uses [Black](https://pypi.org/project/black/). @@ -149,5 +167,4 @@ Modify the dependency specification (syntax can be found [here](https://python-p - `poetry update` - `rm poetry.lock && poetry install` -Sometimes `poetry update` can freeze or run forever. Deleting the `poetry.lock` file and calling `poetry install` is guaranteed to update everything but is usually _slower_ than `poetry update` **if `poetry update` works at all**. - +Sometimes `poetry update` can freeze or run forever. Deleting the `poetry.lock` file and calling `poetry install` is guaranteed to update everything but is usually _slower_ than `poetry update` **if `poetry update` works at all**. diff --git a/README.md b/README.md index 60c9081c..54d4b178 100644 --- a/README.md +++ b/README.md @@ -3,7 +3,7 @@ [![PyPI](https://img.shields.io/pypi/v/databricks-sql-connector?style=flat-square)](https://pypi.org/project/databricks-sql-connector/) [![Downloads](https://pepy.tech/badge/databricks-sql-connector)](https://pepy.tech/project/databricks-sql-connector) -The Databricks SQL Connector for Python allows you to develop Python applications that connect to Databricks clusters and SQL warehouses. It is a Thrift-based client with no dependencies on ODBC or JDBC. It conforms to the [Python DB API 2.0 specification](https://www.python.org/dev/peps/pep-0249/) and exposes a [SQLAlchemy](https://www.sqlalchemy.org/) dialect for use with tools like `pandas` and `alembic` which use SQLAlchemy to execute DDL. +The Databricks SQL Connector for Python allows you to develop Python applications that connect to Databricks clusters and SQL warehouses. It is a Thrift-based client with no dependencies on ODBC or JDBC. It conforms to the [Python DB API 2.0 specification](https://www.python.org/dev/peps/pep-0249/) and exposes a [SQLAlchemy](https://www.sqlalchemy.org/) dialect for use with tools like `pandas` and `alembic` which use SQLAlchemy to execute DDL. Use `pip install databricks-sql-connector[sqlalchemy]` to install with SQLAlchemy's dependencies. `pip install databricks-sql-connector[alembic]` will install alembic's dependencies. This connector uses Arrow as the data-exchange format, and supports APIs to directly fetch Arrow tables. Arrow tables are wrapped in the `ArrowQueue` class to provide a natural API to get several rows at a time. @@ -11,7 +11,7 @@ You are welcome to file an issue here for general use cases. You can also contac ## Requirements -Python 3.7 or above is required. +Python 3.8 or above is required. ## Documentation @@ -24,12 +24,9 @@ For the latest documentation, see Install the library with `pip install databricks-sql-connector` -Note: Don't hard-code authentication secrets into your Python. Use environment variables - ```bash export DATABRICKS_HOST=********.databricks.com export DATABRICKS_HTTP_PATH=/sql/1.0/endpoints/**************** -export DATABRICKS_TOKEN=dapi******************************** ``` Example usage: @@ -39,16 +36,13 @@ from databricks import sql host = os.getenv("DATABRICKS_HOST") http_path = os.getenv("DATABRICKS_HTTP_PATH") -access_token = os.getenv("DATABRICKS_ACCESS_TOKEN") connection = sql.connect( server_hostname=host, - http_path=http_path, - access_token=access_token) + http_path=http_path) cursor = connection.cursor() - -cursor.execute('SELECT * FROM RANGE(10)') +cursor.execute('SELECT :param `p`, * FROM RANGE(10)', {"param": "foo"}) result = cursor.fetchall() for row in result: print(row) @@ -61,7 +55,10 @@ In the above example: - `server-hostname` is the Databricks instance host name. - `http-path` is the HTTP Path either to a Databricks SQL endpoint (e.g. /sql/1.0/endpoints/1234567890abcdef), or to a Databricks Runtime interactive cluster (e.g. /sql/protocolv1/o/1234567890123456/1234-123456-slid123) -- `personal-access-token` is the Databricks Personal Access Token for the account that will execute commands and queries + +> Note: This example uses [Databricks OAuth U2M](https://docs.databricks.com/en/dev-tools/auth/oauth-u2m.html) +> to authenticate the target Databricks user account and needs to open the browser for authentication. So it +> can only run on the user's machine. ## Contributing diff --git a/conftest.py b/conftest.py new file mode 100644 index 00000000..c8b350be --- /dev/null +++ b/conftest.py @@ -0,0 +1,44 @@ +import os +import pytest + + +@pytest.fixture(scope="session") +def host(): + return os.getenv("DATABRICKS_SERVER_HOSTNAME") + + +@pytest.fixture(scope="session") +def http_path(): + return os.getenv("DATABRICKS_HTTP_PATH") + + +@pytest.fixture(scope="session") +def access_token(): + return os.getenv("DATABRICKS_TOKEN") + + +@pytest.fixture(scope="session") +def ingestion_user(): + return os.getenv("DATABRICKS_USER") + + +@pytest.fixture(scope="session") +def catalog(): + return os.getenv("DATABRICKS_CATALOG") + + +@pytest.fixture(scope="session") +def schema(): + return os.getenv("DATABRICKS_SCHEMA", "default") + + +@pytest.fixture(scope="session", autouse=True) +def connection_details(host, http_path, access_token, ingestion_user, catalog, schema): + return { + "host": host, + "http_path": http_path, + "access_token": access_token, + "ingestion_user": ingestion_user, + "catalog": catalog, + "schema": schema, + } diff --git a/docs/parameters.md b/docs/parameters.md new file mode 100644 index 00000000..a538af1a --- /dev/null +++ b/docs/parameters.md @@ -0,0 +1,255 @@ +# Using Native Parameters + +This connector supports native parameterized query execution. When you execute a query that includes variable markers, then you can pass a collection of parameters which are sent separately to Databricks Runtime for safe execution. This prevents SQL injection and can improve query performance. + +This behaviour is distinct from legacy "inline" parameterized execution in versions below 3.0.0. The legacy behavior is preserved behind a flag called `use_inline_params`, which will be removed in a future release. See [Using Inline Parameters](#using-inline-parameters) for more information. + +See **[below](#migrating-to-native-parameters)** for details about updating your client code to use native parameters. + +See `examples/parameters.py` in this repository for a runnable demo. + +## Requirements + +- `databricks-sql-connector>=3.0.0` +- A SQL warehouse or all-purpose cluster running Databricks Runtime >=14.2 + +## Limitations + +- A query executed with native parameters can contain at most 255 parameter markers +- The maximum size of all parameterized values cannot exceed 1MB + +## SQL Syntax + +Variables in your SQL query can use one of three PEP-249 [paramstyles](https://peps.python.org/pep-0249/#paramstyle). A parameterized query can use exactly one paramstyle. + +|paramstyle|example|comment| +|-|-|-| +|`named`|`:param`|Parameters must be named| +|`qmark`|`?`|Parameter names are ignored| +|`pyformat`|`%(param)s`|Legacy syntax. Will be deprecated. Parameters must be named.| + +#### Example + +```sql +-- named paramstyle +SELECT * FROM table WHERE field = :value + +-- qmark paramstyle +SELECT * FROM table WHERE field = ? + +-- pyformat paramstyle (legacy) +SELECT * FROM table WHERE field = %(value)s +``` + +## Python Syntax + +This connector follows the [PEP-249 interface](https://peps.python.org/pep-0249/#id20). The expected structure of the parameter collection follows the paramstyle of the variables in your query. + +### `named` paramstyle Usage Example + +When your SQL query uses `named` paramstyle variable markers, you need specify a name for each value that corresponds to a variable marker in your query. + +Generally, you do this by passing `parameters` as a dictionary whose keys match the variables in your query. The length of the dictionary must exactly match the count of variable markers or an exception will be raised. + +```python +from databricks import sql + +with sql.connect(...) as conn: + with conn.cursor() as cursor(): + query = "SELECT field FROM table WHERE field = :value1 AND another_field = :value2" + parameters = {"value1": "foo", "value2": 20} + result = cursor.execute(query, parameters=parameters).fetchone() +``` + +This paramstyle is a drop-in replacement for the `pyformat` paramstyle which was used in connector versions below 3.0.0. It should be used going forward. + +### `qmark` paramstyle Usage Example + +When your SQL query uses `qmark` paramstyle variable markers, you only need to specify a value for each variable marker in your query. + +You do this by passing `parameters` as a list. The order of values in the list corresponds to the order of `qmark` variables in your query. The length of the list must exactly match the count of variable markers in your query or an exception will be raised. + +```python +from databricks import sql + +with sql.connect(...) as conn: + with conn.cursor() as cursor(): + query = "SELECT field FROM table WHERE field = ? AND another_field = ?" + parameters = ["foo", 20] + result = cursor.execute(query, parameters=parameters).fetchone() +``` + +The result of the above two examples is identical. + +### Legacy `pyformat` paramstyle Usage Example + +Databricks Runtime expects variable markers to use either `named` or `qmark` paramstyles. Historically, this connector used `pyformat` which Databricks Runtime does not support. So to assist assist customers transitioning their codebases from `pyformat` → `named`, we can dynamically rewrite the variable markers before sending the query to Databricks. This happens only when `use_inline_params=False`. + + This dynamic rewrite will be deprecated in a future release. New queries should be written using the `named` paramstyle instead. And users should update their client code to replace `pyformat` markers with `named` markers. + +For example: + +```sql +-- a query written for databricks-sql-connector==2.9.3 and below + +SELECT field1, field2, %(param1)s FROM table WHERE field4 = %(param2)s + +-- rewritten for databricks-sql-connector==3.0.0 and above + +SELECT field1, field2, :param1 FROM table WHERE field4 = :param2 +``` + + +**Note:** While named `pyformat` markers are transparently replaced when `use_inline_params=False`, un-named inline `%s`-style markers are ignored. If your client code makes extensive use of `%s` markers, these queries will need to be updated to use `?` markers before you can execute them when `use_inline_params=False`. See [When to use inline parameters](#when-to-use-inline-parameters) for more information. + +### Type inference + +Under the covers, parameter values are annotated with a valid Databricks SQL type. As shown in the examples above, this connector accepts primitive Python types like `int`, `str`, and `Decimal`. When this happens, the connector infers the corresponding Databricks SQL type (e.g. `INT`, `STRING`, `DECIMAL`) automatically. This means that the parameters passed to `cursor.execute()` are always wrapped in a `TDbsqlParameter` subtype prior to execution. + +Automatic inferrence is sufficient for most usages. But you can bypass the inference by explicitly setting the Databricks SQL type in your client code. All supported Databricks SQL types have `TDbsqlParameter` implementations which you can import from `databricks.sql.parameters`. + +`TDbsqlParameter` objects must always be passed within a list. Either paramstyle (`:named` or `?`) may be used. However, if your query uses the `named` paramstyle, all `TDbsqlParameter` objects must be provided a `name` when they are constructed. + +```python +from databricks import sql +from databricks.sql.parameters import StringParameter, IntegerParameter + +# with `named` markers +with sql.connect(...) as conn: + with conn.cursor() as cursor(): + query = "SELECT field FROM table WHERE field = :value1 AND another_field = :value2" + parameters = [ + StringParameter(name="value1", value="foo"), + IntegerParameter(name="value2", value=20) + ] + result = cursor.execute(query, parameters=parameters).fetchone() + +# with `?` markers +with sql.connect(...) as conn: + with conn.cursor() as cursor(): + query = "SELECT field FROM table WHERE field = ? AND another_field = ?" + parameters = [ + StringParameter(value="foo"), + IntegerParameter(value=20) + ] + result = cursor.execute(query, parameters=parameters).fetchone() +``` + +In general, we recommend using `?` markers when passing `TDbsqlParameter`'s directly. + +**Note**: When using `?` markers, you can bypass inference for _some_ parameters by passing a list containing both primitive Python types and `TDbsqlParameter` objects. `TDbsqlParameter` objects can never be passed in a dictionary. + +# Using Inline Parameters + +Since its initial release, this connector's `cursor.execute()` method has supported passing a sequence or mapping of parameter values. Prior to Databricks Runtime introducing native parameter support, however, "parameterized" queries could not be executed in a guaranteed safe manner. Instead, the connector made a best effort to escape parameter values and and render those strings inline with the query. + +This approach has several drawbacks: + +- It's not guaranteed to be safe from SQL injection +- The server could not boost performance by caching prepared statements +- The parameter marker syntax conflicted with SQL syntax in some cases + +Nevertheless, this behaviour is preserved in version 3.0.0 and above for legacy purposes. It will be removed in a subsequent major release. To enable this legacy code path, you must now construct your connection with `use_inline_params=True`. + +## Requirements + +Rendering parameters inline is supported on all versions of DBR since these queries are indistinguishable from ad-hoc query text. + + +## SQL Syntax + +Variables in your SQL query can look like `%(param)s` or like `%s`. + +#### Example + +```sql +-- pyformat paramstyle is used for named parameters +SELECT * FROM table WHERE field = %(value)s + +-- %s is used for positional parameters +SELECT * FROM table WHERE field = %s +``` + +## Python Syntax + +This connector follows the [PEP-249 interface](https://peps.python.org/pep-0249/#id20). The expected structure of the parameter collection follows the paramstyle of the variables in your query. + +### `pyformat` paramstyle Usage Example + +Parameters must be passed as a dictionary. + +```python +from databricks import sql + +with sql.connect(..., use_inline_params=True) as conn: + with conn.cursor() as cursor(): + query = "SELECT field FROM table WHERE field = %(value1)s AND another_field = %(value2)s" + parameters = {"value1": "foo", "value2": 20} + result = cursor.execute(query, parameters=parameters).fetchone() +``` + +The above query would be rendered into the following SQL: + +```sql +SELECT field FROM table WHERE field = 'foo' AND another_field = 20 +``` + +### `%s` paramstyle Usage Example + +Parameters must be passed as a list. + +```python +from databricks import sql + +with sql.connect(..., use_inline_params=True) as conn: + with conn.cursor() as cursor(): + query = "SELECT field FROM table WHERE field = %s AND another_field = %s" + parameters = ["foo", 20] + result = cursor.execute(query, parameters=parameters).fetchone() +``` + +The result of the above two examples is identical. + +**Note**: `%s` is not compliant with PEP-249 and only works due to the specific implementation of our inline renderer. + +**Note:** This `%s` syntax overlaps with valid SQL syntax around the usage of `LIKE` DML. For example if your query includes a clause like `WHERE field LIKE '%sequence'`, the parameter inlining function will raise an exception because this string appears to include an inline marker but none is provided. This means that connector versions below 3.0.0 it has been impossible to execute a query that included both parameters and LIKE wildcards. When `use_inline_params=False`, we will pass `%s` occurrences along to the database, allowing it to be used as expected in `LIKE` statements. + +### Passing sequences as parameter values + +Parameter values can also be passed as a sequence. This is typically used when writing `WHERE ... IN` clauses: + +```python +from databricks import sql + +with sql.connect(..., use_inline_params=True) as conn: + with conn.cursor() as cursor(): + query = "SELECT field FROM table WHERE field IN %(value_list)s" + parameters = {"value_list": [1,2,3,4,5]} + result = cursor.execute(query, parameters=parameters).fetchone() +``` + +Output: + +```sql +SELECT field FROM table WHERE field IN (1,2,3,4,5) +``` + +**Note**: this behavior is not specified by PEP-249 and only works due to the specific implementation of our inline renderer. + +### Migrating to native parameters + +Native parameters are meant to be a drop-in replacement for inline parameters. In most use-cases, upgrading to `databricks-sql-connector>=3.0.0` will grant an immediate improvement to safety. Plus, native parameters allow you to use SQL LIKE wildcards (`%`) in your queries which is impossible with inline parameters. Future improvements to parameterization (such as support for binding complex types like `STRUCT`, `MAP`, and `ARRAY`) will only be available when `use_inline_params=False`. + +To completely migrate, you need to [revise your SQL queries](#legacy-pyformat-paramstyle-usage-example) to use the new paramstyles. + + +### When to use inline parameters + +You should only set `use_inline_params=True` in the following cases: + +1. Your client code passes more than 255 parameters in a single query execution +2. Your client code passes parameter values greater than 1MB in a single query execution +3. Your client code makes extensive use of [`%s` positional parameter markers](#s-paramstyle-usage-example) +4. Your client code uses [sequences as parameter values](#passing-sequences-as-parameter-values) + +We expect limitations (1) and (2) to be addressed in a future Databricks Runtime release. diff --git a/examples/README.md b/examples/README.md index 4fbe8527..43d248da 100644 --- a/examples/README.md +++ b/examples/README.md @@ -7,7 +7,7 @@ We provide example scripts so you can see the connector in action for basic usag - DATABRICKS_TOKEN Follow the quick start in our [README](../README.md) to install `databricks-sql-connector` and see -how to find the hostname, http path, and access token. Note that for the OAuth examples below a +how to find the hostname, http path, and access token. Note that for the OAuth examples below a personal access token is not needed. @@ -33,9 +33,12 @@ To run all of these examples you can clone the entire repository to your disk. O - **`insert_data.py`** adds a tables called `squares` to your default catalog and inserts one hundred rows of example data. Then it fetches this data and prints it to the screen. - **`query_cancel.py`** shows how to cancel a query assuming that you can access the `Cursor` executing that query from a different thread. This is necessary because `databricks-sql-connector` does not yet implement an asynchronous API; calling `.execute()` blocks the current thread until execution completes. Therefore, the connector can't cancel queries from the same thread where they began. - **`interactive_oauth.py`** shows the simplest example of authenticating by OAuth (no need for a PAT generated in the DBSQL UI) while Bring Your Own IDP is in public preview. When you run the script it will open a browser window so you can authenticate. Afterward, the script fetches some sample data from Databricks and prints it to the screen. For this script, the OAuth token is not persisted which means you need to authenticate every time you run the script. +- **`m2m_oauth.py`** shows the simplest example of authenticating by using OAuth M2M (machine-to-machine) for service principal. - **`persistent_oauth.py`** shows a more advanced example of authenticating by OAuth while Bring Your Own IDP is in public preview. In this case, it shows how to use a sublcass of `OAuthPersistence` to reuse an OAuth token across script executions. - **`set_user_agent.py`** shows how to customize the user agent header used for Thrift commands. In this example the string `ExamplePartnerTag` will be added to the the user agent on every request. - **`staging_ingestion.py`** shows how the connector handles Databricks' experimental staging ingestion commands `GET`, `PUT`, and `REMOVE`. -- **`sqlalchemy.py`** shows a basic example of connecting to Databricks with [SQLAlchemy](https://www.sqlalchemy.org/). -- **`custom_cred_provider.py`** shows how to pass a custom credential provider to bypass connector authentication. Please install databricks-sdk prior to running this example. \ No newline at end of file +- **`sqlalchemy.py`** shows a basic example of connecting to Databricks with [SQLAlchemy 2.0](https://www.sqlalchemy.org/). +- **`custom_cred_provider.py`** shows how to pass a custom credential provider to bypass connector authentication. Please install databricks-sdk prior to running this example. +- **`v3_retries_query_execute.py`** shows how to enable v3 retries in connector version 2.9.x including how to enable retries for non-default retry cases. +- **`parameters.py`** shows how to use parameters in native and inline modes. diff --git a/examples/insert_data.py b/examples/insert_data.py index 511986aa..b304a0e9 100644 --- a/examples/insert_data.py +++ b/examples/insert_data.py @@ -18,4 +18,4 @@ result = cursor.fetchall() for row in result: - print(row) \ No newline at end of file + print(row) diff --git a/examples/interactive_oauth.py b/examples/interactive_oauth.py index c520d96a..dad5cac6 100644 --- a/examples/interactive_oauth.py +++ b/examples/interactive_oauth.py @@ -1,34 +1,20 @@ from databricks import sql import os -"""Bring Your Own Identity Provider with fined grained OAuth scopes is currently public preview on -Databricks in AWS. databricks-sql-connector supports user to machine OAuth login which means the -end user has to be present to login in a browser which will be popped up by the Python process. You -must enable OAuth in your Databricks account to run this example. More information on how to enable -OAuth in your Databricks Account in AWS can be found here: - -https://docs.databricks.com/administration-guide/account-settings-e2/single-sign-on.html +"""databricks-sql-connector supports user to machine OAuth login which means the +end user has to be present to login in a browser which will be popped up by the Python process. Pre-requisites: -- You have a Databricks account in AWS. -- You have configured OAuth in Databricks account in AWS using the link above. - You have installed a browser (Chrome, Firefox, Safari, Internet Explorer, etc) that will be accessible on the machine for performing OAuth login. This code does not persist the auth token. Hence after the Python process terminates the end user will have to login again. See examples/persistent_oauth.py to learn about persisting the token across script executions. - -Bring Your Own Identity Provider is in public preview. The API may change prior to becoming GA. -You can monitor these two links to find out when it will become generally available: - - 1. https://docs.databricks.com/administration-guide/account-settings-e2/single-sign-on.html - 2. https://docs.databricks.com/dev-tools/python-sql-connector.html """ with sql.connect(server_hostname = os.getenv("DATABRICKS_SERVER_HOSTNAME"), - http_path = os.getenv("DATABRICKS_HTTP_PATH"), - auth_type="databricks-oauth") as connection: + http_path = os.getenv("DATABRICKS_HTTP_PATH")) as connection: for x in range(1, 100): cursor = connection.cursor() diff --git a/examples/m2m_oauth.py b/examples/m2m_oauth.py new file mode 100644 index 00000000..eba2095c --- /dev/null +++ b/examples/m2m_oauth.py @@ -0,0 +1,41 @@ +import os + +from databricks.sdk.core import oauth_service_principal, Config +from databricks import sql + +""" +This example shows how to use OAuth M2M (machine-to-machine) for service principal + +Pre-requisites: +- Create service principal and OAuth secret in Account Console +- Assign the service principal to the workspace + +See more https://docs.databricks.com/en/dev-tools/authentication-oauth.html) +""" + +server_hostname = os.getenv("DATABRICKS_SERVER_HOSTNAME") + + +def credential_provider(): + config = Config( + host=f"https://{server_hostname}", + # Service Principal UUID + client_id=os.getenv("DATABRICKS_CLIENT_ID"), + # Service Principal Secret + client_secret=os.getenv("DATABRICKS_CLIENT_SECRET")) + return oauth_service_principal(config) + + +with sql.connect( + server_hostname=server_hostname, + http_path=os.getenv("DATABRICKS_HTTP_PATH"), + credentials_provider=credential_provider) as connection: + for x in range(1, 100): + cursor = connection.cursor() + cursor.execute('SELECT 1+1') + result = cursor.fetchall() + for row in result: + print(row) + cursor.close() + + connection.close() diff --git a/examples/parameters.py b/examples/parameters.py new file mode 100644 index 00000000..93136ec7 --- /dev/null +++ b/examples/parameters.py @@ -0,0 +1,121 @@ +""" +This example demonstrates how to use parameters in both native (default) and inline (legacy) mode. +""" + +from decimal import Decimal +from databricks import sql +from databricks.sql.parameters import * + +import os +from databricks import sql +from datetime import datetime +import pytz + +host = os.getenv("DATABRICKS_SERVER_HOSTNAME") +http_path = os.getenv("DATABRICKS_HTTP_PATH") +access_token = os.getenv("DATABRICKS_TOKEN") + + +native_connection = sql.connect( + server_hostname=host, http_path=http_path, access_token=access_token +) + +inline_connection = sql.connect( + server_hostname=host, + http_path=http_path, + access_token=access_token, + use_inline_params="silent", +) + +# Example 1 demonstrates how in most cases, queries written for databricks-sql-connector<3.0.0 will work +# with databricks-sql-connector>=3.0.0. This is because the default mode is native mode, which is backwards +# compatible with the legacy inline mode. + +LEGACY_NAMED_QUERY = "SELECT %(name)s `name`, %(age)s `age`, %(active)s `active`" +EX1_PARAMS = {"name": "Jane", "age": 30, "active": True} + +with native_connection.cursor() as cursor: + ex1_native_result = cursor.execute(LEGACY_NAMED_QUERY, EX1_PARAMS).fetchone() + +with inline_connection.cursor() as cursor: + ex1_inline_result = cursor.execute(LEGACY_NAMED_QUERY, EX1_PARAMS).fetchone() + +print("\nEXAMPLE 1") +print("Example 1 result in native mode\t→\t", ex1_native_result) +print("Example 1 result in inline mode\t→\t", ex1_inline_result) + + +# Example 2 shows how to update example 1 to use the new `named` parameter markers. +# This query would fail in inline mode. + +# This is an example of the automatic transformation from pyformat → named. +# The output looks like this: +# SELECT :name `name`, :age `age`, :active `active` +NATIVE_NAMED_QUERY = LEGACY_NAMED_QUERY % { + "name": ":name", + "age": ":age", + "active": ":active", +} +EX2_PARAMS = EX1_PARAMS + +with native_connection.cursor() as cursor: + ex2_named_result = cursor.execute(NATIVE_NAMED_QUERY, EX1_PARAMS).fetchone() + +with native_connection.cursor() as cursor: + ex2_pyformat_result = cursor.execute(LEGACY_NAMED_QUERY, EX1_PARAMS).fetchone() + +print("\nEXAMPLE 2") +print("Example 2 result with pyformat \t→\t", ex2_named_result) +print("Example 2 result with named \t→\t", ex2_pyformat_result) + + +# Example 3 shows how to use positional parameters. Notice the syntax is different between native and inline modes. +# No automatic transformation is done here. So the LEGACY_POSITIONAL_QUERY will not work in native mode. + +NATIVE_POSITIONAL_QUERY = "SELECT ? `name`, ? `age`, ? `active`" +LEGACY_POSITIONAL_QUERY = "SELECT %s `name`, %s `age`, %s `active`" + +EX3_PARAMS = ["Jane", 30, True] + +with native_connection.cursor() as cursor: + ex3_native_result = cursor.execute(NATIVE_POSITIONAL_QUERY, EX3_PARAMS).fetchone() + +with inline_connection.cursor() as cursor: + ex3_inline_result = cursor.execute(LEGACY_POSITIONAL_QUERY, EX3_PARAMS).fetchone() + +print("\nEXAMPLE 3") +print("Example 3 result in native mode\t→\t", ex3_native_result) +print("Example 3 result in inline mode\t→\t", ex3_inline_result) + +# Example 4 shows how to bypass type inference and set an exact Databricks SQL type for a parameter. +# This is only possible when use_inline_params=False + + +moment = datetime(2012, 10, 15, 12, 57, 18) +chicago_timezone = pytz.timezone("America/Chicago") + +# For this parameter value, we don't bypass inference. So we know that the connector +# will infer the datetime object to be a TIMESTAMP, which preserves the timezone info. +ex4_p1 = chicago_timezone.localize(moment) + +# For this parameter value, we bypass inference and set the type to TIMESTAMP_NTZ, +# which does not preserve the timezone info. Therefore we expect the timezone +# will be dropped in the roundtrip. +ex4_p2 = TimestampNTZParameter(value=ex4_p1) + +# For this parameter, we don't bypass inference. So we know that the connector +# will infer the Decimal to be a DECIMAL and will preserve its current precision and scale. +ex4_p3 = Decimal("12.3456") + +# For this parameter value, we bind a decimal with custom scale and precision +# that will result in the decimal being truncated. +ex4_p4 = DecimalParameter(value=ex4_p3, scale=4, precision=2) + +EX4_QUERY = "SELECT ? `p1`, ? `p2`, ? `p3`, ? `p4`" +EX4_PARAMS = [ex4_p1, ex4_p2, ex4_p3, ex4_p4] +with native_connection.cursor() as cursor: + result = cursor.execute(EX4_QUERY, EX4_PARAMS).fetchone() + +print("\nEXAMPLE 4") +print("Example 4 inferred result\t→\t {}\t{}".format(result.p1, result.p3)) +print("Example 4 explicit result\t→\t {}\t\t{}".format(result.p2, result.p4)) diff --git a/examples/persistent_oauth.py b/examples/persistent_oauth.py index b5b14d15..0f2ba077 100644 --- a/examples/persistent_oauth.py +++ b/examples/persistent_oauth.py @@ -1,14 +1,7 @@ -"""Bring Your Own Identity Provider with fined grained OAuth scopes is currently public preview on -Databricks in AWS. databricks-sql-connector supports user to machine OAuth login which means the -end user has to be present to login in a browser which will be popped up by the Python process. You -must enable OAuth in your Databricks account to run this example. More information on how to enable -OAuth in your Databricks Account in AWS can be found here: - -https://docs.databricks.com/administration-guide/account-settings-e2/single-sign-on.html +"""databricks-sql-connector supports user to machine OAuth login which means the +end user has to be present to login in a browser which will be popped up by the Python process. Pre-requisites: -- You have a Databricks account in AWS. -- You have configured OAuth in Databricks account in AWS using the link above. - You have installed a browser (Chrome, Firefox, Safari, Internet Explorer, etc) that will be accessible on the machine for performing OAuth login. @@ -18,12 +11,6 @@ shows which methods you may implement. For this example, the DevOnlyFilePersistence class is provided. Do not use this in production. - -Bring Your Own Identity Provider is in public preview. The API may change prior to becoming GA. -You can monitor these two links to find out when it will become generally available: - - 1. https://docs.databricks.com/administration-guide/account-settings-e2/single-sign-on.html - 2. https://docs.databricks.com/dev-tools/python-sql-connector.html """ import os @@ -36,10 +23,10 @@ class SampleOAuthPersistence(OAuthPersistence): def persist(self, hostname: str, oauth_token: OAuthToken): """To be implemented by the end user to persist in the preferred storage medium. - + OAuthToken has two properties: 1. OAuthToken.access_token - 2. OAuthToken.refresh_token + 2. OAuthToken.refresh_token Both should be persisted. """ diff --git a/examples/query_cancel.py b/examples/query_cancel.py index 59202088..4e0b74a5 100644 --- a/examples/query_cancel.py +++ b/examples/query_cancel.py @@ -19,13 +19,13 @@ def execute_really_long_query(): print("It looks like this query was cancelled.") exec_thread = threading.Thread(target=execute_really_long_query) - + print("\n Beginning to execute long query") exec_thread.start() - + # Make sure the query has started before cancelling print("\n Waiting 15 seconds before canceling", end="", flush=True) - + seconds_waited = 0 while seconds_waited < 15: seconds_waited += 1 @@ -34,7 +34,7 @@ def execute_really_long_query(): print("\n Cancelling the cursor's operation. This can take a few seconds.") cursor.cancel() - + print("\n Now checking the cursor status:") exec_thread.join(5) @@ -42,7 +42,7 @@ def execute_really_long_query(): print("\n The previous command was successfully canceled") print("\n Now reusing the cursor to run a separate query.") - + # We can still execute a new command on the cursor cursor.execute("SELECT * FROM range(3)") diff --git a/examples/query_execute.py b/examples/query_execute.py index ec79fd0e..a851ab50 100644 --- a/examples/query_execute.py +++ b/examples/query_execute.py @@ -10,4 +10,4 @@ result = cursor.fetchall() for row in result: - print(row) \ No newline at end of file + print(row) diff --git a/examples/sqlalchemy.py b/examples/sqlalchemy.py index 2c0b693a..7492dc5a 100644 --- a/examples/sqlalchemy.py +++ b/examples/sqlalchemy.py @@ -1,49 +1,45 @@ """ -databricks-sql-connector includes a SQLAlchemy dialect compatible with Databricks SQL. -It aims to be a drop-in replacement for the crflynn/sqlalchemy-databricks project, that implements -more of the Databricks API, particularly around table reflection, Alembic usage, and data -ingestion with pandas. - -Expected URI format is: databricks+thrift://token:dapi***@***.cloud.databricks.com?http_path=/sql/*** - -Because of the extent of SQLAlchemy's capabilities it isn't feasible to provide examples of every -usage in a single script, so we only provide a basic one here. More examples are found in our test -suite at tests/e2e/sqlalchemy/test_basic.py and in the PR that implements this change: - -https://github.com/databricks/databricks-sql-python/pull/57 - -# What's already supported - -Most of the functionality is demonstrated in the e2e tests mentioned above. The below list we -derived from those test method names: - - - Create and drop tables with SQLAlchemy Core - - Create and drop tables with SQLAlchemy ORM - - Read created tables via reflection - - Modify column nullability - - Insert records manually - - Insert records with pandas.to_sql (note that this does not work for DataFrames with indexes) - -This connector also aims to support Alembic for programmatic delta table schema maintenance. This -behaviour is not yet backed by integration tests, which will follow in a subsequent PR as we learn -more about customer use cases there. That said, the following behaviours have been tested manually: - - - Autogenerate revisions with alembic revision --autogenerate - - Upgrade and downgrade between revisions with `alembic upgrade ` and - `alembic downgrade ` - -# Known Gaps - - MAP, ARRAY, and STRUCT types: this dialect can read these types out as strings. But you cannot - define a SQLAlchemy model with databricks.sqlalchemy.dialect.types.DatabricksMap (e.g.) because - we haven't implemented them yet. - - Constraints: with the addition of information_schema to Unity Catalog, Databricks SQL supports - foreign key and primary key constraints. This dialect can write these constraints but the ability - for alembic to reflect and modify them programmatically has not been tested. +databricks-sql-connector includes a SQLAlchemy 2.0 dialect compatible with Databricks SQL. To install +its dependencies you can run `pip install databricks-sql-connector[sqlalchemy]`. + +The expected connection string format which you can pass to create_engine() is: + +databricks://token:dapi***@***.cloud.databricks.com?http_path=/sql/***&catalog=**&schema=** + +Our dialect implements the majority of SQLAlchemy 2.0's API. Because of the extent of SQLAlchemy's +capabilities it isn't feasible to provide examples of every usage in a single script, so we only +provide a basic one here. Learn more about usage in README.sqlalchemy.md in this repo. """ +# fmt: off + import os -from sqlalchemy.orm import declarative_base, Session -from sqlalchemy import Column, String, Integer, BOOLEAN, create_engine, select +from datetime import date, datetime, time, timedelta, timezone +from decimal import Decimal +from uuid import UUID + +# By convention, backend-specific SQLA types are defined in uppercase +# This dialect exposes Databricks SQL's TIMESTAMP and TINYINT types +# as these are not covered by the generic, camelcase types shown below +from databricks.sqlalchemy import TIMESTAMP, TINYINT + +# Beside the CamelCase types shown below, line comments reflect +# the underlying Databricks SQL / Delta table type +from sqlalchemy import ( + BigInteger, # BIGINT + Boolean, # BOOLEAN + Column, + Date, # DATE + DateTime, # TIMESTAMP_NTZ + Integer, # INTEGER + Numeric, # DECIMAL + String, # STRING + Time, # STRING + Uuid, # STRING + create_engine, + select, +) +from sqlalchemy.orm import DeclarativeBase, Session host = os.getenv("DATABRICKS_SERVER_HOSTNAME") http_path = os.getenv("DATABRICKS_HTTP_PATH") @@ -52,43 +48,127 @@ schema = os.getenv("DATABRICKS_SCHEMA") -# Extra arguments are passed untouched to the driver -# See thrift_backend.py for complete list +# Extra arguments are passed untouched to databricks-sql-connector +# See src/databricks/sql/thrift_backend.py for complete list extra_connect_args = { "_tls_verify_hostname": True, "_user_agent_entry": "PySQL Example Script", } + engine = create_engine( f"databricks://token:{access_token}@{host}?http_path={http_path}&catalog={catalog}&schema={schema}", - connect_args=extra_connect_args, + connect_args=extra_connect_args, echo=True, ) -session = Session(bind=engine) -base = declarative_base(bind=engine) - - -class SampleObject(base): - __tablename__ = "mySampleTable" - name = Column(String(255), primary_key=True) - episodes = Column(Integer) - some_bool = Column(BOOLEAN) - - -base.metadata.create_all() - -sample_object_1 = SampleObject(name="Bim Adewunmi", episodes=6, some_bool=True) -sample_object_2 = SampleObject(name="Miki Meek", episodes=12, some_bool=False) - -session.add(sample_object_1) -session.add(sample_object_2) +class Base(DeclarativeBase): + pass + + +# This object gives a usage example for each supported type +# for more details on these, see README.sqlalchemy.md +class SampleObject(Base): + __tablename__ = "pysql_sqlalchemy_example_table" + + bigint_col = Column(BigInteger, primary_key=True) + string_col = Column(String) + tinyint_col = Column(TINYINT) + int_col = Column(Integer) + numeric_col = Column(Numeric(10, 2)) + boolean_col = Column(Boolean) + date_col = Column(Date) + datetime_col = Column(TIMESTAMP) + datetime_col_ntz = Column(DateTime) + time_col = Column(Time) + uuid_col = Column(Uuid) + +# This generates a CREATE TABLE statement against the catalog and schema +# specified in the connection string +Base.metadata.create_all(engine) + +# Output SQL is: +# CREATE TABLE pysql_sqlalchemy_example_table ( +# bigint_col BIGINT NOT NULL, +# string_col STRING, +# tinyint_col SMALLINT, +# int_col INT, +# numeric_col DECIMAL(10, 2), +# boolean_col BOOLEAN, +# date_col DATE, +# datetime_col TIMESTAMP, +# datetime_col_ntz TIMESTAMP_NTZ, +# time_col STRING, +# uuid_col STRING, +# PRIMARY KEY (bigint_col) +# ) USING DELTA + +# The code that follows will INSERT a record using SQLAlchemy ORM containing these values +# and then SELECT it back out. The output is compared to the input to demonstrate that +# all type information is preserved. +sample_object = { + "bigint_col": 1234567890123456789, + "string_col": "foo", + "tinyint_col": -100, + "int_col": 5280, + "numeric_col": Decimal("525600.01"), + "boolean_col": True, + "date_col": date(2020, 12, 25), + "datetime_col": datetime( + 1991, 8, 3, 21, 30, 5, tzinfo=timezone(timedelta(hours=-8)) + ), + "datetime_col_ntz": datetime(1990, 12, 4, 6, 33, 41), + "time_col": time(23, 59, 59), + "uuid_col": UUID(int=255), +} +sa_obj = SampleObject(**sample_object) +session = Session(engine) +session.add(sa_obj) session.commit() -stmt = select(SampleObject).where(SampleObject.name.in_(["Bim Adewunmi", "Miki Meek"])) - -output = [i for i in session.scalars(stmt)] -assert len(output) == 2 - -base.metadata.drop_all() +# Output SQL is: +# INSERT INTO +# pysql_sqlalchemy_example_table ( +# bigint_col, +# string_col, +# tinyint_col, +# int_col, +# numeric_col, +# boolean_col, +# date_col, +# datetime_col, +# datetime_col_ntz, +# time_col, +# uuid_col +# ) +# VALUES +# ( +# :bigint_col, +# :string_col, +# :tinyint_col, +# :int_col, +# :numeric_col, +# :boolean_col, +# :date_col, +# :datetime_col, +# :datetime_col_ntz, +# :time_col, +# :uuid_col +# ) + +# Here we build a SELECT query using ORM +stmt = select(SampleObject).where(SampleObject.int_col == 5280) + +# Then fetch one result with session.scalar() +result = session.scalar(stmt) + +# Finally, we read out the input data and compare it to the output +compare = {key: getattr(result, key) for key in sample_object.keys()} +assert compare == sample_object + +# Then we drop the demonstration table +Base.metadata.drop_all(engine) + +# Output SQL is: +# DROP TABLE pysql_sqlalchemy_example_table diff --git a/examples/staging_ingestion.py b/examples/staging_ingestion.py index 2980506d..a55be477 100644 --- a/examples/staging_ingestion.py +++ b/examples/staging_ingestion.py @@ -24,7 +24,7 @@ Additionally, the connection can only manipulate files within the cloud storage location of the authenticated user. -To run this script: +To run this script: 1. Set the INGESTION_USER constant to the account email address of the authenticated user 2. Set the FILEPATH constant to the path of a file that will be uploaded (this example assumes its a CSV file) diff --git a/examples/v3_retries_query_execute.py b/examples/v3_retries_query_execute.py new file mode 100644 index 00000000..4b6772fe --- /dev/null +++ b/examples/v3_retries_query_execute.py @@ -0,0 +1,43 @@ +from databricks import sql +import os + +# Users of connector versions >= 2.9.0 and <= 3.0.0 can use the v3 retry behaviour by setting _enable_v3_retries=True +# This flag will be deprecated in databricks-sql-connector~=3.0.0 as it will become the default. +# +# The new retry behaviour is defined in src/databricks/sql/auth/retry.py +# +# The new retry behaviour allows users to force the connector to automatically retry requests that fail with codes +# that are not retried by default (in most cases only codes 429 and 503 are retried by default). Additional HTTP +# codes to retry are specified as a list passed to `_retry_dangerous_codes`. +# +# Note that, as implied in the name, doing this is *dangerous* and should not be configured in all usages. +# With the default behaviour, ExecuteStatement Thrift commands are only retried for codes 429 and 503 because +# we can be certain at run-time that the statement never reached Databricks compute. These codes are returned by +# the SQL gateway / load balancer. So there is no risk that retrying the request would result in a doubled +# (or tripled etc) command execution. These codes are always accompanied by a Retry-After header, which we honour. +# +# However, if your use-case emits idempotent queries such as SELECT statements, it can be helpful to retry +# for 502 (Bad Gateway) codes etc. In these cases, there is a possibility that the initial command _did_ reach +# Databricks compute and retrying it could result in additional executions. Retrying under these conditions uses +# an exponential back-off since a Retry-After header is not present. +# +# This new retry behaviour allows you to configure the maximum number of redirects that the connector will follow. +# Just set `_retry_max_redirects` to the integer number of redirects you want to allow. The default is None, +# which means all redirects will be followed. In this case, a redirect will count toward the +# _retry_stop_after_attempts_count which means that by default the connector will not enter an endless retry loop. +# +# For complete information about configuring retries, see the docstring for databricks.sql.thrift_backend.ThriftBackend + +with sql.connect(server_hostname = os.getenv("DATABRICKS_SERVER_HOSTNAME"), + http_path = os.getenv("DATABRICKS_HTTP_PATH"), + access_token = os.getenv("DATABRICKS_TOKEN"), + _enable_v3_retries = True, + _retry_dangerous_codes=[502,400], + _retry_max_redirects=2) as connection: + + with connection.cursor() as cursor: + cursor.execute("SELECT * FROM default.diamonds LIMIT 2") + result = cursor.fetchall() + + for row in result: + print(row) diff --git a/poetry.lock b/poetry.lock old mode 100644 new mode 100755 index 3c95a628..9fe49690 --- a/poetry.lock +++ b/poetry.lock @@ -1,10 +1,15 @@ +# This file is automatically @generated by Poetry 1.8.3 and should not be changed by hand. + [[package]] name = "alembic" -version = "1.10.4" +version = "1.13.2" description = "A database migration tool for SQLAlchemy." -category = "main" -optional = false -python-versions = ">=3.7" +optional = true +python-versions = ">=3.8" +files = [ + {file = "alembic-1.13.2-py3-none-any.whl", hash = "sha256:6b8733129a6224a9a711e17c99b08462dbf7cc9670ba8f2e2ae9af860ceb1953"}, + {file = "alembic-1.13.2.tar.gz", hash = "sha256:1ff0ae32975f4fd96028c39ed9bb3c867fe3af956bd7bb37343b54c9fe7445ef"}, +] [package.dependencies] importlib-metadata = {version = "*", markers = "python_version < \"3.9\""} @@ -14,30 +19,42 @@ SQLAlchemy = ">=1.3.0" typing-extensions = ">=4" [package.extras] -tz = ["python-dateutil"] +tz = ["backports.zoneinfo"] [[package]] name = "astroid" -version = "2.11.7" +version = "3.2.2" description = "An abstract syntax tree for Python with inference support." -category = "dev" optional = false -python-versions = ">=3.6.2" +python-versions = ">=3.8.0" +files = [ + {file = "astroid-3.2.2-py3-none-any.whl", hash = "sha256:e8a0083b4bb28fcffb6207a3bfc9e5d0a68be951dd7e336d5dcf639c682388c0"}, + {file = "astroid-3.2.2.tar.gz", hash = "sha256:8ead48e31b92b2e217b6c9733a21afafe479d52d6e164dd25fb1a770c7c3cf94"}, +] [package.dependencies] -lazy-object-proxy = ">=1.4.0" -setuptools = ">=20.0" -typed-ast = {version = ">=1.4.0,<2.0", markers = "implementation_name == \"cpython\" and python_version < \"3.8\""} -typing-extensions = {version = ">=3.10", markers = "python_version < \"3.10\""} -wrapt = ">=1.11,<2" +typing-extensions = {version = ">=4.0.0", markers = "python_version < \"3.11\""} [[package]] name = "black" version = "22.12.0" description = "The uncompromising code formatter." -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "black-22.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eedd20838bd5d75b80c9f5487dbcb06836a43833a37846cf1d8c1cc01cef59d"}, + {file = "black-22.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:159a46a4947f73387b4d83e87ea006dbb2337eab6c879620a3ba52699b1f4351"}, + {file = "black-22.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d30b212bffeb1e252b31dd269dfae69dd17e06d92b87ad26e23890f3efea366f"}, + {file = "black-22.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:7412e75863aa5c5411886804678b7d083c7c28421210180d67dfd8cf1221e1f4"}, + {file = "black-22.12.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c116eed0efb9ff870ded8b62fe9f28dd61ef6e9ddd28d83d7d264a38417dcee2"}, + {file = "black-22.12.0-cp37-cp37m-win_amd64.whl", hash = "sha256:1f58cbe16dfe8c12b7434e50ff889fa479072096d79f0a7f25e4ab8e94cd8350"}, + {file = "black-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77d86c9f3db9b1bf6761244bc0b3572a546f5fe37917a044e02f3166d5aafa7d"}, + {file = "black-22.12.0-cp38-cp38-win_amd64.whl", hash = "sha256:82d9fe8fee3401e02e79767016b4907820a7dc28d70d137eb397b92ef3cc5bfc"}, + {file = "black-22.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:101c69b23df9b44247bd88e1d7e90154336ac4992502d4197bdac35dd7ee3320"}, + {file = "black-22.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:559c7a1ba9a006226f09e4916060982fd27334ae1998e7a38b3f33a37f7a2148"}, + {file = "black-22.12.0-py3-none-any.whl", hash = "sha256:436cc9167dd28040ad90d3b404aec22cedf24a6e4d7de221bec2730ec0c97bcf"}, + {file = "black-22.12.0.tar.gz", hash = "sha256:229351e5a18ca30f447bf724d007f890f97e13af070bb6ad4c0a441cd7596a2f"}, +] [package.dependencies] click = ">=8.0.0" @@ -45,7 +62,6 @@ mypy-extensions = ">=0.4.3" pathspec = ">=0.9.0" platformdirs = ">=2" tomli = {version = ">=1.1.0", markers = "python_full_version < \"3.11.0a7\""} -typed-ast = {version = ">=1.4.2", markers = "python_version < \"3.8\" and implementation_name == \"cpython\""} typing-extensions = {version = ">=3.10.0.0", markers = "python_version < \"3.10\""} [package.extras] @@ -56,159 +72,367 @@ uvloop = ["uvloop (>=0.15.2)"] [[package]] name = "certifi" -version = "2023.5.7" +version = "2024.6.2" description = "Python package for providing Mozilla's CA Bundle." -category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "certifi-2024.6.2-py3-none-any.whl", hash = "sha256:ddc6c8ce995e6987e7faf5e3f1b02b302836a0e5d98ece18392cb1a36c72ad56"}, + {file = "certifi-2024.6.2.tar.gz", hash = "sha256:3cd43f1c6fa7dedc5899d69d3ad0398fd018ad1a17fba83ddaf78aa46c747516"}, +] [[package]] name = "charset-normalizer" -version = "3.1.0" +version = "3.3.2" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." -category = "main" optional = false python-versions = ">=3.7.0" +files = [ + {file = "charset-normalizer-3.3.2.tar.gz", hash = "sha256:f30c3cb33b24454a82faecaf01b19c18562b1e89558fb6c56de4d9118a032fd5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:25baf083bf6f6b341f4121c2f3c548875ee6f5339300e08be3f2b2ba1721cdd3"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:06435b539f889b1f6f4ac1758871aae42dc3a8c0e24ac9e60c2384973ad73027"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9063e24fdb1e498ab71cb7419e24622516c4a04476b17a2dab57e8baa30d6e03"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6897af51655e3691ff853668779c7bad41579facacf5fd7253b0133308cf000d"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1d3193f4a680c64b4b6a9115943538edb896edc190f0b222e73761716519268e"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:cd70574b12bb8a4d2aaa0094515df2463cb429d8536cfb6c7ce983246983e5a6"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8465322196c8b4d7ab6d1e049e4c5cb460d0394da4a27d23cc242fbf0034b6b5"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:a9a8e9031d613fd2009c182b69c7b2c1ef8239a0efb1df3f7c8da66d5dd3d537"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:beb58fe5cdb101e3a055192ac291b7a21e3b7ef4f67fa1d74e331a7f2124341c"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e06ed3eb3218bc64786f7db41917d4e686cc4856944f53d5bdf83a6884432e12"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:2e81c7b9c8979ce92ed306c249d46894776a909505d8f5a4ba55b14206e3222f"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:572c3763a264ba47b3cf708a44ce965d98555f618ca42c926a9c1616d8f34269"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fd1abc0d89e30cc4e02e4064dc67fcc51bd941eb395c502aac3ec19fab46b519"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win32.whl", hash = "sha256:3d47fa203a7bd9c5b6cee4736ee84ca03b8ef23193c0d1ca99b5089f72645c73"}, + {file = "charset_normalizer-3.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:10955842570876604d404661fbccbc9c7e684caf432c09c715ec38fbae45ae09"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:802fe99cca7457642125a8a88a084cef28ff0cf9407060f7b93dca5aa25480db"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:573f6eac48f4769d667c4442081b1794f52919e7edada77495aaed9236d13a96"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:549a3a73da901d5bc3ce8d24e0600d1fa85524c10287f6004fbab87672bf3e1e"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f27273b60488abe721a075bcca6d7f3964f9f6f067c8c4c605743023d7d3944f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1ceae2f17a9c33cb48e3263960dc5fc8005351ee19db217e9b1bb15d28c02574"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:65f6f63034100ead094b8744b3b97965785388f308a64cf8d7c34f2f2e5be0c4"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:753f10e867343b4511128c6ed8c82f7bec3bd026875576dfd88483c5c73b2fd8"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4a78b2b446bd7c934f5dcedc588903fb2f5eec172f3d29e52a9096a43722adfc"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:e537484df0d8f426ce2afb2d0f8e1c3d0b114b83f8850e5f2fbea0e797bd82ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:eb6904c354526e758fda7167b33005998fb68c46fbc10e013ca97f21ca5c8887"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:deb6be0ac38ece9ba87dea880e438f25ca3eddfac8b002a2ec3d9183a454e8ae"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4ab2fe47fae9e0f9dee8c04187ce5d09f48eabe611be8259444906793ab7cbce"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:80402cd6ee291dcb72644d6eac93785fe2c8b9cb30893c1af5b8fdd753b9d40f"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win32.whl", hash = "sha256:7cd13a2e3ddeed6913a65e66e94b51d80a041145a026c27e6bb76c31a853c6ab"}, + {file = "charset_normalizer-3.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:663946639d296df6a2bb2aa51b60a2454ca1cb29835324c640dafb5ff2131a77"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:0b2b64d2bb6d3fb9112bafa732def486049e63de9618b5843bcdd081d8144cd8"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:ddbb2551d7e0102e7252db79ba445cdab71b26640817ab1e3e3648dad515003b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:55086ee1064215781fff39a1af09518bc9255b50d6333f2e4c74ca09fac6a8f6"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8f4a014bc36d3c57402e2977dada34f9c12300af536839dc38c0beab8878f38a"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a10af20b82360ab00827f916a6058451b723b4e65030c5a18577c8b2de5b3389"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:8d756e44e94489e49571086ef83b2bb8ce311e730092d2c34ca8f7d925cb20aa"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:90d558489962fd4918143277a773316e56c72da56ec7aa3dc3dbbe20fdfed15b"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6ac7ffc7ad6d040517be39eb591cac5ff87416c2537df6ba3cba3bae290c0fed"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:7ed9e526742851e8d5cc9e6cf41427dfc6068d4f5a3bb03659444b4cabf6bc26"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:8bdb58ff7ba23002a4c5808d608e4e6c687175724f54a5dade5fa8c67b604e4d"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:6b3251890fff30ee142c44144871185dbe13b11bab478a88887a639655be1068"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b4a23f61ce87adf89be746c8a8974fe1c823c891d8f86eb218bb957c924bb143"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:efcb3f6676480691518c177e3b465bcddf57cea040302f9f4e6e191af91174d4"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win32.whl", hash = "sha256:d965bba47ddeec8cd560687584e88cf699fd28f192ceb452d1d7ee807c5597b7"}, + {file = "charset_normalizer-3.3.2-cp312-cp312-win_amd64.whl", hash = "sha256:96b02a3dc4381e5494fad39be677abcb5e6634bf7b4fa83a6dd3112607547001"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:95f2a5796329323b8f0512e09dbb7a1860c46a39da62ecb2324f116fa8fdc85c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c002b4ffc0be611f0d9da932eb0f704fe2602a9a949d1f738e4c34c75b0863d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a981a536974bbc7a512cf44ed14938cf01030a99e9b3a06dd59578882f06f985"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3287761bc4ee9e33561a7e058c72ac0938c4f57fe49a09eae428fd88aafe7bb6"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:42cb296636fcc8b0644486d15c12376cb9fa75443e00fb25de0b8602e64c1714"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0a55554a2fa0d408816b3b5cedf0045f4b8e1a6065aec45849de2d6f3f8e9786"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:c083af607d2515612056a31f0a8d9e0fcb5876b7bfc0abad3ecd275bc4ebc2d5"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:87d1351268731db79e0f8e745d92493ee2841c974128ef629dc518b937d9194c"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:bd8f7df7d12c2db9fab40bdd87a7c09b1530128315d047a086fa3ae3435cb3a8"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:c180f51afb394e165eafe4ac2936a14bee3eb10debc9d9e4db8958fe36afe711"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:8c622a5fe39a48f78944a87d4fb8a53ee07344641b0562c540d840748571b811"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win32.whl", hash = "sha256:db364eca23f876da6f9e16c9da0df51aa4f104a972735574842618b8c6d999d4"}, + {file = "charset_normalizer-3.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:86216b5cee4b06df986d214f664305142d9c76df9b6512be2738aa72a2048f99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:6463effa3186ea09411d50efc7d85360b38d5f09b870c48e4600f63af490e56a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:6c4caeef8fa63d06bd437cd4bdcf3ffefe6738fb1b25951440d80dc7df8c03ac"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37e55c8e51c236f95b033f6fb391d7d7970ba5fe7ff453dad675e88cf303377a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fb69256e180cb6c8a894fee62b3afebae785babc1ee98b81cdf68bbca1987f33"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ae5f4161f18c61806f411a13b0310bea87f987c7d2ecdbdaad0e94eb2e404238"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b2b0a0c0517616b6869869f8c581d4eb2dd83a4d79e0ebcb7d373ef9956aeb0a"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:45485e01ff4d3630ec0d9617310448a8702f70e9c01906b0d0118bdf9d124cf2"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:eb00ed941194665c332bf8e078baf037d6c35d7c4f3102ea2d4f16ca94a26dc8"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:2127566c664442652f024c837091890cb1942c30937add288223dc895793f898"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:a50aebfa173e157099939b17f18600f72f84eed3049e743b68ad15bd69b6bf99"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:4d0d1650369165a14e14e1e47b372cfcb31d6ab44e6e33cb2d4e57265290044d"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:923c0c831b7cfcb071580d3f46c4baf50f174be571576556269530f4bbd79d04"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:06a81e93cd441c56a9b65d8e1d043daeb97a3d0856d177d5c90ba85acb3db087"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win32.whl", hash = "sha256:6ef1d82a3af9d3eecdba2321dc1b3c238245d890843e040e41e470ffa64c3e25"}, + {file = "charset_normalizer-3.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:eb8821e09e916165e160797a6c17edda0679379a4be5c716c260e836e122f54b"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:c235ebd9baae02f1b77bcea61bce332cb4331dc3617d254df3323aa01ab47bd4"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b4c145409bef602a690e7cfad0a15a55c13320ff7a3ad7ca59c13bb8ba4d45d"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:68d1f8a9e9e37c1223b656399be5d6b448dea850bed7d0f87a8311f1ff3dabb0"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22afcb9f253dac0696b5a4be4a1c0f8762f8239e21b99680099abd9b2b1b2269"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e27ad930a842b4c5eb8ac0016b0a54f5aebbe679340c26101df33424142c143c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1f79682fbe303db92bc2b1136016a38a42e835d932bab5b3b1bfcfbf0640e519"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b261ccdec7821281dade748d088bb6e9b69e6d15b30652b74cbbac25e280b796"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:122c7fa62b130ed55f8f285bfd56d5f4b4a5b503609d181f9ad85e55c89f4185"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:d0eccceffcb53201b5bfebb52600a5fb483a20b61da9dbc885f8b103cbe7598c"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:9f96df6923e21816da7e0ad3fd47dd8f94b2a5ce594e00677c0013018b813458"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:7f04c839ed0b6b98b1a7501a002144b76c18fb1c1850c8b98d458ac269e26ed2"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:34d1c8da1e78d2e001f363791c98a272bb734000fcef47a491c1e3b0505657a8"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:ff8fa367d09b717b2a17a052544193ad76cd49979c805768879cb63d9ca50561"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win32.whl", hash = "sha256:aed38f6e4fb3f5d6bf81bfa990a07806be9d83cf7bacef998ab1a9bd660a581f"}, + {file = "charset_normalizer-3.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:b01b88d45a6fcb69667cd6d2f7a9aeb4bf53760d7fc536bf679ec94fe9f3ff3d"}, + {file = "charset_normalizer-3.3.2-py3-none-any.whl", hash = "sha256:3e4d1f6587322d2788836a99c69062fbb091331ec940e02d12d179c1d53e25fc"}, +] [[package]] name = "click" -version = "8.1.3" +version = "8.1.7" description = "Composable command line interface toolkit" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "click-8.1.7-py3-none-any.whl", hash = "sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28"}, + {file = "click-8.1.7.tar.gz", hash = "sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de"}, +] [package.dependencies] colorama = {version = "*", markers = "platform_system == \"Windows\""} -importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} [[package]] name = "colorama" version = "0.4.6" description = "Cross-platform colored terminal text." -category = "dev" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*,>=2.7" +files = [ + {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, + {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, +] [[package]] name = "dill" -version = "0.3.6" -description = "serialize all of python" -category = "dev" +version = "0.3.8" +description = "serialize all of Python" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +files = [ + {file = "dill-0.3.8-py3-none-any.whl", hash = "sha256:c36ca9ffb54365bdd2f8eb3eff7d2a21237f8452b57ace88b1ac615b7e815bd7"}, + {file = "dill-0.3.8.tar.gz", hash = "sha256:3ebe3c479ad625c4553aca177444d89b486b1d84982eeacded644afc0cf797ca"}, +] [package.extras] graph = ["objgraph (>=1.7.2)"] +profile = ["gprof2dot (>=2022.7.29)"] [[package]] name = "et-xmlfile" version = "1.1.0" description = "An implementation of lxml.xmlfile for the standard library" -category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "et_xmlfile-1.1.0-py3-none-any.whl", hash = "sha256:a2ba85d1d6a74ef63837eed693bcb89c3f752169b0e3e7ae5b16ca5e1b3deada"}, + {file = "et_xmlfile-1.1.0.tar.gz", hash = "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c"}, +] [[package]] name = "exceptiongroup" -version = "1.1.1" +version = "1.2.1" description = "Backport of PEP 654 (exception groups)" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "exceptiongroup-1.2.1-py3-none-any.whl", hash = "sha256:5258b9ed329c5bbdd31a309f53cbfb0b155341807f6ff7606a1e801a891b29ad"}, + {file = "exceptiongroup-1.2.1.tar.gz", hash = "sha256:a4785e48b045528f5bfe627b6ad554ff32def154f42372786903b7abcfe1aa16"}, +] [package.extras] test = ["pytest (>=6)"] [[package]] name = "greenlet" -version = "2.0.2" +version = "3.0.3" description = "Lightweight in-process concurrent programming" -category = "main" -optional = false -python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*" +optional = true +python-versions = ">=3.7" +files = [ + {file = "greenlet-3.0.3-cp310-cp310-macosx_11_0_universal2.whl", hash = "sha256:9da2bd29ed9e4f15955dd1595ad7bc9320308a3b766ef7f837e23ad4b4aac31a"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d353cadd6083fdb056bb46ed07e4340b0869c305c8ca54ef9da3421acbdf6881"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:dca1e2f3ca00b84a396bc1bce13dd21f680f035314d2379c4160c98153b2059b"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:3ed7fb269f15dc662787f4119ec300ad0702fa1b19d2135a37c2c4de6fadfd4a"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd4f49ae60e10adbc94b45c0b5e6a179acc1736cf7a90160b404076ee283cf83"}, + {file = "greenlet-3.0.3-cp310-cp310-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:73a411ef564e0e097dbe7e866bb2dda0f027e072b04da387282b02c308807405"}, + {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:7f362975f2d179f9e26928c5b517524e89dd48530a0202570d55ad6ca5d8a56f"}, + {file = "greenlet-3.0.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:649dde7de1a5eceb258f9cb00bdf50e978c9db1b996964cd80703614c86495eb"}, + {file = "greenlet-3.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:68834da854554926fbedd38c76e60c4a2e3198c6fbed520b106a8986445caaf9"}, + {file = "greenlet-3.0.3-cp311-cp311-macosx_11_0_universal2.whl", hash = "sha256:b1b5667cced97081bf57b8fa1d6bfca67814b0afd38208d52538316e9422fc61"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:52f59dd9c96ad2fc0d5724107444f76eb20aaccb675bf825df6435acb7703559"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:afaff6cf5200befd5cec055b07d1c0a5a06c040fe5ad148abcd11ba6ab9b114e"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:fe754d231288e1e64323cfad462fcee8f0288654c10bdf4f603a39ed923bef33"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2797aa5aedac23af156bbb5a6aa2cd3427ada2972c828244eb7d1b9255846379"}, + {file = "greenlet-3.0.3-cp311-cp311-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:b7f009caad047246ed379e1c4dbcb8b020f0a390667ea74d2387be2998f58a22"}, + {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:c5e1536de2aad7bf62e27baf79225d0d64360d4168cf2e6becb91baf1ed074f3"}, + {file = "greenlet-3.0.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:894393ce10ceac937e56ec00bb71c4c2f8209ad516e96033e4b3b1de270e200d"}, + {file = "greenlet-3.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:1ea188d4f49089fc6fb283845ab18a2518d279c7cd9da1065d7a84e991748728"}, + {file = "greenlet-3.0.3-cp312-cp312-macosx_11_0_universal2.whl", hash = "sha256:70fb482fdf2c707765ab5f0b6655e9cfcf3780d8d87355a063547b41177599be"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d4d1ac74f5c0c0524e4a24335350edad7e5f03b9532da7ea4d3c54d527784f2e"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:149e94a2dd82d19838fe4b2259f1b6b9957d5ba1b25640d2380bea9c5df37676"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:15d79dd26056573940fcb8c7413d84118086f2ec1a8acdfa854631084393efcc"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:881b7db1ebff4ba09aaaeae6aa491daeb226c8150fc20e836ad00041bcb11230"}, + {file = "greenlet-3.0.3-cp312-cp312-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:fcd2469d6a2cf298f198f0487e0a5b1a47a42ca0fa4dfd1b6862c999f018ebbf"}, + {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:1f672519db1796ca0d8753f9e78ec02355e862d0998193038c7073045899f305"}, + {file = "greenlet-3.0.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:2516a9957eed41dd8f1ec0c604f1cdc86758b587d964668b5b196a9db5bfcde6"}, + {file = "greenlet-3.0.3-cp312-cp312-win_amd64.whl", hash = "sha256:bba5387a6975598857d86de9eac14210a49d554a77eb8261cc68b7d082f78ce2"}, + {file = "greenlet-3.0.3-cp37-cp37m-macosx_11_0_universal2.whl", hash = "sha256:5b51e85cb5ceda94e79d019ed36b35386e8c37d22f07d6a751cb659b180d5274"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:daf3cb43b7cf2ba96d614252ce1684c1bccee6b2183a01328c98d36fcd7d5cb0"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:99bf650dc5d69546e076f413a87481ee1d2d09aaaaaca058c9251b6d8c14783f"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:2dd6e660effd852586b6a8478a1d244b8dc90ab5b1321751d2ea15deb49ed414"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e3391d1e16e2a5a1507d83e4a8b100f4ee626e8eca43cf2cadb543de69827c4c"}, + {file = "greenlet-3.0.3-cp37-cp37m-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:e1f145462f1fa6e4a4ae3c0f782e580ce44d57c8f2c7aae1b6fa88c0b2efdb41"}, + {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:1a7191e42732df52cb5f39d3527217e7ab73cae2cb3694d241e18f53d84ea9a7"}, + {file = "greenlet-3.0.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0448abc479fab28b00cb472d278828b3ccca164531daab4e970a0458786055d6"}, + {file = "greenlet-3.0.3-cp37-cp37m-win32.whl", hash = "sha256:b542be2440edc2d48547b5923c408cbe0fc94afb9f18741faa6ae970dbcb9b6d"}, + {file = "greenlet-3.0.3-cp37-cp37m-win_amd64.whl", hash = "sha256:01bc7ea167cf943b4c802068e178bbf70ae2e8c080467070d01bfa02f337ee67"}, + {file = "greenlet-3.0.3-cp38-cp38-macosx_11_0_universal2.whl", hash = "sha256:1996cb9306c8595335bb157d133daf5cf9f693ef413e7673cb07e3e5871379ca"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3ddc0f794e6ad661e321caa8d2f0a55ce01213c74722587256fb6566049a8b04"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:c9db1c18f0eaad2f804728c67d6c610778456e3e1cc4ab4bbd5eeb8e6053c6fc"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:7170375bcc99f1a2fbd9c306f5be8764eaf3ac6b5cb968862cad4c7057756506"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b66c9c1e7ccabad3a7d037b2bcb740122a7b17a53734b7d72a344ce39882a1b"}, + {file = "greenlet-3.0.3-cp38-cp38-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:098d86f528c855ead3479afe84b49242e174ed262456c342d70fc7f972bc13c4"}, + {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:81bb9c6d52e8321f09c3d165b2a78c680506d9af285bfccbad9fb7ad5a5da3e5"}, + {file = "greenlet-3.0.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:fd096eb7ffef17c456cfa587523c5f92321ae02427ff955bebe9e3c63bc9f0da"}, + {file = "greenlet-3.0.3-cp38-cp38-win32.whl", hash = "sha256:d46677c85c5ba00a9cb6f7a00b2bfa6f812192d2c9f7d9c4f6a55b60216712f3"}, + {file = "greenlet-3.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:419b386f84949bf0e7c73e6032e3457b82a787c1ab4a0e43732898a761cc9dbf"}, + {file = "greenlet-3.0.3-cp39-cp39-macosx_11_0_universal2.whl", hash = "sha256:da70d4d51c8b306bb7a031d5cff6cc25ad253affe89b70352af5f1cb68e74b53"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:086152f8fbc5955df88382e8a75984e2bb1c892ad2e3c80a2508954e52295257"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:d73a9fe764d77f87f8ec26a0c85144d6a951a6c438dfe50487df5595c6373eac"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:b7dcbe92cc99f08c8dd11f930de4d99ef756c3591a5377d1d9cd7dd5e896da71"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1551a8195c0d4a68fac7a4325efac0d541b48def35feb49d803674ac32582f61"}, + {file = "greenlet-3.0.3-cp39-cp39-manylinux_2_24_x86_64.manylinux_2_28_x86_64.whl", hash = "sha256:64d7675ad83578e3fc149b617a444fab8efdafc9385471f868eb5ff83e446b8b"}, + {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:b37eef18ea55f2ffd8f00ff8fe7c8d3818abd3e25fb73fae2ca3b672e333a7a6"}, + {file = "greenlet-3.0.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:77457465d89b8263bca14759d7c1684df840b6811b2499838cc5b040a8b5b113"}, + {file = "greenlet-3.0.3-cp39-cp39-win32.whl", hash = "sha256:57e8974f23e47dac22b83436bdcf23080ade568ce77df33159e019d161ce1d1e"}, + {file = "greenlet-3.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:c5ee858cfe08f34712f548c3c363e807e7186f03ad7a5039ebadb29e8c6be067"}, + {file = "greenlet-3.0.3.tar.gz", hash = "sha256:43374442353259554ce33599da8b692d5aa96f8976d567d4badf263371fbe491"}, +] [package.extras] -docs = ["Sphinx", "docutils (<0.18)"] +docs = ["Sphinx", "furo"] test = ["objgraph", "psutil"] [[package]] name = "idna" -version = "3.4" +version = "3.7" description = "Internationalized Domain Names in Applications (IDNA)" -category = "main" optional = false python-versions = ">=3.5" +files = [ + {file = "idna-3.7-py3-none-any.whl", hash = "sha256:82fee1fc78add43492d3a1898bfa6d8a904cc97d8427f683ed8e798d07761aa0"}, + {file = "idna-3.7.tar.gz", hash = "sha256:028ff3aadf0609c1fd278d8ea3089299412a7a8b9bd005dd08b9f8285bcb5cfc"}, +] [[package]] name = "importlib-metadata" -version = "6.6.0" +version = "8.0.0" description = "Read metadata from Python packages" -category = "main" -optional = false -python-versions = ">=3.7" +optional = true +python-versions = ">=3.8" +files = [ + {file = "importlib_metadata-8.0.0-py3-none-any.whl", hash = "sha256:15584cf2b1bf449d98ff8a6ff1abef57bf20f3ac6454f431736cd3e660921b2f"}, + {file = "importlib_metadata-8.0.0.tar.gz", hash = "sha256:188bd24e4c346d3f0a933f275c2fec67050326a856b9a359881d7c2a697e8812"}, +] [package.dependencies] -typing-extensions = {version = ">=3.6.4", markers = "python_version < \"3.8\""} zipp = ">=0.5" [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] perf = ["ipython"] -testing = ["flake8 (<5)", "flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf (>=0.9.2)"] +test = ["flufl.flake8", "importlib-resources (>=1.3)", "jaraco.test (>=5.4)", "packaging", "pyfakefs", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-perf (>=0.9.2)", "pytest-ruff (>=0.2.1)"] [[package]] name = "importlib-resources" -version = "5.12.0" +version = "6.4.0" description = "Read resources from Python packages" -category = "main" -optional = false -python-versions = ">=3.7" +optional = true +python-versions = ">=3.8" +files = [ + {file = "importlib_resources-6.4.0-py3-none-any.whl", hash = "sha256:50d10f043df931902d4194ea07ec57960f66a80449ff867bfe782b4c486ba78c"}, + {file = "importlib_resources-6.4.0.tar.gz", hash = "sha256:cdb2b453b8046ca4e3798eb1d84f3cce1446a0e8e7b5ef4efb600f19fc398145"}, +] [package.dependencies] zipp = {version = ">=3.1.0", markers = "python_version < \"3.10\""} [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +docs = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (<7.2.5)", "sphinx (>=3.5)", "sphinx-lint"] +testing = ["jaraco.test (>=5.4)", "pytest (>=6)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-mypy", "pytest-ruff (>=0.2.1)", "zipp (>=3.17)"] [[package]] name = "iniconfig" version = "2.0.0" description = "brain-dead simple config-ini parsing" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, + {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, +] [[package]] name = "isort" -version = "5.11.5" +version = "5.13.2" description = "A Python utility / library to sort Python imports." -category = "dev" optional = false -python-versions = ">=3.7.0" +python-versions = ">=3.8.0" +files = [ + {file = "isort-5.13.2-py3-none-any.whl", hash = "sha256:8ca5e72a8d85860d5a3fa69b8745237f2939afe12dbf656afbcb47fe72d947a6"}, + {file = "isort-5.13.2.tar.gz", hash = "sha256:48fdfcb9face5d58a4f6dde2e72a1fb8dcaf8ab26f95ab49fab84c2ddefb0109"}, +] [package.extras] -colors = ["colorama (>=0.4.3,<0.5.0)"] -pipfile-deprecated-finder = ["pip-shims (>=0.5.2)", "pipreqs", "requirementslib"] -plugins = ["setuptools"] -requirements-deprecated-finder = ["pip-api", "pipreqs"] - -[[package]] -name = "lazy-object-proxy" -version = "1.9.0" -description = "A fast and thorough lazy object proxy." -category = "dev" -optional = false -python-versions = ">=3.7" +colors = ["colorama (>=0.4.6)"] [[package]] name = "lz4" -version = "4.3.2" +version = "4.3.3" description = "LZ4 Bindings for Python" -category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +files = [ + {file = "lz4-4.3.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b891880c187e96339474af2a3b2bfb11a8e4732ff5034be919aa9029484cd201"}, + {file = "lz4-4.3.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:222a7e35137d7539c9c33bb53fcbb26510c5748779364014235afc62b0ec797f"}, + {file = "lz4-4.3.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f76176492ff082657ada0d0f10c794b6da5800249ef1692b35cf49b1e93e8ef7"}, + {file = "lz4-4.3.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f1d18718f9d78182c6b60f568c9a9cec8a7204d7cb6fad4e511a2ef279e4cb05"}, + {file = "lz4-4.3.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6cdc60e21ec70266947a48839b437d46025076eb4b12c76bd47f8e5eb8a75dcc"}, + {file = "lz4-4.3.3-cp310-cp310-win32.whl", hash = "sha256:c81703b12475da73a5d66618856d04b1307e43428a7e59d98cfe5a5d608a74c6"}, + {file = "lz4-4.3.3-cp310-cp310-win_amd64.whl", hash = "sha256:43cf03059c0f941b772c8aeb42a0813d68d7081c009542301637e5782f8a33e2"}, + {file = "lz4-4.3.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:30e8c20b8857adef7be045c65f47ab1e2c4fabba86a9fa9a997d7674a31ea6b6"}, + {file = "lz4-4.3.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2f7b1839f795315e480fb87d9bc60b186a98e3e5d17203c6e757611ef7dcef61"}, + {file = "lz4-4.3.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:edfd858985c23523f4e5a7526ca6ee65ff930207a7ec8a8f57a01eae506aaee7"}, + {file = "lz4-4.3.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0e9c410b11a31dbdc94c05ac3c480cb4b222460faf9231f12538d0074e56c563"}, + {file = "lz4-4.3.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d2507ee9c99dbddd191c86f0e0c8b724c76d26b0602db9ea23232304382e1f21"}, + {file = "lz4-4.3.3-cp311-cp311-win32.whl", hash = "sha256:f180904f33bdd1e92967923a43c22899e303906d19b2cf8bb547db6653ea6e7d"}, + {file = "lz4-4.3.3-cp311-cp311-win_amd64.whl", hash = "sha256:b14d948e6dce389f9a7afc666d60dd1e35fa2138a8ec5306d30cd2e30d36b40c"}, + {file = "lz4-4.3.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:e36cd7b9d4d920d3bfc2369840da506fa68258f7bb176b8743189793c055e43d"}, + {file = "lz4-4.3.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:31ea4be9d0059c00b2572d700bf2c1bc82f241f2c3282034a759c9a4d6ca4dc2"}, + {file = "lz4-4.3.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:33c9a6fd20767ccaf70649982f8f3eeb0884035c150c0b818ea660152cf3c809"}, + {file = "lz4-4.3.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:bca8fccc15e3add173da91be8f34121578dc777711ffd98d399be35487c934bf"}, + {file = "lz4-4.3.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e7d84b479ddf39fe3ea05387f10b779155fc0990125f4fb35d636114e1c63a2e"}, + {file = "lz4-4.3.3-cp312-cp312-win32.whl", hash = "sha256:337cb94488a1b060ef1685187d6ad4ba8bc61d26d631d7ba909ee984ea736be1"}, + {file = "lz4-4.3.3-cp312-cp312-win_amd64.whl", hash = "sha256:5d35533bf2cee56f38ced91f766cd0038b6abf46f438a80d50c52750088be93f"}, + {file = "lz4-4.3.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:363ab65bf31338eb364062a15f302fc0fab0a49426051429866d71c793c23394"}, + {file = "lz4-4.3.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0a136e44a16fc98b1abc404fbabf7f1fada2bdab6a7e970974fb81cf55b636d0"}, + {file = "lz4-4.3.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:abc197e4aca8b63f5ae200af03eb95fb4b5055a8f990079b5bdf042f568469dd"}, + {file = "lz4-4.3.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:56f4fe9c6327adb97406f27a66420b22ce02d71a5c365c48d6b656b4aaeb7775"}, + {file = "lz4-4.3.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f0e822cd7644995d9ba248cb4b67859701748a93e2ab7fc9bc18c599a52e4604"}, + {file = "lz4-4.3.3-cp38-cp38-win32.whl", hash = "sha256:24b3206de56b7a537eda3a8123c644a2b7bf111f0af53bc14bed90ce5562d1aa"}, + {file = "lz4-4.3.3-cp38-cp38-win_amd64.whl", hash = "sha256:b47839b53956e2737229d70714f1d75f33e8ac26e52c267f0197b3189ca6de24"}, + {file = "lz4-4.3.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6756212507405f270b66b3ff7f564618de0606395c0fe10a7ae2ffcbbe0b1fba"}, + {file = "lz4-4.3.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ee9ff50557a942d187ec85462bb0960207e7ec5b19b3b48949263993771c6205"}, + {file = "lz4-4.3.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2b901c7784caac9a1ded4555258207d9e9697e746cc8532129f150ffe1f6ba0d"}, + {file = "lz4-4.3.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b6d9ec061b9eca86e4dcc003d93334b95d53909afd5a32c6e4f222157b50c071"}, + {file = "lz4-4.3.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f4c7bf687303ca47d69f9f0133274958fd672efaa33fb5bcde467862d6c621f0"}, + {file = "lz4-4.3.3-cp39-cp39-win32.whl", hash = "sha256:054b4631a355606e99a42396f5db4d22046a3397ffc3269a348ec41eaebd69d2"}, + {file = "lz4-4.3.3-cp39-cp39-win_amd64.whl", hash = "sha256:eac9af361e0d98335a02ff12fb56caeb7ea1196cf1a49dbf6f17828a131da807"}, + {file = "lz4-4.3.3.tar.gz", hash = "sha256:01fe674ef2889dbb9899d8a67361e0c4a2c833af5aeb37dd505727cf5d2a131e"}, +] [package.extras] docs = ["sphinx (>=1.6.0)", "sphinx-bootstrap-theme"] @@ -217,14 +441,16 @@ tests = ["psutil", "pytest (!=3.3.0)", "pytest-cov"] [[package]] name = "mako" -version = "1.2.4" +version = "1.3.5" description = "A super-fast templating language that borrows the best ideas from the existing templating languages." -category = "main" -optional = false -python-versions = ">=3.7" +optional = true +python-versions = ">=3.8" +files = [ + {file = "Mako-1.3.5-py3-none-any.whl", hash = "sha256:260f1dbc3a519453a9c856dedfe4beb4e50bd5a26d96386cb6c80856556bb91a"}, + {file = "Mako-1.3.5.tar.gz", hash = "sha256:48dbc20568c1d276a2698b36d968fa76161bf127194907ea6fc594fa81f943bc"}, +] [package.dependencies] -importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} MarkupSafe = ">=0.9.2" [package.extras] @@ -234,70 +460,234 @@ testing = ["pytest"] [[package]] name = "markupsafe" -version = "2.1.2" +version = "2.1.5" description = "Safely add untrusted strings to HTML/XML markup." -category = "main" -optional = false +optional = true python-versions = ">=3.7" +files = [ + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:a17a92de5231666cfbe003f0e4b9b3a7ae3afb1ec2845aadc2bacc93ff85febc"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:72b6be590cc35924b02c78ef34b467da4ba07e4e0f0454a2c5907f473fc50ce5"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e61659ba32cf2cf1481e575d0462554625196a1f2fc06a1c777d3f48e8865d46"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2174c595a0d73a3080ca3257b40096db99799265e1c27cc5a610743acd86d62f"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ae2ad8ae6ebee9d2d94b17fb62763125f3f374c25618198f40cbb8b525411900"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:075202fa5b72c86ad32dc7d0b56024ebdbcf2048c0ba09f1cde31bfdd57bcfff"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:598e3276b64aff0e7b3451b72e94fa3c238d452e7ddcd893c3ab324717456bad"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:fce659a462a1be54d2ffcacea5e3ba2d74daa74f30f5f143fe0c58636e355fdd"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win32.whl", hash = "sha256:d9fad5155d72433c921b782e58892377c44bd6252b5af2f67f16b194987338a4"}, + {file = "MarkupSafe-2.1.5-cp310-cp310-win_amd64.whl", hash = "sha256:bf50cd79a75d181c9181df03572cdce0fbb75cc353bc350712073108cba98de5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:629ddd2ca402ae6dbedfceeba9c46d5f7b2a61d9749597d4307f943ef198fc1f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:5b7b716f97b52c5a14bffdf688f971b2d5ef4029127f1ad7a513973cfd818df2"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ec585f69cec0aa07d945b20805be741395e28ac1627333b1c5b0105962ffced"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b91c037585eba9095565a3556f611e3cbfaa42ca1e865f7b8015fe5c7336d5a5"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7502934a33b54030eaf1194c21c692a534196063db72176b0c4028e140f8f32c"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:0e397ac966fdf721b2c528cf028494e86172b4feba51d65f81ffd65c63798f3f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:c061bb86a71b42465156a3ee7bd58c8c2ceacdbeb95d05a99893e08b8467359a"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:3a57fdd7ce31c7ff06cdfbf31dafa96cc533c21e443d57f5b1ecc6cdc668ec7f"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win32.whl", hash = "sha256:397081c1a0bfb5124355710fe79478cdbeb39626492b15d399526ae53422b906"}, + {file = "MarkupSafe-2.1.5-cp311-cp311-win_amd64.whl", hash = "sha256:2b7c57a4dfc4f16f7142221afe5ba4e093e09e728ca65c51f5620c9aaeb9a617"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_universal2.whl", hash = "sha256:8dec4936e9c3100156f8a2dc89c4b88d5c435175ff03413b443469c7c8c5f4d1"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3c6b973f22eb18a789b1460b4b91bf04ae3f0c4234a0a6aa6b0a92f6f7b951d4"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ac07bad82163452a6884fe8fa0963fb98c2346ba78d779ec06bd7a6262132aee"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f5dfb42c4604dddc8e4305050aa6deb084540643ed5804d7455b5df8fe16f5e5"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ea3d8a3d18833cf4304cd2fc9cbb1efe188ca9b5efef2bdac7adc20594a0e46b"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:d050b3361367a06d752db6ead6e7edeb0009be66bc3bae0ee9d97fb326badc2a"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bec0a414d016ac1a18862a519e54b2fd0fc8bbfd6890376898a6c0891dd82e9f"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:58c98fee265677f63a4385256a6d7683ab1832f3ddd1e66fe948d5880c21a169"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win32.whl", hash = "sha256:8590b4ae07a35970728874632fed7bd57b26b0102df2d2b233b6d9d82f6c62ad"}, + {file = "MarkupSafe-2.1.5-cp312-cp312-win_amd64.whl", hash = "sha256:823b65d8706e32ad2df51ed89496147a42a2a6e01c13cfb6ffb8b1e92bc910bb"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:c8b29db45f8fe46ad280a7294f5c3ec36dbac9491f2d1c17345be8e69cc5928f"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ec6a563cff360b50eed26f13adc43e61bc0c04d94b8be985e6fb24b81f6dcfdf"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a549b9c31bec33820e885335b451286e2969a2d9e24879f83fe904a5ce59d70a"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4f11aa001c540f62c6166c7726f71f7573b52c68c31f014c25cc7901deea0b52"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:7b2e5a267c855eea6b4283940daa6e88a285f5f2a67f2220203786dfa59b37e9"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:2d2d793e36e230fd32babe143b04cec8a8b3eb8a3122d2aceb4a371e6b09b8df"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:ce409136744f6521e39fd8e2a24c53fa18ad67aa5bc7c2cf83645cce5b5c4e50"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win32.whl", hash = "sha256:4096e9de5c6fdf43fb4f04c26fb114f61ef0bf2e5604b6ee3019d51b69e8c371"}, + {file = "MarkupSafe-2.1.5-cp37-cp37m-win_amd64.whl", hash = "sha256:4275d846e41ecefa46e2015117a9f491e57a71ddd59bbead77e904dc02b1bed2"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:656f7526c69fac7f600bd1f400991cc282b417d17539a1b228617081106feb4a"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:97cafb1f3cbcd3fd2b6fbfb99ae11cdb14deea0736fc2b0952ee177f2b813a46"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1f3fbcb7ef1f16e48246f704ab79d79da8a46891e2da03f8783a5b6fa41a9532"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fa9db3f79de01457b03d4f01b34cf91bc0048eb2c3846ff26f66687c2f6d16ab"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ffee1f21e5ef0d712f9033568f8344d5da8cc2869dbd08d87c84656e6a2d2f68"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:5dedb4db619ba5a2787a94d877bc8ffc0566f92a01c0ef214865e54ecc9ee5e0"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:30b600cf0a7ac9234b2638fbc0fb6158ba5bdcdf46aeb631ead21248b9affbc4"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:8dd717634f5a044f860435c1d8c16a270ddf0ef8588d4887037c5028b859b0c3"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win32.whl", hash = "sha256:daa4ee5a243f0f20d528d939d06670a298dd39b1ad5f8a72a4275124a7819eff"}, + {file = "MarkupSafe-2.1.5-cp38-cp38-win_amd64.whl", hash = "sha256:619bc166c4f2de5caa5a633b8b7326fbe98e0ccbfacabd87268a2b15ff73a029"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:7a68b554d356a91cce1236aa7682dc01df0edba8d043fd1ce607c49dd3c1edcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:db0b55e0f3cc0be60c1f19efdde9a637c32740486004f20d1cff53c3c0ece4d2"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e53af139f8579a6d5f7b76549125f0d94d7e630761a2111bc431fd820e163b8"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:17b950fccb810b3293638215058e432159d2b71005c74371d784862b7e4683f3"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4c31f53cdae6ecfa91a77820e8b151dba54ab528ba65dfd235c80b086d68a465"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:bff1b4290a66b490a2f4719358c0cdcd9bafb6b8f061e45c7a2460866bf50c2e"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:bc1667f8b83f48511b94671e0e441401371dfd0f0a795c7daa4a3cd1dde55bea"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:5049256f536511ee3f7e1b3f87d1d1209d327e818e6ae1365e8653d7e3abb6a6"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win32.whl", hash = "sha256:00e046b6dd71aa03a41079792f8473dc494d564611a8f89bbbd7cb93295ebdcf"}, + {file = "MarkupSafe-2.1.5-cp39-cp39-win_amd64.whl", hash = "sha256:fa173ec60341d6bb97a89f5ea19c85c5643c1e7dedebc22f5181eb73573142c5"}, + {file = "MarkupSafe-2.1.5.tar.gz", hash = "sha256:d283d37a890ba4c1ae73ffadf8046435c76e7bc2247bbb63c00bd1a709c6544b"}, +] [[package]] name = "mccabe" version = "0.7.0" description = "McCabe checker, plugin for flake8" -category = "dev" optional = false python-versions = ">=3.6" +files = [ + {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, + {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, +] [[package]] name = "mypy" -version = "0.950" +version = "1.10.1" description = "Optional static typing for Python" -category = "dev" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" +files = [ + {file = "mypy-1.10.1-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e36f229acfe250dc660790840916eb49726c928e8ce10fbdf90715090fe4ae02"}, + {file = "mypy-1.10.1-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:51a46974340baaa4145363b9e051812a2446cf583dfaeba124af966fa44593f7"}, + {file = "mypy-1.10.1-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:901c89c2d67bba57aaaca91ccdb659aa3a312de67f23b9dfb059727cce2e2e0a"}, + {file = "mypy-1.10.1-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:0cd62192a4a32b77ceb31272d9e74d23cd88c8060c34d1d3622db3267679a5d9"}, + {file = "mypy-1.10.1-cp310-cp310-win_amd64.whl", hash = "sha256:a2cbc68cb9e943ac0814c13e2452d2046c2f2b23ff0278e26599224cf164e78d"}, + {file = "mypy-1.10.1-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:bd6f629b67bb43dc0d9211ee98b96d8dabc97b1ad38b9b25f5e4c4d7569a0c6a"}, + {file = "mypy-1.10.1-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:a1bbb3a6f5ff319d2b9d40b4080d46cd639abe3516d5a62c070cf0114a457d84"}, + {file = "mypy-1.10.1-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b8edd4e9bbbc9d7b79502eb9592cab808585516ae1bcc1446eb9122656c6066f"}, + {file = "mypy-1.10.1-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:6166a88b15f1759f94a46fa474c7b1b05d134b1b61fca627dd7335454cc9aa6b"}, + {file = "mypy-1.10.1-cp311-cp311-win_amd64.whl", hash = "sha256:5bb9cd11c01c8606a9d0b83ffa91d0b236a0e91bc4126d9ba9ce62906ada868e"}, + {file = "mypy-1.10.1-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:d8681909f7b44d0b7b86e653ca152d6dff0eb5eb41694e163c6092124f8246d7"}, + {file = "mypy-1.10.1-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:378c03f53f10bbdd55ca94e46ec3ba255279706a6aacaecac52ad248f98205d3"}, + {file = "mypy-1.10.1-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6bacf8f3a3d7d849f40ca6caea5c055122efe70e81480c8328ad29c55c69e93e"}, + {file = "mypy-1.10.1-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:701b5f71413f1e9855566a34d6e9d12624e9e0a8818a5704d74d6b0402e66c04"}, + {file = "mypy-1.10.1-cp312-cp312-win_amd64.whl", hash = "sha256:3c4c2992f6ea46ff7fce0072642cfb62af7a2484efe69017ed8b095f7b39ef31"}, + {file = "mypy-1.10.1-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:604282c886497645ffb87b8f35a57ec773a4a2721161e709a4422c1636ddde5c"}, + {file = "mypy-1.10.1-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:37fd87cab83f09842653f08de066ee68f1182b9b5282e4634cdb4b407266bade"}, + {file = "mypy-1.10.1-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8addf6313777dbb92e9564c5d32ec122bf2c6c39d683ea64de6a1fd98b90fe37"}, + {file = "mypy-1.10.1-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:5cc3ca0a244eb9a5249c7c583ad9a7e881aa5d7b73c35652296ddcdb33b2b9c7"}, + {file = "mypy-1.10.1-cp38-cp38-win_amd64.whl", hash = "sha256:1b3a2ffce52cc4dbaeee4df762f20a2905aa171ef157b82192f2e2f368eec05d"}, + {file = "mypy-1.10.1-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:fe85ed6836165d52ae8b88f99527d3d1b2362e0cb90b005409b8bed90e9059b3"}, + {file = "mypy-1.10.1-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:c2ae450d60d7d020d67ab440c6e3fae375809988119817214440033f26ddf7bf"}, + {file = "mypy-1.10.1-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6be84c06e6abd72f960ba9a71561c14137a583093ffcf9bbfaf5e613d63fa531"}, + {file = "mypy-1.10.1-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:2189ff1e39db399f08205e22a797383613ce1cb0cb3b13d8bcf0170e45b96cc3"}, + {file = "mypy-1.10.1-cp39-cp39-win_amd64.whl", hash = "sha256:97a131ee36ac37ce9581f4220311247ab6cba896b4395b9c87af0675a13a755f"}, + {file = "mypy-1.10.1-py3-none-any.whl", hash = "sha256:71d8ac0b906354ebda8ef1673e5fde785936ac1f29ff6987c7483cfbd5a4235a"}, + {file = "mypy-1.10.1.tar.gz", hash = "sha256:1f8f492d7db9e3593ef42d4f115f04e556130f2819ad33ab84551403e97dd4c0"}, +] [package.dependencies] -mypy-extensions = ">=0.4.3" +mypy-extensions = ">=1.0.0" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} -typed-ast = {version = ">=1.4.0,<2", markers = "python_version < \"3.8\""} -typing-extensions = ">=3.10" +typing-extensions = ">=4.1.0" [package.extras] dmypy = ["psutil (>=4.0)"] -python2 = ["typed-ast (>=1.4.0,<2)"] +install-types = ["pip"] +mypyc = ["setuptools (>=50)"] reports = ["lxml"] [[package]] name = "mypy-extensions" version = "1.0.0" description = "Type system extensions for programs checked with the mypy type checker." -category = "dev" optional = false python-versions = ">=3.5" +files = [ + {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, + {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, +] [[package]] name = "numpy" -version = "1.21.6" -description = "NumPy is the fundamental package for array computing with Python." -category = "main" +version = "1.24.4" +description = "Fundamental package for array computing in Python" optional = false -python-versions = ">=3.7,<3.11" +python-versions = ">=3.8" +files = [ + {file = "numpy-1.24.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:c0bfb52d2169d58c1cdb8cc1f16989101639b34c7d3ce60ed70b19c63eba0b64"}, + {file = "numpy-1.24.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ed094d4f0c177b1b8e7aa9cba7d6ceed51c0e569a5318ac0ca9a090680a6a1b1"}, + {file = "numpy-1.24.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79fc682a374c4a8ed08b331bef9c5f582585d1048fa6d80bc6c35bc384eee9b4"}, + {file = "numpy-1.24.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ffe43c74893dbf38c2b0a1f5428760a1a9c98285553c89e12d70a96a7f3a4d6"}, + {file = "numpy-1.24.4-cp310-cp310-win32.whl", hash = "sha256:4c21decb6ea94057331e111a5bed9a79d335658c27ce2adb580fb4d54f2ad9bc"}, + {file = "numpy-1.24.4-cp310-cp310-win_amd64.whl", hash = "sha256:b4bea75e47d9586d31e892a7401f76e909712a0fd510f58f5337bea9572c571e"}, + {file = "numpy-1.24.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f136bab9c2cfd8da131132c2cf6cc27331dd6fae65f95f69dcd4ae3c3639c810"}, + {file = "numpy-1.24.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:e2926dac25b313635e4d6cf4dc4e51c8c0ebfed60b801c799ffc4c32bf3d1254"}, + {file = "numpy-1.24.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:222e40d0e2548690405b0b3c7b21d1169117391c2e82c378467ef9ab4c8f0da7"}, + {file = "numpy-1.24.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7215847ce88a85ce39baf9e89070cb860c98fdddacbaa6c0da3ffb31b3350bd5"}, + {file = "numpy-1.24.4-cp311-cp311-win32.whl", hash = "sha256:4979217d7de511a8d57f4b4b5b2b965f707768440c17cb70fbf254c4b225238d"}, + {file = "numpy-1.24.4-cp311-cp311-win_amd64.whl", hash = "sha256:b7b1fc9864d7d39e28f41d089bfd6353cb5f27ecd9905348c24187a768c79694"}, + {file = "numpy-1.24.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:1452241c290f3e2a312c137a9999cdbf63f78864d63c79039bda65ee86943f61"}, + {file = "numpy-1.24.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:04640dab83f7c6c85abf9cd729c5b65f1ebd0ccf9de90b270cd61935eef0197f"}, + {file = "numpy-1.24.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a5425b114831d1e77e4b5d812b69d11d962e104095a5b9c3b641a218abcc050e"}, + {file = "numpy-1.24.4-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd80e219fd4c71fc3699fc1dadac5dcf4fd882bfc6f7ec53d30fa197b8ee22dc"}, + {file = "numpy-1.24.4-cp38-cp38-win32.whl", hash = "sha256:4602244f345453db537be5314d3983dbf5834a9701b7723ec28923e2889e0bb2"}, + {file = "numpy-1.24.4-cp38-cp38-win_amd64.whl", hash = "sha256:692f2e0f55794943c5bfff12b3f56f99af76f902fc47487bdfe97856de51a706"}, + {file = "numpy-1.24.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2541312fbf09977f3b3ad449c4e5f4bb55d0dbf79226d7724211acc905049400"}, + {file = "numpy-1.24.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:9667575fb6d13c95f1b36aca12c5ee3356bf001b714fc354eb5465ce1609e62f"}, + {file = "numpy-1.24.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3a86ed21e4f87050382c7bc96571755193c4c1392490744ac73d660e8f564a9"}, + {file = "numpy-1.24.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d11efb4dbecbdf22508d55e48d9c8384db795e1b7b51ea735289ff96613ff74d"}, + {file = "numpy-1.24.4-cp39-cp39-win32.whl", hash = "sha256:6620c0acd41dbcb368610bb2f4d83145674040025e5536954782467100aa8835"}, + {file = "numpy-1.24.4-cp39-cp39-win_amd64.whl", hash = "sha256:befe2bf740fd8373cf56149a5c23a0f601e82869598d41f8e188a0e9869926f8"}, + {file = "numpy-1.24.4-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:31f13e25b4e304632a4619d0e0777662c2ffea99fcae2029556b17d8ff958aef"}, + {file = "numpy-1.24.4-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95f7ac6540e95bc440ad77f56e520da5bf877f87dca58bd095288dce8940532a"}, + {file = "numpy-1.24.4-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:e98f220aa76ca2a977fe435f5b04d7b3470c0a2e6312907b37ba6068f26787f2"}, + {file = "numpy-1.24.4.tar.gz", hash = "sha256:80f5e3a4e498641401868df4208b74581206afbee7cf7b8329daae82676d9463"}, +] [[package]] name = "numpy" -version = "1.24.3" +version = "1.26.4" description = "Fundamental package for array computing in Python" -category = "main" optional = false -python-versions = ">=3.8" +python-versions = ">=3.9" +files = [ + {file = "numpy-1.26.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:9ff0f4f29c51e2803569d7a51c2304de5554655a60c5d776e35b4a41413830d0"}, + {file = "numpy-1.26.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:2e4ee3380d6de9c9ec04745830fd9e2eccb3e6cf790d39d7b98ffd19b0dd754a"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d209d8969599b27ad20994c8e41936ee0964e6da07478d6c35016bc386b66ad4"}, + {file = "numpy-1.26.4-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ffa75af20b44f8dba823498024771d5ac50620e6915abac414251bd971b4529f"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:62b8e4b1e28009ef2846b4c7852046736bab361f7aeadeb6a5b89ebec3c7055a"}, + {file = "numpy-1.26.4-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a4abb4f9001ad2858e7ac189089c42178fcce737e4169dc61321660f1a96c7d2"}, + {file = "numpy-1.26.4-cp310-cp310-win32.whl", hash = "sha256:bfe25acf8b437eb2a8b2d49d443800a5f18508cd811fea3181723922a8a82b07"}, + {file = "numpy-1.26.4-cp310-cp310-win_amd64.whl", hash = "sha256:b97fe8060236edf3662adfc2c633f56a08ae30560c56310562cb4f95500022d5"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:4c66707fabe114439db9068ee468c26bbdf909cac0fb58686a42a24de1760c71"}, + {file = "numpy-1.26.4-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:edd8b5fe47dab091176d21bb6de568acdd906d1887a4584a15a9a96a1dca06ef"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7ab55401287bfec946ced39700c053796e7cc0e3acbef09993a9ad2adba6ca6e"}, + {file = "numpy-1.26.4-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:666dbfb6ec68962c033a450943ded891bed2d54e6755e35e5835d63f4f6931d5"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:96ff0b2ad353d8f990b63294c8986f1ec3cb19d749234014f4e7eb0112ceba5a"}, + {file = "numpy-1.26.4-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:60dedbb91afcbfdc9bc0b1f3f402804070deed7392c23eb7a7f07fa857868e8a"}, + {file = "numpy-1.26.4-cp311-cp311-win32.whl", hash = "sha256:1af303d6b2210eb850fcf03064d364652b7120803a0b872f5211f5234b399f20"}, + {file = "numpy-1.26.4-cp311-cp311-win_amd64.whl", hash = "sha256:cd25bcecc4974d09257ffcd1f098ee778f7834c3ad767fe5db785be9a4aa9cb2"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:b3ce300f3644fb06443ee2222c2201dd3a89ea6040541412b8fa189341847218"}, + {file = "numpy-1.26.4-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:03a8c78d01d9781b28a6989f6fa1bb2c4f2d51201cf99d3dd875df6fbd96b23b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9fad7dcb1aac3c7f0584a5a8133e3a43eeb2fe127f47e3632d43d677c66c102b"}, + {file = "numpy-1.26.4-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:675d61ffbfa78604709862923189bad94014bef562cc35cf61d3a07bba02a7ed"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:ab47dbe5cc8210f55aa58e4805fe224dac469cde56b9f731a4c098b91917159a"}, + {file = "numpy-1.26.4-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:1dda2e7b4ec9dd512f84935c5f126c8bd8b9f2fc001e9f54af255e8c5f16b0e0"}, + {file = "numpy-1.26.4-cp312-cp312-win32.whl", hash = "sha256:50193e430acfc1346175fcbdaa28ffec49947a06918b7b92130744e81e640110"}, + {file = "numpy-1.26.4-cp312-cp312-win_amd64.whl", hash = "sha256:08beddf13648eb95f8d867350f6a018a4be2e5ad54c8d8caed89ebca558b2818"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:7349ab0fa0c429c82442a27a9673fc802ffdb7c7775fad780226cb234965e53c"}, + {file = "numpy-1.26.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:52b8b60467cd7dd1e9ed082188b4e6bb35aa5cdd01777621a1658910745b90be"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5241e0a80d808d70546c697135da2c613f30e28251ff8307eb72ba696945764"}, + {file = "numpy-1.26.4-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f870204a840a60da0b12273ef34f7051e98c3b5961b61b0c2c1be6dfd64fbcd3"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:679b0076f67ecc0138fd2ede3a8fd196dddc2ad3254069bcb9faf9a79b1cebcd"}, + {file = "numpy-1.26.4-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:47711010ad8555514b434df65f7d7b076bb8261df1ca9bb78f53d3b2db02e95c"}, + {file = "numpy-1.26.4-cp39-cp39-win32.whl", hash = "sha256:a354325ee03388678242a4d7ebcd08b5c727033fcff3b2f536aea978e15ee9e6"}, + {file = "numpy-1.26.4-cp39-cp39-win_amd64.whl", hash = "sha256:3373d5d70a5fe74a2c1bb6d2cfd9609ecf686d47a2d7b1d37a8f3b6bf6003aea"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-macosx_10_9_x86_64.whl", hash = "sha256:afedb719a9dcfc7eaf2287b839d8198e06dcd4cb5d276a3df279231138e83d30"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:95a7476c59002f2f6c590b9b7b998306fba6a5aa646b1e22ddfeaf8f78c3a29c"}, + {file = "numpy-1.26.4-pp39-pypy39_pp73-win_amd64.whl", hash = "sha256:7e50d0a0cc3189f9cb0aeb3a6a6af18c16f59f004b866cd2be1c14b36134a4a0"}, + {file = "numpy-1.26.4.tar.gz", hash = "sha256:2a02aba9ed12e4ac4eb3ea9421c420301a0c6460d9830d74a9df87efa4912010"}, +] [[package]] name = "oauthlib" version = "3.2.2" description = "A generic, spec-compliant, thorough implementation of the OAuth request-signing logic" -category = "main" optional = false python-versions = ">=3.6" +files = [ + {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, + {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, +] [package.extras] rsa = ["cryptography (>=3.0.0)"] @@ -306,77 +696,133 @@ signedtoken = ["cryptography (>=3.0.0)", "pyjwt (>=2.0.0,<3)"] [[package]] name = "openpyxl" -version = "3.1.2" +version = "3.1.5" description = "A Python library to read/write Excel 2010 xlsx/xlsm files" -category = "main" optional = false -python-versions = ">=3.6" +python-versions = ">=3.8" +files = [ + {file = "openpyxl-3.1.5-py2.py3-none-any.whl", hash = "sha256:5282c12b107bffeef825f4617dc029afaf41d0ea60823bbb665ef3079dc79de2"}, + {file = "openpyxl-3.1.5.tar.gz", hash = "sha256:cf0e3cf56142039133628b5acffe8ef0c12bc902d2aadd3e0fe5878dc08d1050"}, +] [package.dependencies] et-xmlfile = "*" [[package]] name = "packaging" -version = "23.1" +version = "24.1" description = "Core utilities for Python packages" -category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +files = [ + {file = "packaging-24.1-py3-none-any.whl", hash = "sha256:5b8f2217dbdbd2f7f384c41c628544e6d52f2d0f53c6d0c3ea61aa5d1d7ff124"}, + {file = "packaging-24.1.tar.gz", hash = "sha256:026ed72c8ed3fcce5bf8950572258698927fd1dbda10a5e981cdf0ac37f4f002"}, +] [[package]] name = "pandas" -version = "1.3.5" +version = "2.0.3" description = "Powerful data structures for data analysis, time series, and statistics" -category = "main" optional = false -python-versions = ">=3.7.1" +python-versions = ">=3.8" +files = [ + {file = "pandas-2.0.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:e4c7c9f27a4185304c7caf96dc7d91bc60bc162221152de697c98eb0b2648dd8"}, + {file = "pandas-2.0.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:f167beed68918d62bffb6ec64f2e1d8a7d297a038f86d4aed056b9493fca407f"}, + {file = "pandas-2.0.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ce0c6f76a0f1ba361551f3e6dceaff06bde7514a374aa43e33b588ec10420183"}, + {file = "pandas-2.0.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ba619e410a21d8c387a1ea6e8a0e49bb42216474436245718d7f2e88a2f8d7c0"}, + {file = "pandas-2.0.3-cp310-cp310-win32.whl", hash = "sha256:3ef285093b4fe5058eefd756100a367f27029913760773c8bf1d2d8bebe5d210"}, + {file = "pandas-2.0.3-cp310-cp310-win_amd64.whl", hash = "sha256:9ee1a69328d5c36c98d8e74db06f4ad518a1840e8ccb94a4ba86920986bb617e"}, + {file = "pandas-2.0.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:b084b91d8d66ab19f5bb3256cbd5ea661848338301940e17f4492b2ce0801fe8"}, + {file = "pandas-2.0.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:37673e3bdf1551b95bf5d4ce372b37770f9529743d2498032439371fc7b7eb26"}, + {file = "pandas-2.0.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b9cb1e14fdb546396b7e1b923ffaeeac24e4cedd14266c3497216dd4448e4f2d"}, + {file = "pandas-2.0.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d9cd88488cceb7635aebb84809d087468eb33551097d600c6dad13602029c2df"}, + {file = "pandas-2.0.3-cp311-cp311-win32.whl", hash = "sha256:694888a81198786f0e164ee3a581df7d505024fbb1f15202fc7db88a71d84ebd"}, + {file = "pandas-2.0.3-cp311-cp311-win_amd64.whl", hash = "sha256:6a21ab5c89dcbd57f78d0ae16630b090eec626360085a4148693def5452d8a6b"}, + {file = "pandas-2.0.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9e4da0d45e7f34c069fe4d522359df7d23badf83abc1d1cef398895822d11061"}, + {file = "pandas-2.0.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:32fca2ee1b0d93dd71d979726b12b61faa06aeb93cf77468776287f41ff8fdc5"}, + {file = "pandas-2.0.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:258d3624b3ae734490e4d63c430256e716f488c4fcb7c8e9bde2d3aa46c29089"}, + {file = "pandas-2.0.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eae3dc34fa1aa7772dd3fc60270d13ced7346fcbcfee017d3132ec625e23bb0"}, + {file = "pandas-2.0.3-cp38-cp38-win32.whl", hash = "sha256:f3421a7afb1a43f7e38e82e844e2bca9a6d793d66c1a7f9f0ff39a795bbc5e02"}, + {file = "pandas-2.0.3-cp38-cp38-win_amd64.whl", hash = "sha256:69d7f3884c95da3a31ef82b7618af5710dba95bb885ffab339aad925c3e8ce78"}, + {file = "pandas-2.0.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5247fb1ba347c1261cbbf0fcfba4a3121fbb4029d95d9ef4dc45406620b25c8b"}, + {file = "pandas-2.0.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:81af086f4543c9d8bb128328b5d32e9986e0c84d3ee673a2ac6fb57fd14f755e"}, + {file = "pandas-2.0.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1994c789bf12a7c5098277fb43836ce090f1073858c10f9220998ac74f37c69b"}, + {file = "pandas-2.0.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5ec591c48e29226bcbb316e0c1e9423622bc7a4eaf1ef7c3c9fa1a3981f89641"}, + {file = "pandas-2.0.3-cp39-cp39-win32.whl", hash = "sha256:04dbdbaf2e4d46ca8da896e1805bc04eb85caa9a82e259e8eed00254d5e0c682"}, + {file = "pandas-2.0.3-cp39-cp39-win_amd64.whl", hash = "sha256:1168574b036cd8b93abc746171c9b4f1b83467438a5e45909fed645cf8692dbc"}, + {file = "pandas-2.0.3.tar.gz", hash = "sha256:c02f372a88e0d17f36d3093a644c73cfc1788e876a7c4bcb4020a77512e2043c"}, +] [package.dependencies] numpy = [ - {version = ">=1.17.3", markers = "platform_machine != \"aarch64\" and platform_machine != \"arm64\" and python_version < \"3.10\""}, - {version = ">=1.19.2", markers = "platform_machine == \"aarch64\" and python_version < \"3.10\""}, - {version = ">=1.20.0", markers = "platform_machine == \"arm64\" and python_version < \"3.10\""}, - {version = ">=1.21.0", markers = "python_version >= \"3.10\""}, + {version = ">=1.20.3", markers = "python_version < \"3.10\""}, + {version = ">=1.21.0", markers = "python_version >= \"3.10\" and python_version < \"3.11\""}, + {version = ">=1.23.2", markers = "python_version >= \"3.11\""}, ] -python-dateutil = ">=2.7.3" -pytz = ">=2017.3" +python-dateutil = ">=2.8.2" +pytz = ">=2020.1" +tzdata = ">=2022.1" [package.extras] -test = ["hypothesis (>=3.58)", "pytest (>=6.0)", "pytest-xdist"] +all = ["PyQt5 (>=5.15.1)", "SQLAlchemy (>=1.4.16)", "beautifulsoup4 (>=4.9.3)", "bottleneck (>=1.3.2)", "brotlipy (>=0.7.0)", "fastparquet (>=0.6.3)", "fsspec (>=2021.07.0)", "gcsfs (>=2021.07.0)", "html5lib (>=1.1)", "hypothesis (>=6.34.2)", "jinja2 (>=3.0.0)", "lxml (>=4.6.3)", "matplotlib (>=3.6.1)", "numba (>=0.53.1)", "numexpr (>=2.7.3)", "odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pandas-gbq (>=0.15.0)", "psycopg2 (>=2.8.6)", "pyarrow (>=7.0.0)", "pymysql (>=1.0.2)", "pyreadstat (>=1.1.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)", "python-snappy (>=0.6.0)", "pyxlsb (>=1.0.8)", "qtpy (>=2.2.0)", "s3fs (>=2021.08.0)", "scipy (>=1.7.1)", "tables (>=3.6.1)", "tabulate (>=0.8.9)", "xarray (>=0.21.0)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)", "zstandard (>=0.15.2)"] +aws = ["s3fs (>=2021.08.0)"] +clipboard = ["PyQt5 (>=5.15.1)", "qtpy (>=2.2.0)"] +compression = ["brotlipy (>=0.7.0)", "python-snappy (>=0.6.0)", "zstandard (>=0.15.2)"] +computation = ["scipy (>=1.7.1)", "xarray (>=0.21.0)"] +excel = ["odfpy (>=1.4.1)", "openpyxl (>=3.0.7)", "pyxlsb (>=1.0.8)", "xlrd (>=2.0.1)", "xlsxwriter (>=1.4.3)"] +feather = ["pyarrow (>=7.0.0)"] +fss = ["fsspec (>=2021.07.0)"] +gcp = ["gcsfs (>=2021.07.0)", "pandas-gbq (>=0.15.0)"] +hdf5 = ["tables (>=3.6.1)"] +html = ["beautifulsoup4 (>=4.9.3)", "html5lib (>=1.1)", "lxml (>=4.6.3)"] +mysql = ["SQLAlchemy (>=1.4.16)", "pymysql (>=1.0.2)"] +output-formatting = ["jinja2 (>=3.0.0)", "tabulate (>=0.8.9)"] +parquet = ["pyarrow (>=7.0.0)"] +performance = ["bottleneck (>=1.3.2)", "numba (>=0.53.1)", "numexpr (>=2.7.1)"] +plot = ["matplotlib (>=3.6.1)"] +postgresql = ["SQLAlchemy (>=1.4.16)", "psycopg2 (>=2.8.6)"] +spss = ["pyreadstat (>=1.1.2)"] +sql-other = ["SQLAlchemy (>=1.4.16)"] +test = ["hypothesis (>=6.34.2)", "pytest (>=7.3.2)", "pytest-asyncio (>=0.17.0)", "pytest-xdist (>=2.2.0)"] +xml = ["lxml (>=4.6.3)"] [[package]] name = "pathspec" -version = "0.11.1" +version = "0.12.1" description = "Utility library for gitignore style pattern matching of file paths." -category = "dev" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +files = [ + {file = "pathspec-0.12.1-py3-none-any.whl", hash = "sha256:a0d503e138a4c123b27490a4f7beda6a01c6f288df0e4a8b79c7eb0dc7b4cc08"}, + {file = "pathspec-0.12.1.tar.gz", hash = "sha256:a482d51503a1ab33b1c67a6c3813a26953dbdc71c31dacaef9a838c4e29f5712"}, +] [[package]] name = "platformdirs" -version = "3.5.0" -description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." -category = "dev" +version = "4.2.2" +description = "A small Python package for determining appropriate platform-specific dirs, e.g. a `user data dir`." optional = false -python-versions = ">=3.7" - -[package.dependencies] -typing-extensions = {version = ">=4.5", markers = "python_version < \"3.8\""} +python-versions = ">=3.8" +files = [ + {file = "platformdirs-4.2.2-py3-none-any.whl", hash = "sha256:2d7a1657e36a80ea911db832a8a6ece5ee53d8de21edd5cc5879af6530b1bfee"}, + {file = "platformdirs-4.2.2.tar.gz", hash = "sha256:38b7b51f512eed9e84a22788b4bce1de17c0adb134d6becb09836e37d8654cd3"}, +] [package.extras] -docs = ["furo (>=2023.3.27)", "proselint (>=0.13)", "sphinx (>=6.1.3)", "sphinx-autodoc-typehints (>=1.23,!=1.23.4)"] -test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.3.1)", "pytest-cov (>=4)", "pytest-mock (>=3.10)"] +docs = ["furo (>=2023.9.10)", "proselint (>=0.13)", "sphinx (>=7.2.6)", "sphinx-autodoc-typehints (>=1.25.2)"] +test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4.3)", "pytest-cov (>=4.1)", "pytest-mock (>=3.12)"] +type = ["mypy (>=1.8)"] [[package]] name = "pluggy" -version = "1.0.0" +version = "1.5.0" description = "plugin and hook calling mechanisms for python" -category = "dev" optional = false -python-versions = ">=3.6" - -[package.dependencies] -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} +python-versions = ">=3.8" +files = [ + {file = "pluggy-1.5.0-py3-none-any.whl", hash = "sha256:44e1ad92c8ca002de6377e165f3e0f1be63266ab4d554740532335b9d75ea669"}, + {file = "pluggy-1.5.0.tar.gz", hash = "sha256:2cffa88e94fdc978c4c574f15f9e59b7f4201d439195c3715ca9e2486f1d0cf1"}, +] [package.extras] dev = ["pre-commit", "tox"] @@ -384,82 +830,168 @@ testing = ["pytest", "pytest-benchmark"] [[package]] name = "pyarrow" -version = "12.0.0" +version = "16.1.0" description = "Python library for Apache Arrow" -category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +files = [ + {file = "pyarrow-16.1.0-cp310-cp310-macosx_10_15_x86_64.whl", hash = "sha256:17e23b9a65a70cc733d8b738baa6ad3722298fa0c81d88f63ff94bf25eaa77b9"}, + {file = "pyarrow-16.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:4740cc41e2ba5d641071d0ab5e9ef9b5e6e8c7611351a5cb7c1d175eaf43674a"}, + {file = "pyarrow-16.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:98100e0268d04e0eec47b73f20b39c45b4006f3c4233719c3848aa27a03c1aef"}, + {file = "pyarrow-16.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f68f409e7b283c085f2da014f9ef81e885d90dcd733bd648cfba3ef265961848"}, + {file = "pyarrow-16.1.0-cp310-cp310-manylinux_2_28_aarch64.whl", hash = "sha256:a8914cd176f448e09746037b0c6b3a9d7688cef451ec5735094055116857580c"}, + {file = "pyarrow-16.1.0-cp310-cp310-manylinux_2_28_x86_64.whl", hash = "sha256:48be160782c0556156d91adbdd5a4a7e719f8d407cb46ae3bb4eaee09b3111bd"}, + {file = "pyarrow-16.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:9cf389d444b0f41d9fe1444b70650fea31e9d52cfcb5f818b7888b91b586efff"}, + {file = "pyarrow-16.1.0-cp311-cp311-macosx_10_15_x86_64.whl", hash = "sha256:d0ebea336b535b37eee9eee31761813086d33ed06de9ab6fc6aaa0bace7b250c"}, + {file = "pyarrow-16.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e73cfc4a99e796727919c5541c65bb88b973377501e39b9842ea71401ca6c1c"}, + {file = "pyarrow-16.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bf9251264247ecfe93e5f5a0cd43b8ae834f1e61d1abca22da55b20c788417f6"}, + {file = "pyarrow-16.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ddf5aace92d520d3d2a20031d8b0ec27b4395cab9f74e07cc95edf42a5cc0147"}, + {file = "pyarrow-16.1.0-cp311-cp311-manylinux_2_28_aarch64.whl", hash = "sha256:25233642583bf658f629eb230b9bb79d9af4d9f9229890b3c878699c82f7d11e"}, + {file = "pyarrow-16.1.0-cp311-cp311-manylinux_2_28_x86_64.whl", hash = "sha256:a33a64576fddfbec0a44112eaf844c20853647ca833e9a647bfae0582b2ff94b"}, + {file = "pyarrow-16.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:185d121b50836379fe012753cf15c4ba9638bda9645183ab36246923875f8d1b"}, + {file = "pyarrow-16.1.0-cp312-cp312-macosx_10_15_x86_64.whl", hash = "sha256:2e51ca1d6ed7f2e9d5c3c83decf27b0d17bb207a7dea986e8dc3e24f80ff7d6f"}, + {file = "pyarrow-16.1.0-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:06ebccb6f8cb7357de85f60d5da50e83507954af617d7b05f48af1621d331c9a"}, + {file = "pyarrow-16.1.0-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b04707f1979815f5e49824ce52d1dceb46e2f12909a48a6a753fe7cafbc44a0c"}, + {file = "pyarrow-16.1.0-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0d32000693deff8dc5df444b032b5985a48592c0697cb6e3071a5d59888714e2"}, + {file = "pyarrow-16.1.0-cp312-cp312-manylinux_2_28_aarch64.whl", hash = "sha256:8785bb10d5d6fd5e15d718ee1d1f914fe768bf8b4d1e5e9bf253de8a26cb1628"}, + {file = "pyarrow-16.1.0-cp312-cp312-manylinux_2_28_x86_64.whl", hash = "sha256:e1369af39587b794873b8a307cc6623a3b1194e69399af0efd05bb202195a5a7"}, + {file = "pyarrow-16.1.0-cp312-cp312-win_amd64.whl", hash = "sha256:febde33305f1498f6df85e8020bca496d0e9ebf2093bab9e0f65e2b4ae2b3444"}, + {file = "pyarrow-16.1.0-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b5f5705ab977947a43ac83b52ade3b881eb6e95fcc02d76f501d549a210ba77f"}, + {file = "pyarrow-16.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:0d27bf89dfc2576f6206e9cd6cf7a107c9c06dc13d53bbc25b0bd4556f19cf5f"}, + {file = "pyarrow-16.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d07de3ee730647a600037bc1d7b7994067ed64d0eba797ac74b2bc77384f4c2"}, + {file = "pyarrow-16.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fbef391b63f708e103df99fbaa3acf9f671d77a183a07546ba2f2c297b361e83"}, + {file = "pyarrow-16.1.0-cp38-cp38-manylinux_2_28_aarch64.whl", hash = "sha256:19741c4dbbbc986d38856ee7ddfdd6a00fc3b0fc2d928795b95410d38bb97d15"}, + {file = "pyarrow-16.1.0-cp38-cp38-manylinux_2_28_x86_64.whl", hash = "sha256:f2c5fb249caa17b94e2b9278b36a05ce03d3180e6da0c4c3b3ce5b2788f30eed"}, + {file = "pyarrow-16.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:e6b6d3cd35fbb93b70ade1336022cc1147b95ec6af7d36906ca7fe432eb09710"}, + {file = "pyarrow-16.1.0-cp39-cp39-macosx_10_15_x86_64.whl", hash = "sha256:18da9b76a36a954665ccca8aa6bd9f46c1145f79c0bb8f4f244f5f8e799bca55"}, + {file = "pyarrow-16.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:99f7549779b6e434467d2aa43ab2b7224dd9e41bdde486020bae198978c9e05e"}, + {file = "pyarrow-16.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f07fdffe4fd5b15f5ec15c8b64584868d063bc22b86b46c9695624ca3505b7b4"}, + {file = "pyarrow-16.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:ddfe389a08ea374972bd4065d5f25d14e36b43ebc22fc75f7b951f24378bf0b5"}, + {file = "pyarrow-16.1.0-cp39-cp39-manylinux_2_28_aarch64.whl", hash = "sha256:3b20bd67c94b3a2ea0a749d2a5712fc845a69cb5d52e78e6449bbd295611f3aa"}, + {file = "pyarrow-16.1.0-cp39-cp39-manylinux_2_28_x86_64.whl", hash = "sha256:ba8ac20693c0bb0bf4b238751d4409e62852004a8cf031c73b0e0962b03e45e3"}, + {file = "pyarrow-16.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:31a1851751433d89a986616015841977e0a188662fcffd1a5677453f1df2de0a"}, + {file = "pyarrow-16.1.0.tar.gz", hash = "sha256:15fbb22ea96d11f0b5768504a3f961edab25eaf4197c341720c4a387f6c60315"}, +] [package.dependencies] numpy = ">=1.16.6" [[package]] name = "pylint" -version = "2.13.9" +version = "3.2.5" description = "python code static checker" -category = "dev" optional = false -python-versions = ">=3.6.2" +python-versions = ">=3.8.0" +files = [ + {file = "pylint-3.2.5-py3-none-any.whl", hash = "sha256:32cd6c042b5004b8e857d727708720c54a676d1e22917cf1a2df9b4d4868abd6"}, + {file = "pylint-3.2.5.tar.gz", hash = "sha256:e9b7171e242dcc6ebd0aaa7540481d1a72860748a0a7816b8fe6cf6c80a6fe7e"}, +] [package.dependencies] -astroid = ">=2.11.5,<=2.12.0-dev0" -colorama = {version = "*", markers = "sys_platform == \"win32\""} -dill = ">=0.2" -isort = ">=4.2.5,<6" +astroid = ">=3.2.2,<=3.3.0-dev0" +colorama = {version = ">=0.4.5", markers = "sys_platform == \"win32\""} +dill = [ + {version = ">=0.2", markers = "python_version < \"3.11\""}, + {version = ">=0.3.7", markers = "python_version >= \"3.12\""}, + {version = ">=0.3.6", markers = "python_version >= \"3.11\" and python_version < \"3.12\""}, +] +isort = ">=4.2.5,<5.13.0 || >5.13.0,<6" mccabe = ">=0.6,<0.8" platformdirs = ">=2.2.0" tomli = {version = ">=1.1.0", markers = "python_version < \"3.11\""} +tomlkit = ">=0.10.1" typing-extensions = {version = ">=3.10.0", markers = "python_version < \"3.10\""} [package.extras] -testutil = ["gitpython (>3)"] +spelling = ["pyenchant (>=3.2,<4.0)"] +testutils = ["gitpython (>3)"] [[package]] name = "pytest" -version = "7.3.1" +version = "7.4.4" description = "pytest: simple powerful testing with Python" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "pytest-7.4.4-py3-none-any.whl", hash = "sha256:b090cdf5ed60bf4c45261be03239c2c1c22df034fbffe691abe93cd80cea01d8"}, + {file = "pytest-7.4.4.tar.gz", hash = "sha256:2cf0005922c6ace4a3e2ec8b4080eb0d9753fdc93107415332f50ce9e7994280"}, +] [package.dependencies] colorama = {version = "*", markers = "sys_platform == \"win32\""} exceptiongroup = {version = ">=1.0.0rc8", markers = "python_version < \"3.11\""} -importlib-metadata = {version = ">=0.12", markers = "python_version < \"3.8\""} iniconfig = "*" packaging = "*" pluggy = ">=0.12,<2.0" tomli = {version = ">=1.0.0", markers = "python_version < \"3.11\""} [package.extras] -testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "xmlschema"] +testing = ["argcomplete", "attrs (>=19.2.0)", "hypothesis (>=3.56)", "mock", "nose", "pygments (>=2.7.2)", "requests", "setuptools", "xmlschema"] + +[[package]] +name = "pytest-dotenv" +version = "0.5.2" +description = "A py.test plugin that parses environment files before running tests" +optional = false +python-versions = "*" +files = [ + {file = "pytest-dotenv-0.5.2.tar.gz", hash = "sha256:2dc6c3ac6d8764c71c6d2804e902d0ff810fa19692e95fe138aefc9b1aa73732"}, + {file = "pytest_dotenv-0.5.2-py3-none-any.whl", hash = "sha256:40a2cece120a213898afaa5407673f6bd924b1fa7eafce6bda0e8abffe2f710f"}, +] + +[package.dependencies] +pytest = ">=5.0.0" +python-dotenv = ">=0.9.1" [[package]] name = "python-dateutil" -version = "2.8.2" +version = "2.9.0.post0" description = "Extensions to the standard Python datetime module" -category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" +files = [ + {file = "python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3"}, + {file = "python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427"}, +] [package.dependencies] six = ">=1.5" +[[package]] +name = "python-dotenv" +version = "1.0.1" +description = "Read key-value pairs from a .env file and set them as environment variables" +optional = false +python-versions = ">=3.8" +files = [ + {file = "python-dotenv-1.0.1.tar.gz", hash = "sha256:e324ee90a023d808f1959c46bcbc04446a10ced277783dc6ee09987c37ec10ca"}, + {file = "python_dotenv-1.0.1-py3-none-any.whl", hash = "sha256:f7b63ef50f1b690dddf550d03497b66d609393b40b564ed0d674909a68ebf16a"}, +] + +[package.extras] +cli = ["click (>=5.0)"] + [[package]] name = "pytz" -version = "2023.3" +version = "2024.1" description = "World timezone definitions, modern and historical" -category = "main" optional = false python-versions = "*" +files = [ + {file = "pytz-2024.1-py2.py3-none-any.whl", hash = "sha256:328171f4e3623139da4983451950b28e95ac706e13f3f2630a879749e7a8b319"}, + {file = "pytz-2024.1.tar.gz", hash = "sha256:2a29735ea9c18baf14b448846bde5a48030ed267578472d8955cd0e7443a9812"}, +] [[package]] name = "requests" -version = "2.30.0" +version = "2.32.3" description = "Python HTTP for Humans." -category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +files = [ + {file = "requests-2.32.3-py3-none-any.whl", hash = "sha256:70761cfe03c773ceb22aa2f671b4757976145175cdfca038c02654d061d6dcc6"}, + {file = "requests-2.32.3.tar.gz", hash = "sha256:55365417734eb18255590a9ff9eb97e9e1da868d4ccd6402399eaf68af20a760"}, +] [package.dependencies] certifi = ">=2017.4.17" @@ -471,67 +1003,113 @@ urllib3 = ">=1.21.1,<3" socks = ["PySocks (>=1.5.6,!=1.5.7)"] use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] -[[package]] -name = "setuptools" -version = "67.7.2" -description = "Easily download, build, install, upgrade, and uninstall Python packages" -category = "dev" -optional = false -python-versions = ">=3.7" - -[package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "pygments-github-lexers (==0.0.5)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-favicon", "sphinx-hoverxref (<2)", "sphinx-inline-tabs", "sphinx-lint", "sphinx-notfound-page (==0.8.3)", "sphinx-reredirects", "sphinxcontrib-towncrier"] -testing = ["build[virtualenv]", "filelock (>=3.4.0)", "flake8 (<5)", "flake8-2020", "ini2toml[lite] (>=0.9)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pip (>=19.1)", "pip-run (>=8.8)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)", "pytest-perf", "pytest-timeout", "pytest-xdist", "tomli-w (>=1.0.0)", "virtualenv (>=13.0.0)", "wheel"] -testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs (>=2.2)", "jaraco.path (>=3.2.0)", "pytest", "pytest-enabler", "pytest-xdist", "tomli", "virtualenv (>=13.0.0)", "wheel"] - [[package]] name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" -category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" +files = [ + {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, + {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, +] [[package]] name = "sqlalchemy" -version = "1.4.48" +version = "2.0.31" description = "Database Abstraction Library" -category = "main" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" +optional = true +python-versions = ">=3.7" +files = [ + {file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:f2a213c1b699d3f5768a7272de720387ae0122f1becf0901ed6eaa1abd1baf6c"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:9fea3d0884e82d1e33226935dac990b967bef21315cbcc894605db3441347443"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f3ad7f221d8a69d32d197e5968d798217a4feebe30144986af71ada8c548e9fa"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f2bee229715b6366f86a95d497c347c22ddffa2c7c96143b59a2aa5cc9eebbc"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-musllinux_1_2_aarch64.whl", hash = "sha256:cd5b94d4819c0c89280b7c6109c7b788a576084bf0a480ae17c227b0bc41e109"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-musllinux_1_2_x86_64.whl", hash = "sha256:750900a471d39a7eeba57580b11983030517a1f512c2cb287d5ad0fcf3aebd58"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-win32.whl", hash = "sha256:7bd112be780928c7f493c1a192cd8c5fc2a2a7b52b790bc5a84203fb4381c6be"}, + {file = "SQLAlchemy-2.0.31-cp310-cp310-win_amd64.whl", hash = "sha256:5a48ac4d359f058474fadc2115f78a5cdac9988d4f99eae44917f36aa1476327"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:f68470edd70c3ac3b6cd5c2a22a8daf18415203ca1b036aaeb9b0fb6f54e8298"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:2e2c38c2a4c5c634fe6c3c58a789712719fa1bf9b9d6ff5ebfce9a9e5b89c1ca"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bd15026f77420eb2b324dcb93551ad9c5f22fab2c150c286ef1dc1160f110203"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2196208432deebdfe3b22185d46b08f00ac9d7b01284e168c212919891289396"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-musllinux_1_2_aarch64.whl", hash = "sha256:352b2770097f41bff6029b280c0e03b217c2dcaddc40726f8f53ed58d8a85da4"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-musllinux_1_2_x86_64.whl", hash = "sha256:56d51ae825d20d604583f82c9527d285e9e6d14f9a5516463d9705dab20c3740"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-win32.whl", hash = "sha256:6e2622844551945db81c26a02f27d94145b561f9d4b0c39ce7bfd2fda5776dac"}, + {file = "SQLAlchemy-2.0.31-cp311-cp311-win_amd64.whl", hash = "sha256:ccaf1b0c90435b6e430f5dd30a5aede4764942a695552eb3a4ab74ed63c5b8d3"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:3b74570d99126992d4b0f91fb87c586a574a5872651185de8297c6f90055ae42"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:6f77c4f042ad493cb8595e2f503c7a4fe44cd7bd59c7582fd6d78d7e7b8ec52c"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cd1591329333daf94467e699e11015d9c944f44c94d2091f4ac493ced0119449"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:74afabeeff415e35525bf7a4ecdab015f00e06456166a2eba7590e49f8db940e"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-musllinux_1_2_aarch64.whl", hash = "sha256:b9c01990d9015df2c6f818aa8f4297d42ee71c9502026bb074e713d496e26b67"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-musllinux_1_2_x86_64.whl", hash = "sha256:66f63278db425838b3c2b1c596654b31939427016ba030e951b292e32b99553e"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-win32.whl", hash = "sha256:0b0f658414ee4e4b8cbcd4a9bb0fd743c5eeb81fc858ca517217a8013d282c96"}, + {file = "SQLAlchemy-2.0.31-cp312-cp312-win_amd64.whl", hash = "sha256:fa4b1af3e619b5b0b435e333f3967612db06351217c58bfb50cee5f003db2a5a"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:f43e93057cf52a227eda401251c72b6fbe4756f35fa6bfebb5d73b86881e59b0"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d337bf94052856d1b330d5fcad44582a30c532a2463776e1651bd3294ee7e58b"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c06fb43a51ccdff3b4006aafee9fcf15f63f23c580675f7734245ceb6b6a9e05"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-musllinux_1_2_aarch64.whl", hash = "sha256:b6e22630e89f0e8c12332b2b4c282cb01cf4da0d26795b7eae16702a608e7ca1"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-musllinux_1_2_x86_64.whl", hash = "sha256:79a40771363c5e9f3a77f0e28b3302801db08040928146e6808b5b7a40749c88"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-win32.whl", hash = "sha256:501ff052229cb79dd4c49c402f6cb03b5a40ae4771efc8bb2bfac9f6c3d3508f"}, + {file = "SQLAlchemy-2.0.31-cp37-cp37m-win_amd64.whl", hash = "sha256:597fec37c382a5442ffd471f66ce12d07d91b281fd474289356b1a0041bdf31d"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:dc6d69f8829712a4fd799d2ac8d79bdeff651c2301b081fd5d3fe697bd5b4ab9"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:23b9fbb2f5dd9e630db70fbe47d963c7779e9c81830869bd7d137c2dc1ad05fb"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2a21c97efcbb9f255d5c12a96ae14da873233597dfd00a3a0c4ce5b3e5e79704"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:26a6a9837589c42b16693cf7bf836f5d42218f44d198f9343dd71d3164ceeeac"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-musllinux_1_2_aarch64.whl", hash = "sha256:dc251477eae03c20fae8db9c1c23ea2ebc47331bcd73927cdcaecd02af98d3c3"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-musllinux_1_2_x86_64.whl", hash = "sha256:2fd17e3bb8058359fa61248c52c7b09a97cf3c820e54207a50af529876451808"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-win32.whl", hash = "sha256:c76c81c52e1e08f12f4b6a07af2b96b9b15ea67ccdd40ae17019f1c373faa227"}, + {file = "SQLAlchemy-2.0.31-cp38-cp38-win_amd64.whl", hash = "sha256:4b600e9a212ed59355813becbcf282cfda5c93678e15c25a0ef896b354423238"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:5b6cf796d9fcc9b37011d3f9936189b3c8074a02a4ed0c0fbbc126772c31a6d4"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:78fe11dbe37d92667c2c6e74379f75746dc947ee505555a0197cfba9a6d4f1a4"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fc47dc6185a83c8100b37acda27658fe4dbd33b7d5e7324111f6521008ab4fe"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8a41514c1a779e2aa9a19f67aaadeb5cbddf0b2b508843fcd7bafdf4c6864005"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-musllinux_1_2_aarch64.whl", hash = "sha256:afb6dde6c11ea4525318e279cd93c8734b795ac8bb5dda0eedd9ebaca7fa23f1"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-musllinux_1_2_x86_64.whl", hash = "sha256:3f9faef422cfbb8fd53716cd14ba95e2ef655400235c3dfad1b5f467ba179c8c"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-win32.whl", hash = "sha256:fc6b14e8602f59c6ba893980bea96571dd0ed83d8ebb9c4479d9ed5425d562e9"}, + {file = "SQLAlchemy-2.0.31-cp39-cp39-win_amd64.whl", hash = "sha256:3cb8a66b167b033ec72c3812ffc8441d4e9f5f78f5e31e54dcd4c90a4ca5bebc"}, + {file = "SQLAlchemy-2.0.31-py3-none-any.whl", hash = "sha256:69f3e3c08867a8e4856e92d7afb618b95cdee18e0bc1647b77599722c9a28911"}, + {file = "SQLAlchemy-2.0.31.tar.gz", hash = "sha256:b607489dd4a54de56984a0c7656247504bd5523d9d0ba799aef59d4add009484"}, +] [package.dependencies] -greenlet = {version = "!=0.4.17", markers = "python_version >= \"3\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} -importlib-metadata = {version = "*", markers = "python_version < \"3.8\""} +greenlet = {version = "!=0.4.17", markers = "python_version < \"3.13\" and (platform_machine == \"aarch64\" or platform_machine == \"ppc64le\" or platform_machine == \"x86_64\" or platform_machine == \"amd64\" or platform_machine == \"AMD64\" or platform_machine == \"win32\" or platform_machine == \"WIN32\")"} +typing-extensions = ">=4.6.0" [package.extras] -aiomysql = ["aiomysql", "greenlet (!=0.4.17)"] +aiomysql = ["aiomysql (>=0.2.0)", "greenlet (!=0.4.17)"] +aioodbc = ["aioodbc", "greenlet (!=0.4.17)"] aiosqlite = ["aiosqlite", "greenlet (!=0.4.17)", "typing_extensions (!=3.10.0.1)"] asyncio = ["greenlet (!=0.4.17)"] -asyncmy = ["asyncmy (>=0.2.3,!=0.2.4)", "greenlet (!=0.4.17)"] -mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2)"] +asyncmy = ["asyncmy (>=0.2.3,!=0.2.4,!=0.2.6)", "greenlet (!=0.4.17)"] +mariadb-connector = ["mariadb (>=1.0.1,!=1.1.2,!=1.1.5)"] mssql = ["pyodbc"] mssql-pymssql = ["pymssql"] mssql-pyodbc = ["pyodbc"] -mypy = ["mypy (>=0.910)", "sqlalchemy2-stubs"] -mysql = ["mysqlclient (>=1.4.0)", "mysqlclient (>=1.4.0,<2)"] +mypy = ["mypy (>=0.910)"] +mysql = ["mysqlclient (>=1.4.0)"] mysql-connector = ["mysql-connector-python"] -oracle = ["cx_oracle (>=7)", "cx_oracle (>=7,<8)"] +oracle = ["cx_oracle (>=8)"] +oracle-oracledb = ["oracledb (>=1.0.1)"] postgresql = ["psycopg2 (>=2.7)"] postgresql-asyncpg = ["asyncpg", "greenlet (!=0.4.17)"] -postgresql-pg8000 = ["pg8000 (>=1.16.6,!=1.29.0)"] +postgresql-pg8000 = ["pg8000 (>=1.29.1)"] +postgresql-psycopg = ["psycopg (>=3.0.7)"] postgresql-psycopg2binary = ["psycopg2-binary"] postgresql-psycopg2cffi = ["psycopg2cffi"] -pymysql = ["pymysql", "pymysql (<1)"] +postgresql-psycopgbinary = ["psycopg[binary] (>=3.0.7)"] +pymysql = ["pymysql"] sqlcipher = ["sqlcipher3_binary"] [[package]] name = "thrift" -version = "0.16.0" +version = "0.20.0" description = "Python bindings for the Apache Thrift RPC system" -category = "main" optional = false python-versions = "*" +files = [ + {file = "thrift-0.20.0.tar.gz", hash = "sha256:4dd662eadf6b8aebe8a41729527bd69adf6ceaa2a8681cbef64d1273b3e8feba"}, +] [package.dependencies] six = ">=1.7.2" @@ -545,764 +1123,83 @@ twisted = ["twisted"] name = "tomli" version = "2.0.1" description = "A lil' TOML parser" -category = "dev" optional = false python-versions = ">=3.7" +files = [ + {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, + {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, +] [[package]] -name = "typed-ast" -version = "1.5.4" -description = "a fork of Python 2 and 3 ast modules with type comment support" -category = "dev" +name = "tomlkit" +version = "0.12.5" +description = "Style preserving TOML library" optional = false -python-versions = ">=3.6" +python-versions = ">=3.7" +files = [ + {file = "tomlkit-0.12.5-py3-none-any.whl", hash = "sha256:af914f5a9c59ed9d0762c7b64d3b5d5df007448eb9cd2edc8a46b1eafead172f"}, + {file = "tomlkit-0.12.5.tar.gz", hash = "sha256:eef34fba39834d4d6b73c9ba7f3e4d1c417a4e56f89a7e96e090dd0d24b8fb3c"}, +] [[package]] name = "typing-extensions" -version = "4.5.0" -description = "Backported and Experimental Type Hints for Python 3.7+" -category = "main" +version = "4.12.2" +description = "Backported and Experimental Type Hints for Python 3.8+" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +files = [ + {file = "typing_extensions-4.12.2-py3-none-any.whl", hash = "sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d"}, + {file = "typing_extensions-4.12.2.tar.gz", hash = "sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8"}, +] + +[[package]] +name = "tzdata" +version = "2024.1" +description = "Provider of IANA time zone data" +optional = false +python-versions = ">=2" +files = [ + {file = "tzdata-2024.1-py2.py3-none-any.whl", hash = "sha256:9068bc196136463f5245e51efda838afa15aaeca9903f49050dfa2679db4d252"}, + {file = "tzdata-2024.1.tar.gz", hash = "sha256:2674120f8d891909751c38abcdfd386ac0a5a1127954fbc332af6b5ceae07efd"}, +] [[package]] name = "urllib3" -version = "2.0.2" +version = "2.2.2" description = "HTTP library with thread-safe connection pooling, file post, and more." -category = "main" optional = false -python-versions = ">=3.7" +python-versions = ">=3.8" +files = [ + {file = "urllib3-2.2.2-py3-none-any.whl", hash = "sha256:a448b2f64d686155468037e1ace9f2d2199776e17f0a46610480d311f73e3472"}, + {file = "urllib3-2.2.2.tar.gz", hash = "sha256:dd505485549a7a552833da5e6063639d0d177c04f23bc3864e41e5dc5f612168"}, +] [package.extras] brotli = ["brotli (>=1.0.9)", "brotlicffi (>=0.8.0)"] -secure = ["certifi", "cryptography (>=1.9)", "idna (>=2.0.0)", "pyopenssl (>=17.1.0)", "urllib3-secure-extra"] +h2 = ["h2 (>=4,<5)"] socks = ["pysocks (>=1.5.6,!=1.5.7,<2.0)"] zstd = ["zstandard (>=0.18.0)"] -[[package]] -name = "wrapt" -version = "1.15.0" -description = "Module for decorators, wrappers and monkey patching." -category = "dev" -optional = false -python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,>=2.7" - [[package]] name = "zipp" -version = "3.15.0" +version = "3.19.2" description = "Backport of pathlib-compatible object wrapper for zip files" -category = "main" -optional = false -python-versions = ">=3.7" +optional = true +python-versions = ">=3.8" +files = [ + {file = "zipp-3.19.2-py3-none-any.whl", hash = "sha256:f091755f667055f2d02b32c53771a7a6c8b47e1fdbc4b72a8b9072b3eef8015c"}, + {file = "zipp-3.19.2.tar.gz", hash = "sha256:bf1dcf6450f873a13e952a29504887c89e6de7506209e5b1bcc3460135d4de19"}, +] [package.extras] -docs = ["furo", "jaraco.packaging (>=9)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] -testing = ["big-O", "flake8 (<5)", "jaraco.functools", "jaraco.itertools", "more-itertools", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=1.3)", "pytest-flake8", "pytest-mypy (>=0.9.1)"] +doc = ["furo", "jaraco.packaging (>=9.3)", "jaraco.tidelift (>=1.4)", "rst.linker (>=1.9)", "sphinx (>=3.5)", "sphinx-lint"] +test = ["big-O", "importlib-resources", "jaraco.functools", "jaraco.itertools", "jaraco.test", "more-itertools", "pytest (>=6,!=8.1.*)", "pytest-checkdocs (>=2.4)", "pytest-cov", "pytest-enabler (>=2.2)", "pytest-ignore-flaky", "pytest-mypy", "pytest-ruff (>=0.2.1)"] + +[extras] +alembic = ["alembic", "sqlalchemy"] +sqlalchemy = ["sqlalchemy"] [metadata] -lock-version = "1.1" -python-versions = "^3.7.1" -content-hash = "8432ddba9b066e5b1c34ca44918443f1f7566d95e4f0c0a9b630dd95b95bb71e" - -[metadata.files] -alembic = [ - {file = "alembic-1.10.4-py3-none-any.whl", hash = "sha256:43942c3d4bf2620c466b91c0f4fca136fe51ae972394a0cc8b90810d664e4f5c"}, - {file = "alembic-1.10.4.tar.gz", hash = "sha256:295b54bbb92c4008ab6a7dcd1e227e668416d6f84b98b3c4446a2bc6214a556b"}, -] -astroid = [ - {file = "astroid-2.11.7-py3-none-any.whl", hash = "sha256:86b0a340a512c65abf4368b80252754cda17c02cdbbd3f587dddf98112233e7b"}, - {file = "astroid-2.11.7.tar.gz", hash = "sha256:bb24615c77f4837c707669d16907331374ae8a964650a66999da3f5ca68dc946"}, -] -black = [ - {file = "black-22.12.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9eedd20838bd5d75b80c9f5487dbcb06836a43833a37846cf1d8c1cc01cef59d"}, - {file = "black-22.12.0-cp310-cp310-win_amd64.whl", hash = "sha256:159a46a4947f73387b4d83e87ea006dbb2337eab6c879620a3ba52699b1f4351"}, - {file = "black-22.12.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d30b212bffeb1e252b31dd269dfae69dd17e06d92b87ad26e23890f3efea366f"}, - {file = "black-22.12.0-cp311-cp311-win_amd64.whl", hash = "sha256:7412e75863aa5c5411886804678b7d083c7c28421210180d67dfd8cf1221e1f4"}, - {file = "black-22.12.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c116eed0efb9ff870ded8b62fe9f28dd61ef6e9ddd28d83d7d264a38417dcee2"}, - {file = "black-22.12.0-cp37-cp37m-win_amd64.whl", hash = "sha256:1f58cbe16dfe8c12b7434e50ff889fa479072096d79f0a7f25e4ab8e94cd8350"}, - {file = "black-22.12.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:77d86c9f3db9b1bf6761244bc0b3572a546f5fe37917a044e02f3166d5aafa7d"}, - {file = "black-22.12.0-cp38-cp38-win_amd64.whl", hash = "sha256:82d9fe8fee3401e02e79767016b4907820a7dc28d70d137eb397b92ef3cc5bfc"}, - {file = "black-22.12.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:101c69b23df9b44247bd88e1d7e90154336ac4992502d4197bdac35dd7ee3320"}, - {file = "black-22.12.0-cp39-cp39-win_amd64.whl", hash = "sha256:559c7a1ba9a006226f09e4916060982fd27334ae1998e7a38b3f33a37f7a2148"}, - {file = "black-22.12.0-py3-none-any.whl", hash = "sha256:436cc9167dd28040ad90d3b404aec22cedf24a6e4d7de221bec2730ec0c97bcf"}, - {file = "black-22.12.0.tar.gz", hash = "sha256:229351e5a18ca30f447bf724d007f890f97e13af070bb6ad4c0a441cd7596a2f"}, -] -certifi = [ - {file = "certifi-2023.5.7-py3-none-any.whl", hash = "sha256:c6c2e98f5c7869efca1f8916fed228dd91539f9f1b444c314c06eef02980c716"}, - {file = "certifi-2023.5.7.tar.gz", hash = "sha256:0f0d56dc5a6ad56fd4ba36484d6cc34451e1c6548c61daad8c320169f91eddc7"}, -] -charset-normalizer = [ - {file = "charset-normalizer-3.1.0.tar.gz", hash = "sha256:34e0a2f9c370eb95597aae63bf85eb5e96826d81e3dcf88b8886012906f509b5"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:e0ac8959c929593fee38da1c2b64ee9778733cdf03c482c9ff1d508b6b593b2b"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:d7fc3fca01da18fbabe4625d64bb612b533533ed10045a2ac3dd194bfa656b60"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:04eefcee095f58eaabe6dc3cc2262f3bcd776d2c67005880894f447b3f2cb9c1"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:20064ead0717cf9a73a6d1e779b23d149b53daf971169289ed2ed43a71e8d3b0"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1435ae15108b1cb6fffbcea2af3d468683b7afed0169ad718451f8db5d1aff6f"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c84132a54c750fda57729d1e2599bb598f5fa0344085dbde5003ba429a4798c0"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:75f2568b4189dda1c567339b48cba4ac7384accb9c2a7ed655cd86b04055c795"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:11d3bcb7be35e7b1bba2c23beedac81ee893ac9871d0ba79effc7fc01167db6c"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:891cf9b48776b5c61c700b55a598621fdb7b1e301a550365571e9624f270c203"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:5f008525e02908b20e04707a4f704cd286d94718f48bb33edddc7d7b584dddc1"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:b06f0d3bf045158d2fb8837c5785fe9ff9b8c93358be64461a1089f5da983137"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:49919f8400b5e49e961f320c735388ee686a62327e773fa5b3ce6721f7e785ce"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:22908891a380d50738e1f978667536f6c6b526a2064156203d418f4856d6e86a"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-win32.whl", hash = "sha256:12d1a39aa6b8c6f6248bb54550efcc1c38ce0d8096a146638fd4738e42284448"}, - {file = "charset_normalizer-3.1.0-cp310-cp310-win_amd64.whl", hash = "sha256:65ed923f84a6844de5fd29726b888e58c62820e0769b76565480e1fdc3d062f8"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:9a3267620866c9d17b959a84dd0bd2d45719b817245e49371ead79ed4f710d19"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:6734e606355834f13445b6adc38b53c0fd45f1a56a9ba06c2058f86893ae8017"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:f8303414c7b03f794347ad062c0516cee0e15f7a612abd0ce1e25caf6ceb47df"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:aaf53a6cebad0eae578f062c7d462155eada9c172bd8c4d250b8c1d8eb7f916a"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:3dc5b6a8ecfdc5748a7e429782598e4f17ef378e3e272eeb1340ea57c9109f41"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:e1b25e3ad6c909f398df8921780d6a3d120d8c09466720226fc621605b6f92b1"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0ca564606d2caafb0abe6d1b5311c2649e8071eb241b2d64e75a0d0065107e62"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:b82fab78e0b1329e183a65260581de4375f619167478dddab510c6c6fb04d9b6"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bd7163182133c0c7701b25e604cf1611c0d87712e56e88e7ee5d72deab3e76b5"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:11d117e6c63e8f495412d37e7dc2e2fff09c34b2d09dbe2bee3c6229577818be"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:cf6511efa4801b9b38dc5546d7547d5b5c6ef4b081c60b23e4d941d0eba9cbeb"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:abc1185d79f47c0a7aaf7e2412a0eb2c03b724581139193d2d82b3ad8cbb00ac"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:cb7b2ab0188829593b9de646545175547a70d9a6e2b63bf2cd87a0a391599324"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-win32.whl", hash = "sha256:c36bcbc0d5174a80d6cccf43a0ecaca44e81d25be4b7f90f0ed7bcfbb5a00909"}, - {file = "charset_normalizer-3.1.0-cp311-cp311-win_amd64.whl", hash = "sha256:cca4def576f47a09a943666b8f829606bcb17e2bc2d5911a46c8f8da45f56755"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:0c95f12b74681e9ae127728f7e5409cbbef9cd914d5896ef238cc779b8152373"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fca62a8301b605b954ad2e9c3666f9d97f63872aa4efcae5492baca2056b74ab"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:ac0aa6cd53ab9a31d397f8303f92c42f534693528fafbdb997c82bae6e477ad9"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c3af8e0f07399d3176b179f2e2634c3ce9c1301379a6b8c9c9aeecd481da494f"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a5fc78f9e3f501a1614a98f7c54d3969f3ad9bba8ba3d9b438c3bc5d047dd28"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:628c985afb2c7d27a4800bfb609e03985aaecb42f955049957814e0491d4006d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:74db0052d985cf37fa111828d0dd230776ac99c740e1a758ad99094be4f1803d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:1e8fcdd8f672a1c4fc8d0bd3a2b576b152d2a349782d1eb0f6b8e52e9954731d"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:04afa6387e2b282cf78ff3dbce20f0cc071c12dc8f685bd40960cc68644cfea6"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:dd5653e67b149503c68c4018bf07e42eeed6b4e956b24c00ccdf93ac79cdff84"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d2686f91611f9e17f4548dbf050e75b079bbc2a82be565832bc8ea9047b61c8c"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-win32.whl", hash = "sha256:4155b51ae05ed47199dc5b2a4e62abccb274cee6b01da5b895099b61b1982974"}, - {file = "charset_normalizer-3.1.0-cp37-cp37m-win_amd64.whl", hash = "sha256:322102cdf1ab682ecc7d9b1c5eed4ec59657a65e1c146a0da342b78f4112db23"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:e633940f28c1e913615fd624fcdd72fdba807bf53ea6925d6a588e84e1151531"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:3a06f32c9634a8705f4ca9946d667609f52cf130d5548881401f1eb2c39b1e2c"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7381c66e0561c5757ffe616af869b916c8b4e42b367ab29fedc98481d1e74e14"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3573d376454d956553c356df45bb824262c397c6e26ce43e8203c4c540ee0acb"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e89df2958e5159b811af9ff0f92614dabf4ff617c03a4c1c6ff53bf1c399e0e1"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:78cacd03e79d009d95635e7d6ff12c21eb89b894c354bd2b2ed0b4763373693b"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:de5695a6f1d8340b12a5d6d4484290ee74d61e467c39ff03b39e30df62cf83a0"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:1c60b9c202d00052183c9be85e5eaf18a4ada0a47d188a83c8f5c5b23252f649"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f645caaf0008bacf349875a974220f1f1da349c5dbe7c4ec93048cdc785a3326"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:ea9f9c6034ea2d93d9147818f17c2a0860d41b71c38b9ce4d55f21b6f9165a11"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:80d1543d58bd3d6c271b66abf454d437a438dff01c3e62fdbcd68f2a11310d4b"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:73dc03a6a7e30b7edc5b01b601e53e7fc924b04e1835e8e407c12c037e81adbd"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:6f5c2e7bc8a4bf7c426599765b1bd33217ec84023033672c1e9a8b35eaeaaaf8"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-win32.whl", hash = "sha256:12a2b561af122e3d94cdb97fe6fb2bb2b82cef0cdca131646fdb940a1eda04f0"}, - {file = "charset_normalizer-3.1.0-cp38-cp38-win_amd64.whl", hash = "sha256:3160a0fd9754aab7d47f95a6b63ab355388d890163eb03b2d2b87ab0a30cfa59"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:38e812a197bf8e71a59fe55b757a84c1f946d0ac114acafaafaf21667a7e169e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:6baf0baf0d5d265fa7944feb9f7451cc316bfe30e8df1a61b1bb08577c554f31"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:8f25e17ab3039b05f762b0a55ae0b3632b2e073d9c8fc88e89aca31a6198e88f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3747443b6a904001473370d7810aa19c3a180ccd52a7157aacc264a5ac79265e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:b116502087ce8a6b7a5f1814568ccbd0e9f6cfd99948aa59b0e241dc57cf739f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:d16fd5252f883eb074ca55cb622bc0bee49b979ae4e8639fff6ca3ff44f9f854"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:21fa558996782fc226b529fdd2ed7866c2c6ec91cee82735c98a197fae39f706"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6f6c7a8a57e9405cad7485f4c9d3172ae486cfef1344b5ddd8e5239582d7355e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:ac3775e3311661d4adace3697a52ac0bab17edd166087d493b52d4f4f553f9f0"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:10c93628d7497c81686e8e5e557aafa78f230cd9e77dd0c40032ef90c18f2230"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:6f4f4668e1831850ebcc2fd0b1cd11721947b6dc7c00bf1c6bd3c929ae14f2c7"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:0be65ccf618c1e7ac9b849c315cc2e8a8751d9cfdaa43027d4f6624bd587ab7e"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:53d0a3fa5f8af98a1e261de6a3943ca631c526635eb5817a87a59d9a57ebf48f"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-win32.whl", hash = "sha256:a04f86f41a8916fe45ac5024ec477f41f886b3c435da2d4e3d2709b22ab02af1"}, - {file = "charset_normalizer-3.1.0-cp39-cp39-win_amd64.whl", hash = "sha256:830d2948a5ec37c386d3170c483063798d7879037492540f10a475e3fd6f244b"}, - {file = "charset_normalizer-3.1.0-py3-none-any.whl", hash = "sha256:3d9098b479e78c85080c98e1e35ff40b4a31d8953102bb0fd7d1b6f8a2111a3d"}, -] -click = [ - {file = "click-8.1.3-py3-none-any.whl", hash = "sha256:bb4d8133cb15a609f44e8213d9b391b0809795062913b383c62be0ee95b1db48"}, - {file = "click-8.1.3.tar.gz", hash = "sha256:7682dc8afb30297001674575ea00d1814d808d6a36af415a82bd481d37ba7b8e"}, -] -colorama = [ - {file = "colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6"}, - {file = "colorama-0.4.6.tar.gz", hash = "sha256:08695f5cb7ed6e0531a20572697297273c47b8cae5a63ffc6d6ed5c201be6e44"}, -] -dill = [ - {file = "dill-0.3.6-py3-none-any.whl", hash = "sha256:a07ffd2351b8c678dfc4a856a3005f8067aea51d6ba6c700796a4d9e280f39f0"}, - {file = "dill-0.3.6.tar.gz", hash = "sha256:e5db55f3687856d8fbdab002ed78544e1c4559a130302693d839dfe8f93f2373"}, -] -et-xmlfile = [ - {file = "et_xmlfile-1.1.0-py3-none-any.whl", hash = "sha256:a2ba85d1d6a74ef63837eed693bcb89c3f752169b0e3e7ae5b16ca5e1b3deada"}, - {file = "et_xmlfile-1.1.0.tar.gz", hash = "sha256:8eb9e2bc2f8c97e37a2dc85a09ecdcdec9d8a396530a6d5a33b30b9a92da0c5c"}, -] -exceptiongroup = [ - {file = "exceptiongroup-1.1.1-py3-none-any.whl", hash = "sha256:232c37c63e4f682982c8b6459f33a8981039e5fb8756b2074364e5055c498c9e"}, - {file = "exceptiongroup-1.1.1.tar.gz", hash = "sha256:d484c3090ba2889ae2928419117447a14daf3c1231d5e30d0aae34f354f01785"}, -] -greenlet = [ - {file = "greenlet-2.0.2-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:bdfea8c661e80d3c1c99ad7c3ff74e6e87184895bbaca6ee8cc61209f8b9b85d"}, - {file = "greenlet-2.0.2-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:9d14b83fab60d5e8abe587d51c75b252bcc21683f24699ada8fb275d7712f5a9"}, - {file = "greenlet-2.0.2-cp27-cp27m-win32.whl", hash = "sha256:6c3acb79b0bfd4fe733dff8bc62695283b57949ebcca05ae5c129eb606ff2d74"}, - {file = "greenlet-2.0.2-cp27-cp27m-win_amd64.whl", hash = "sha256:283737e0da3f08bd637b5ad058507e578dd462db259f7f6e4c5c365ba4ee9343"}, - {file = "greenlet-2.0.2-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:d27ec7509b9c18b6d73f2f5ede2622441de812e7b1a80bbd446cb0633bd3d5ae"}, - {file = "greenlet-2.0.2-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:30bcf80dda7f15ac77ba5af2b961bdd9dbc77fd4ac6105cee85b0d0a5fcf74df"}, - {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:26fbfce90728d82bc9e6c38ea4d038cba20b7faf8a0ca53a9c07b67318d46088"}, - {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:9190f09060ea4debddd24665d6804b995a9c122ef5917ab26e1566dcc712ceeb"}, - {file = "greenlet-2.0.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:d75209eed723105f9596807495d58d10b3470fa6732dd6756595e89925ce2470"}, - {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:3a51c9751078733d88e013587b108f1b7a1fb106d402fb390740f002b6f6551a"}, - {file = "greenlet-2.0.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:76ae285c8104046b3a7f06b42f29c7b73f77683df18c49ab5af7983994c2dd91"}, - {file = "greenlet-2.0.2-cp310-cp310-win_amd64.whl", hash = "sha256:2d4686f195e32d36b4d7cf2d166857dbd0ee9f3d20ae349b6bf8afc8485b3645"}, - {file = "greenlet-2.0.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:c4302695ad8027363e96311df24ee28978162cdcdd2006476c43970b384a244c"}, - {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c48f54ef8e05f04d6eff74b8233f6063cb1ed960243eacc474ee73a2ea8573ca"}, - {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:a1846f1b999e78e13837c93c778dcfc3365902cfb8d1bdb7dd73ead37059f0d0"}, - {file = "greenlet-2.0.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3a06ad5312349fec0ab944664b01d26f8d1f05009566339ac6f63f56589bc1a2"}, - {file = "greenlet-2.0.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:eff4eb9b7eb3e4d0cae3d28c283dc16d9bed6b193c2e1ace3ed86ce48ea8df19"}, - {file = "greenlet-2.0.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:5454276c07d27a740c5892f4907c86327b632127dd9abec42ee62e12427ff7e3"}, - {file = "greenlet-2.0.2-cp311-cp311-win_amd64.whl", hash = "sha256:7cafd1208fdbe93b67c7086876f061f660cfddc44f404279c1585bbf3cdc64c5"}, - {file = "greenlet-2.0.2-cp35-cp35m-macosx_10_14_x86_64.whl", hash = "sha256:910841381caba4f744a44bf81bfd573c94e10b3045ee00de0cbf436fe50673a6"}, - {file = "greenlet-2.0.2-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:18a7f18b82b52ee85322d7a7874e676f34ab319b9f8cce5de06067384aa8ff43"}, - {file = "greenlet-2.0.2-cp35-cp35m-win32.whl", hash = "sha256:03a8f4f3430c3b3ff8d10a2a86028c660355ab637cee9333d63d66b56f09d52a"}, - {file = "greenlet-2.0.2-cp35-cp35m-win_amd64.whl", hash = "sha256:4b58adb399c4d61d912c4c331984d60eb66565175cdf4a34792cd9600f21b394"}, - {file = "greenlet-2.0.2-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:703f18f3fda276b9a916f0934d2fb6d989bf0b4fb5a64825260eb9bfd52d78f0"}, - {file = "greenlet-2.0.2-cp36-cp36m-manylinux2010_x86_64.whl", hash = "sha256:32e5b64b148966d9cccc2c8d35a671409e45f195864560829f395a54226408d3"}, - {file = "greenlet-2.0.2-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2dd11f291565a81d71dab10b7033395b7a3a5456e637cf997a6f33ebdf06f8db"}, - {file = "greenlet-2.0.2-cp36-cp36m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e0f72c9ddb8cd28532185f54cc1453f2c16fb417a08b53a855c4e6a418edd099"}, - {file = "greenlet-2.0.2-cp36-cp36m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:cd021c754b162c0fb55ad5d6b9d960db667faad0fa2ff25bb6e1301b0b6e6a75"}, - {file = "greenlet-2.0.2-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:3c9b12575734155d0c09d6c3e10dbd81665d5c18e1a7c6597df72fd05990c8cf"}, - {file = "greenlet-2.0.2-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b9ec052b06a0524f0e35bd8790686a1da006bd911dd1ef7d50b77bfbad74e292"}, - {file = "greenlet-2.0.2-cp36-cp36m-win32.whl", hash = "sha256:dbfcfc0218093a19c252ca8eb9aee3d29cfdcb586df21049b9d777fd32c14fd9"}, - {file = "greenlet-2.0.2-cp36-cp36m-win_amd64.whl", hash = "sha256:9f35ec95538f50292f6d8f2c9c9f8a3c6540bbfec21c9e5b4b751e0a7c20864f"}, - {file = "greenlet-2.0.2-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:d5508f0b173e6aa47273bdc0a0b5ba055b59662ba7c7ee5119528f466585526b"}, - {file = "greenlet-2.0.2-cp37-cp37m-manylinux2010_x86_64.whl", hash = "sha256:f82d4d717d8ef19188687aa32b8363e96062911e63ba22a0cff7802a8e58e5f1"}, - {file = "greenlet-2.0.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c9c59a2120b55788e800d82dfa99b9e156ff8f2227f07c5e3012a45a399620b7"}, - {file = "greenlet-2.0.2-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:2780572ec463d44c1d3ae850239508dbeb9fed38e294c68d19a24d925d9223ca"}, - {file = "greenlet-2.0.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:937e9020b514ceedb9c830c55d5c9872abc90f4b5862f89c0887033ae33c6f73"}, - {file = "greenlet-2.0.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:36abbf031e1c0f79dd5d596bfaf8e921c41df2bdf54ee1eed921ce1f52999a86"}, - {file = "greenlet-2.0.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:18e98fb3de7dba1c0a852731c3070cf022d14f0d68b4c87a19cc1016f3bb8b33"}, - {file = "greenlet-2.0.2-cp37-cp37m-win32.whl", hash = "sha256:3f6ea9bd35eb450837a3d80e77b517ea5bc56b4647f5502cd28de13675ee12f7"}, - {file = "greenlet-2.0.2-cp37-cp37m-win_amd64.whl", hash = "sha256:7492e2b7bd7c9b9916388d9df23fa49d9b88ac0640db0a5b4ecc2b653bf451e3"}, - {file = "greenlet-2.0.2-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:b864ba53912b6c3ab6bcb2beb19f19edd01a6bfcbdfe1f37ddd1778abfe75a30"}, - {file = "greenlet-2.0.2-cp38-cp38-manylinux2010_x86_64.whl", hash = "sha256:ba2956617f1c42598a308a84c6cf021a90ff3862eddafd20c3333d50f0edb45b"}, - {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fc3a569657468b6f3fb60587e48356fe512c1754ca05a564f11366ac9e306526"}, - {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:8eab883b3b2a38cc1e050819ef06a7e6344d4a990d24d45bc6f2cf959045a45b"}, - {file = "greenlet-2.0.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:acd2162a36d3de67ee896c43effcd5ee3de247eb00354db411feb025aa319857"}, - {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:0bf60faf0bc2468089bdc5edd10555bab6e85152191df713e2ab1fcc86382b5a"}, - {file = "greenlet-2.0.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:b0ef99cdbe2b682b9ccbb964743a6aca37905fda5e0452e5ee239b1654d37f2a"}, - {file = "greenlet-2.0.2-cp38-cp38-win32.whl", hash = "sha256:b80f600eddddce72320dbbc8e3784d16bd3fb7b517e82476d8da921f27d4b249"}, - {file = "greenlet-2.0.2-cp38-cp38-win_amd64.whl", hash = "sha256:4d2e11331fc0c02b6e84b0d28ece3a36e0548ee1a1ce9ddde03752d9b79bba40"}, - {file = "greenlet-2.0.2-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:88d9ab96491d38a5ab7c56dd7a3cc37d83336ecc564e4e8816dbed12e5aaefc8"}, - {file = "greenlet-2.0.2-cp39-cp39-manylinux2010_x86_64.whl", hash = "sha256:561091a7be172ab497a3527602d467e2b3fbe75f9e783d8b8ce403fa414f71a6"}, - {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:971ce5e14dc5e73715755d0ca2975ac88cfdaefcaab078a284fea6cfabf866df"}, - {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:be4ed120b52ae4d974aa40215fcdfde9194d63541c7ded40ee12eb4dda57b76b"}, - {file = "greenlet-2.0.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:94c817e84245513926588caf1152e3b559ff794d505555211ca041f032abbb6b"}, - {file = "greenlet-2.0.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:1a819eef4b0e0b96bb0d98d797bef17dc1b4a10e8d7446be32d1da33e095dbb8"}, - {file = "greenlet-2.0.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7efde645ca1cc441d6dc4b48c0f7101e8d86b54c8530141b09fd31cef5149ec9"}, - {file = "greenlet-2.0.2-cp39-cp39-win32.whl", hash = "sha256:ea9872c80c132f4663822dd2a08d404073a5a9b5ba6155bea72fb2a79d1093b5"}, - {file = "greenlet-2.0.2-cp39-cp39-win_amd64.whl", hash = "sha256:db1a39669102a1d8d12b57de2bb7e2ec9066a6f2b3da35ae511ff93b01b5d564"}, - {file = "greenlet-2.0.2.tar.gz", hash = "sha256:e7c8dc13af7db097bed64a051d2dd49e9f0af495c26995c00a9ee842690d34c0"}, -] -idna = [ - {file = "idna-3.4-py3-none-any.whl", hash = "sha256:90b77e79eaa3eba6de819a0c442c0b4ceefc341a7a2ab77d7562bf49f425c5c2"}, - {file = "idna-3.4.tar.gz", hash = "sha256:814f528e8dead7d329833b91c5faa87d60bf71824cd12a7530b5526063d02cb4"}, -] -importlib-metadata = [ - {file = "importlib_metadata-6.6.0-py3-none-any.whl", hash = "sha256:43dd286a2cd8995d5eaef7fee2066340423b818ed3fd70adf0bad5f1fac53fed"}, - {file = "importlib_metadata-6.6.0.tar.gz", hash = "sha256:92501cdf9cc66ebd3e612f1b4f0c0765dfa42f0fa38ffb319b6bd84dd675d705"}, -] -importlib-resources = [ - {file = "importlib_resources-5.12.0-py3-none-any.whl", hash = "sha256:7b1deeebbf351c7578e09bf2f63fa2ce8b5ffec296e0d349139d43cca061a81a"}, - {file = "importlib_resources-5.12.0.tar.gz", hash = "sha256:4be82589bf5c1d7999aedf2a45159d10cb3ca4f19b2271f8792bc8e6da7b22f6"}, -] -iniconfig = [ - {file = "iniconfig-2.0.0-py3-none-any.whl", hash = "sha256:b6a85871a79d2e3b22d2d1b94ac2824226a63c6b741c88f7ae975f18b6778374"}, - {file = "iniconfig-2.0.0.tar.gz", hash = "sha256:2d91e135bf72d31a410b17c16da610a82cb55f6b0477d1a902134b24a455b8b3"}, -] -isort = [ - {file = "isort-5.11.5-py3-none-any.whl", hash = "sha256:ba1d72fb2595a01c7895a5128f9585a5cc4b6d395f1c8d514989b9a7eb2a8746"}, - {file = "isort-5.11.5.tar.gz", hash = "sha256:6be1f76a507cb2ecf16c7cf14a37e41609ca082330be4e3436a18ef74add55db"}, -] -lazy-object-proxy = [ - {file = "lazy-object-proxy-1.9.0.tar.gz", hash = "sha256:659fb5809fa4629b8a1ac5106f669cfc7bef26fbb389dda53b3e010d1ac4ebae"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b40387277b0ed2d0602b8293b94d7257e17d1479e257b4de114ea11a8cb7f2d7"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:e8c6cfb338b133fbdbc5cfaa10fe3c6aeea827db80c978dbd13bc9dd8526b7d4"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:721532711daa7db0d8b779b0bb0318fa87af1c10d7fe5e52ef30f8eff254d0cd"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:66a3de4a3ec06cd8af3f61b8e1ec67614fbb7c995d02fa224813cb7afefee701"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:1aa3de4088c89a1b69f8ec0dcc169aa725b0ff017899ac568fe44ddc1396df46"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-win32.whl", hash = "sha256:f0705c376533ed2a9e5e97aacdbfe04cecd71e0aa84c7c0595d02ef93b6e4455"}, - {file = "lazy_object_proxy-1.9.0-cp310-cp310-win_amd64.whl", hash = "sha256:ea806fd4c37bf7e7ad82537b0757999264d5f70c45468447bb2b91afdbe73a6e"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:946d27deaff6cf8452ed0dba83ba38839a87f4f7a9732e8f9fd4107b21e6ff07"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:79a31b086e7e68b24b99b23d57723ef7e2c6d81ed21007b6281ebcd1688acb0a"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f699ac1c768270c9e384e4cbd268d6e67aebcfae6cd623b4d7c3bfde5a35db59"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:bfb38f9ffb53b942f2b5954e0f610f1e721ccebe9cce9025a38c8ccf4a5183a4"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:189bbd5d41ae7a498397287c408617fe5c48633e7755287b21d741f7db2706a9"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-win32.whl", hash = "sha256:81fc4d08b062b535d95c9ea70dbe8a335c45c04029878e62d744bdced5141586"}, - {file = "lazy_object_proxy-1.9.0-cp311-cp311-win_amd64.whl", hash = "sha256:f2457189d8257dd41ae9b434ba33298aec198e30adf2dcdaaa3a28b9994f6adb"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:d9e25ef10a39e8afe59a5c348a4dbf29b4868ab76269f81ce1674494e2565a6e"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cbf9b082426036e19c6924a9ce90c740a9861e2bdc27a4834fd0a910742ac1e8"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9f5fa4a61ce2438267163891961cfd5e32ec97a2c444e5b842d574251ade27d2"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:8fa02eaab317b1e9e03f69aab1f91e120e7899b392c4fc19807a8278a07a97e8"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:e7c21c95cae3c05c14aafffe2865bbd5e377cfc1348c4f7751d9dc9a48ca4bda"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-win32.whl", hash = "sha256:f12ad7126ae0c98d601a7ee504c1122bcef553d1d5e0c3bfa77b16b3968d2734"}, - {file = "lazy_object_proxy-1.9.0-cp37-cp37m-win_amd64.whl", hash = "sha256:edd20c5a55acb67c7ed471fa2b5fb66cb17f61430b7a6b9c3b4a1e40293b1671"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2d0daa332786cf3bb49e10dc6a17a52f6a8f9601b4cf5c295a4f85854d61de63"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9cd077f3d04a58e83d04b20e334f678c2b0ff9879b9375ed107d5d07ff160171"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:660c94ea760b3ce47d1855a30984c78327500493d396eac4dfd8bd82041b22be"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:212774e4dfa851e74d393a2370871e174d7ff0ebc980907723bb67d25c8a7c30"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:f0117049dd1d5635bbff65444496c90e0baa48ea405125c088e93d9cf4525b11"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-win32.whl", hash = "sha256:0a891e4e41b54fd5b8313b96399f8b0e173bbbfc03c7631f01efbe29bb0bcf82"}, - {file = "lazy_object_proxy-1.9.0-cp38-cp38-win_amd64.whl", hash = "sha256:9990d8e71b9f6488e91ad25f322898c136b008d87bf852ff65391b004da5e17b"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:9e7551208b2aded9c1447453ee366f1c4070602b3d932ace044715d89666899b"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f83ac4d83ef0ab017683d715ed356e30dd48a93746309c8f3517e1287523ef4"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7322c3d6f1766d4ef1e51a465f47955f1e8123caee67dd641e67d539a534d006"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:18b78ec83edbbeb69efdc0e9c1cb41a3b1b1ed11ddd8ded602464c3fc6020494"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:09763491ce220c0299688940f8dc2c5d05fd1f45af1e42e636b2e8b2303e4382"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-win32.whl", hash = "sha256:9090d8e53235aa280fc9239a86ae3ea8ac58eff66a705fa6aa2ec4968b95c821"}, - {file = "lazy_object_proxy-1.9.0-cp39-cp39-win_amd64.whl", hash = "sha256:db1c1722726f47e10e0b5fdbf15ac3b8adb58c091d12b3ab713965795036985f"}, -] -lz4 = [ - {file = "lz4-4.3.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:1c4c100d99eed7c08d4e8852dd11e7d1ec47a3340f49e3a96f8dfbba17ffb300"}, - {file = "lz4-4.3.2-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:edd8987d8415b5dad25e797043936d91535017237f72fa456601be1479386c92"}, - {file = "lz4-4.3.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f7c50542b4ddceb74ab4f8b3435327a0861f06257ca501d59067a6a482535a77"}, - {file = "lz4-4.3.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:0f5614d8229b33d4a97cb527db2a1ac81308c6e796e7bdb5d1309127289f69d5"}, - {file = "lz4-4.3.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:8f00a9ba98f6364cadda366ae6469b7b3568c0cced27e16a47ddf6b774169270"}, - {file = "lz4-4.3.2-cp310-cp310-win32.whl", hash = "sha256:b10b77dc2e6b1daa2f11e241141ab8285c42b4ed13a8642495620416279cc5b2"}, - {file = "lz4-4.3.2-cp310-cp310-win_amd64.whl", hash = "sha256:86480f14a188c37cb1416cdabacfb4e42f7a5eab20a737dac9c4b1c227f3b822"}, - {file = "lz4-4.3.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:7c2df117def1589fba1327dceee51c5c2176a2b5a7040b45e84185ce0c08b6a3"}, - {file = "lz4-4.3.2-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:1f25eb322eeb24068bb7647cae2b0732b71e5c639e4e4026db57618dcd8279f0"}, - {file = "lz4-4.3.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8df16c9a2377bdc01e01e6de5a6e4bbc66ddf007a6b045688e285d7d9d61d1c9"}, - {file = "lz4-4.3.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f571eab7fec554d3b1db0d666bdc2ad85c81f4b8cb08906c4c59a8cad75e6e22"}, - {file = "lz4-4.3.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7211dc8f636ca625abc3d4fb9ab74e5444b92df4f8d58ec83c8868a2b0ff643d"}, - {file = "lz4-4.3.2-cp311-cp311-win32.whl", hash = "sha256:867664d9ca9bdfce840ac96d46cd8838c9ae891e859eb98ce82fcdf0e103a947"}, - {file = "lz4-4.3.2-cp311-cp311-win_amd64.whl", hash = "sha256:a6a46889325fd60b8a6b62ffc61588ec500a1883db32cddee9903edfba0b7584"}, - {file = "lz4-4.3.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:3a85b430138882f82f354135b98c320dafb96fc8fe4656573d95ab05de9eb092"}, - {file = "lz4-4.3.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65d5c93f8badacfa0456b660285e394e65023ef8071142e0dcbd4762166e1be0"}, - {file = "lz4-4.3.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6b50f096a6a25f3b2edca05aa626ce39979d63c3b160687c8c6d50ac3943d0ba"}, - {file = "lz4-4.3.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:200d05777d61ba1ff8d29cb51c534a162ea0b4fe6d3c28be3571a0a48ff36080"}, - {file = "lz4-4.3.2-cp37-cp37m-win32.whl", hash = "sha256:edc2fb3463d5d9338ccf13eb512aab61937be50aa70734bcf873f2f493801d3b"}, - {file = "lz4-4.3.2-cp37-cp37m-win_amd64.whl", hash = "sha256:83acfacab3a1a7ab9694333bcb7950fbeb0be21660d236fd09c8337a50817897"}, - {file = "lz4-4.3.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7a9eec24ec7d8c99aab54de91b4a5a149559ed5b3097cf30249b665689b3d402"}, - {file = "lz4-4.3.2-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:31d72731c4ac6ebdce57cd9a5cabe0aecba229c4f31ba3e2c64ae52eee3fdb1c"}, - {file = "lz4-4.3.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:83903fe6db92db0be101acedc677aa41a490b561567fe1b3fe68695b2110326c"}, - {file = "lz4-4.3.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:926b26db87ec8822cf1870efc3d04d06062730ec3279bbbd33ba47a6c0a5c673"}, - {file = "lz4-4.3.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:e05afefc4529e97c08e65ef92432e5f5225c0bb21ad89dee1e06a882f91d7f5e"}, - {file = "lz4-4.3.2-cp38-cp38-win32.whl", hash = "sha256:ad38dc6a7eea6f6b8b642aaa0683253288b0460b70cab3216838747163fb774d"}, - {file = "lz4-4.3.2-cp38-cp38-win_amd64.whl", hash = "sha256:7e2dc1bd88b60fa09b9b37f08553f45dc2b770c52a5996ea52b2b40f25445676"}, - {file = "lz4-4.3.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:edda4fb109439b7f3f58ed6bede59694bc631c4b69c041112b1b7dc727fffb23"}, - {file = "lz4-4.3.2-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ca83a623c449295bafad745dcd399cea4c55b16b13ed8cfea30963b004016c9"}, - {file = "lz4-4.3.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d5ea0e788dc7e2311989b78cae7accf75a580827b4d96bbaf06c7e5a03989bd5"}, - {file = "lz4-4.3.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a98b61e504fb69f99117b188e60b71e3c94469295571492a6468c1acd63c37ba"}, - {file = "lz4-4.3.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:4931ab28a0d1c133104613e74eec1b8bb1f52403faabe4f47f93008785c0b929"}, - {file = "lz4-4.3.2-cp39-cp39-win32.whl", hash = "sha256:ec6755cacf83f0c5588d28abb40a1ac1643f2ff2115481089264c7630236618a"}, - {file = "lz4-4.3.2-cp39-cp39-win_amd64.whl", hash = "sha256:4caedeb19e3ede6c7a178968b800f910db6503cb4cb1e9cc9221157572139b49"}, - {file = "lz4-4.3.2.tar.gz", hash = "sha256:e1431d84a9cfb23e6773e72078ce8e65cad6745816d4cbf9ae67da5ea419acda"}, -] -mako = [ - {file = "Mako-1.2.4-py3-none-any.whl", hash = "sha256:c97c79c018b9165ac9922ae4f32da095ffd3c4e6872b45eded42926deea46818"}, - {file = "Mako-1.2.4.tar.gz", hash = "sha256:d60a3903dc3bb01a18ad6a89cdbe2e4eadc69c0bc8ef1e3773ba53d44c3f7a34"}, -] -markupsafe = [ - {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:665a36ae6f8f20a4676b53224e33d456a6f5a72657d9c83c2aa00765072f31f7"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:340bea174e9761308703ae988e982005aedf427de816d1afe98147668cc03036"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:22152d00bf4a9c7c83960521fc558f55a1adbc0631fbb00a9471e097b19d72e1"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:28057e985dace2f478e042eaa15606c7efccb700797660629da387eb289b9323"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:ca244fa73f50a800cf8c3ebf7fd93149ec37f5cb9596aa8873ae2c1d23498601"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d9d971ec1e79906046aa3ca266de79eac42f1dbf3612a05dc9368125952bd1a1"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:7e007132af78ea9df29495dbf7b5824cb71648d7133cf7848a2a5dd00d36f9ff"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:7313ce6a199651c4ed9d7e4cfb4aa56fe923b1adf9af3b420ee14e6d9a73df65"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-win32.whl", hash = "sha256:c4a549890a45f57f1ebf99c067a4ad0cb423a05544accaf2b065246827ed9603"}, - {file = "MarkupSafe-2.1.2-cp310-cp310-win_amd64.whl", hash = "sha256:835fb5e38fd89328e9c81067fd642b3593c33e1e17e2fdbf77f5676abb14a156"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:2ec4f2d48ae59bbb9d1f9d7efb9236ab81429a764dedca114f5fdabbc3788013"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:608e7073dfa9e38a85d38474c082d4281f4ce276ac0010224eaba11e929dd53a"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:65608c35bfb8a76763f37036547f7adfd09270fbdbf96608be2bead319728fcd"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f2bfb563d0211ce16b63c7cb9395d2c682a23187f54c3d79bfec33e6705473c6"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:da25303d91526aac3672ee6d49a2f3db2d9502a4a60b55519feb1a4c7714e07d"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:9cad97ab29dfc3f0249b483412c85c8ef4766d96cdf9dcf5a1e3caa3f3661cf1"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:085fd3201e7b12809f9e6e9bc1e5c96a368c8523fad5afb02afe3c051ae4afcc"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:1bea30e9bf331f3fef67e0a3877b2288593c98a21ccb2cf29b74c581a4eb3af0"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-win32.whl", hash = "sha256:7df70907e00c970c60b9ef2938d894a9381f38e6b9db73c5be35e59d92e06625"}, - {file = "MarkupSafe-2.1.2-cp311-cp311-win_amd64.whl", hash = "sha256:e55e40ff0cc8cc5c07996915ad367fa47da6b3fc091fdadca7f5403239c5fec3"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:a6e40afa7f45939ca356f348c8e23048e02cb109ced1eb8420961b2f40fb373a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:cf877ab4ed6e302ec1d04952ca358b381a882fbd9d1b07cccbfd61783561f98a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:63ba06c9941e46fa389d389644e2d8225e0e3e5ebcc4ff1ea8506dce646f8c8a"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:f1cd098434e83e656abf198f103a8207a8187c0fc110306691a2e94a78d0abb2"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:55f44b440d491028addb3b88f72207d71eeebfb7b5dbf0643f7c023ae1fba619"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:a6f2fcca746e8d5910e18782f976489939d54a91f9411c32051b4aab2bd7c513"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:0b462104ba25f1ac006fdab8b6a01ebbfbce9ed37fd37fd4acd70c67c973e460"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-win32.whl", hash = "sha256:7668b52e102d0ed87cb082380a7e2e1e78737ddecdde129acadb0eccc5423859"}, - {file = "MarkupSafe-2.1.2-cp37-cp37m-win_amd64.whl", hash = "sha256:6d6607f98fcf17e534162f0709aaad3ab7a96032723d8ac8750ffe17ae5a0666"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:a806db027852538d2ad7555b203300173dd1b77ba116de92da9afbc3a3be3eed"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:a4abaec6ca3ad8660690236d11bfe28dfd707778e2442b45addd2f086d6ef094"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f03a532d7dee1bed20bc4884194a16160a2de9ffc6354b3878ec9682bb623c54"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4cf06cdc1dda95223e9d2d3c58d3b178aa5dacb35ee7e3bbac10e4e1faacb419"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:22731d79ed2eb25059ae3df1dfc9cb1546691cc41f4e3130fe6bfbc3ecbbecfa"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:f8ffb705ffcf5ddd0e80b65ddf7bed7ee4f5a441ea7d3419e861a12eaf41af58"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:8db032bf0ce9022a8e41a22598eefc802314e81b879ae093f36ce9ddf39ab1ba"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:2298c859cfc5463f1b64bd55cb3e602528db6fa0f3cfd568d3605c50678f8f03"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-win32.whl", hash = "sha256:50c42830a633fa0cf9e7d27664637532791bfc31c731a87b202d2d8ac40c3ea2"}, - {file = "MarkupSafe-2.1.2-cp38-cp38-win_amd64.whl", hash = "sha256:bb06feb762bade6bf3c8b844462274db0c76acc95c52abe8dbed28ae3d44a147"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:99625a92da8229df6d44335e6fcc558a5037dd0a760e11d84be2260e6f37002f"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:8bca7e26c1dd751236cfb0c6c72d4ad61d986e9a41bbf76cb445f69488b2a2bd"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:40627dcf047dadb22cd25ea7ecfe9cbf3bbbad0482ee5920b582f3809c97654f"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:40dfd3fefbef579ee058f139733ac336312663c6706d1163b82b3003fb1925c4"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:090376d812fb6ac5f171e5938e82e7f2d7adc2b629101cec0db8b267815c85e2"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:2e7821bffe00aa6bd07a23913b7f4e01328c3d5cc0b40b36c0bd81d362faeb65"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:c0a33bc9f02c2b17c3ea382f91b4db0e6cde90b63b296422a939886a7a80de1c"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:b8526c6d437855442cdd3d87eede9c425c4445ea011ca38d937db299382e6fa3"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-win32.whl", hash = "sha256:137678c63c977754abe9086a3ec011e8fd985ab90631145dfb9294ad09c102a7"}, - {file = "MarkupSafe-2.1.2-cp39-cp39-win_amd64.whl", hash = "sha256:0576fe974b40a400449768941d5d0858cc624e3249dfd1e0c33674e5c7ca7aed"}, - {file = "MarkupSafe-2.1.2.tar.gz", hash = "sha256:abcabc8c2b26036d62d4c746381a6f7cf60aafcc653198ad678306986b09450d"}, -] -mccabe = [ - {file = "mccabe-0.7.0-py2.py3-none-any.whl", hash = "sha256:6c2d30ab6be0e4a46919781807b4f0d834ebdd6c6e3dca0bda5a15f863427b6e"}, - {file = "mccabe-0.7.0.tar.gz", hash = "sha256:348e0240c33b60bbdf4e523192ef919f28cb2c3d7d5c7794f74009290f236325"}, -] -mypy = [ - {file = "mypy-0.950-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:cf9c261958a769a3bd38c3e133801ebcd284ffb734ea12d01457cb09eacf7d7b"}, - {file = "mypy-0.950-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:b5b5bd0ffb11b4aba2bb6d31b8643902c48f990cc92fda4e21afac658044f0c0"}, - {file = "mypy-0.950-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:5e7647df0f8fc947388e6251d728189cfadb3b1e558407f93254e35abc026e22"}, - {file = "mypy-0.950-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:eaff8156016487c1af5ffa5304c3e3fd183edcb412f3e9c72db349faf3f6e0eb"}, - {file = "mypy-0.950-cp310-cp310-win_amd64.whl", hash = "sha256:563514c7dc504698fb66bb1cf897657a173a496406f1866afae73ab5b3cdb334"}, - {file = "mypy-0.950-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:dd4d670eee9610bf61c25c940e9ade2d0ed05eb44227275cce88701fee014b1f"}, - {file = "mypy-0.950-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:ca75ecf2783395ca3016a5e455cb322ba26b6d33b4b413fcdedfc632e67941dc"}, - {file = "mypy-0.950-cp36-cp36m-win_amd64.whl", hash = "sha256:6003de687c13196e8a1243a5e4bcce617d79b88f83ee6625437e335d89dfebe2"}, - {file = "mypy-0.950-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4c653e4846f287051599ed8f4b3c044b80e540e88feec76b11044ddc5612ffed"}, - {file = "mypy-0.950-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e19736af56947addedce4674c0971e5dceef1b5ec7d667fe86bcd2b07f8f9075"}, - {file = "mypy-0.950-cp37-cp37m-win_amd64.whl", hash = "sha256:ef7beb2a3582eb7a9f37beaf38a28acfd801988cde688760aea9e6cc4832b10b"}, - {file = "mypy-0.950-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:0112752a6ff07230f9ec2f71b0d3d4e088a910fdce454fdb6553e83ed0eced7d"}, - {file = "mypy-0.950-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:ee0a36edd332ed2c5208565ae6e3a7afc0eabb53f5327e281f2ef03a6bc7687a"}, - {file = "mypy-0.950-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:77423570c04aca807508a492037abbd72b12a1fb25a385847d191cd50b2c9605"}, - {file = "mypy-0.950-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:5ce6a09042b6da16d773d2110e44f169683d8cc8687e79ec6d1181a72cb028d2"}, - {file = "mypy-0.950-cp38-cp38-win_amd64.whl", hash = "sha256:5b231afd6a6e951381b9ef09a1223b1feabe13625388db48a8690f8daa9b71ff"}, - {file = "mypy-0.950-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:0384d9f3af49837baa92f559d3fa673e6d2652a16550a9ee07fc08c736f5e6f8"}, - {file = "mypy-0.950-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:1fdeb0a0f64f2a874a4c1f5271f06e40e1e9779bf55f9567f149466fc7a55038"}, - {file = "mypy-0.950-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:61504b9a5ae166ba5ecfed9e93357fd51aa693d3d434b582a925338a2ff57fd2"}, - {file = "mypy-0.950-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a952b8bc0ae278fc6316e6384f67bb9a396eb30aced6ad034d3a76120ebcc519"}, - {file = "mypy-0.950-cp39-cp39-win_amd64.whl", hash = "sha256:eaea21d150fb26d7b4856766e7addcf929119dd19fc832b22e71d942835201ef"}, - {file = "mypy-0.950-py3-none-any.whl", hash = "sha256:a4d9898f46446bfb6405383b57b96737dcfd0a7f25b748e78ef3e8c576bba3cb"}, - {file = "mypy-0.950.tar.gz", hash = "sha256:1b333cfbca1762ff15808a0ef4f71b5d3eed8528b23ea1c3fb50543c867d68de"}, -] -mypy-extensions = [ - {file = "mypy_extensions-1.0.0-py3-none-any.whl", hash = "sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d"}, - {file = "mypy_extensions-1.0.0.tar.gz", hash = "sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782"}, -] -numpy = [ - {file = "numpy-1.21.6-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:8737609c3bbdd48e380d463134a35ffad3b22dc56295eff6f79fd85bd0eeeb25"}, - {file = "numpy-1.21.6-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:fdffbfb6832cd0b300995a2b08b8f6fa9f6e856d562800fea9182316d99c4e8e"}, - {file = "numpy-1.21.6-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:3820724272f9913b597ccd13a467cc492a0da6b05df26ea09e78b171a0bb9da6"}, - {file = "numpy-1.21.6-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:f17e562de9edf691a42ddb1eb4a5541c20dd3f9e65b09ded2beb0799c0cf29bb"}, - {file = "numpy-1.21.6-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5f30427731561ce75d7048ac254dbe47a2ba576229250fb60f0fb74db96501a1"}, - {file = "numpy-1.21.6-cp310-cp310-win32.whl", hash = "sha256:d4bf4d43077db55589ffc9009c0ba0a94fa4908b9586d6ccce2e0b164c86303c"}, - {file = "numpy-1.21.6-cp310-cp310-win_amd64.whl", hash = "sha256:d136337ae3cc69aa5e447e78d8e1514be8c3ec9b54264e680cf0b4bd9011574f"}, - {file = "numpy-1.21.6-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:6aaf96c7f8cebc220cdfc03f1d5a31952f027dda050e5a703a0d1c396075e3e7"}, - {file = "numpy-1.21.6-cp37-cp37m-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:67c261d6c0a9981820c3a149d255a76918278a6b03b6a036800359aba1256d46"}, - {file = "numpy-1.21.6-cp37-cp37m-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:a6be4cb0ef3b8c9250c19cc122267263093eee7edd4e3fa75395dfda8c17a8e2"}, - {file = "numpy-1.21.6-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7c4068a8c44014b2d55f3c3f574c376b2494ca9cc73d2f1bd692382b6dffe3db"}, - {file = "numpy-1.21.6-cp37-cp37m-win32.whl", hash = "sha256:7c7e5fa88d9ff656e067876e4736379cc962d185d5cd808014a8a928d529ef4e"}, - {file = "numpy-1.21.6-cp37-cp37m-win_amd64.whl", hash = "sha256:bcb238c9c96c00d3085b264e5c1a1207672577b93fa666c3b14a45240b14123a"}, - {file = "numpy-1.21.6-cp38-cp38-macosx_10_9_universal2.whl", hash = "sha256:82691fda7c3f77c90e62da69ae60b5ac08e87e775b09813559f8901a88266552"}, - {file = "numpy-1.21.6-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:643843bcc1c50526b3a71cd2ee561cf0d8773f062c8cbaf9ffac9fdf573f83ab"}, - {file = "numpy-1.21.6-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:357768c2e4451ac241465157a3e929b265dfac85d9214074985b1786244f2ef3"}, - {file = "numpy-1.21.6-cp38-cp38-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:9f411b2c3f3d76bba0865b35a425157c5dcf54937f82bbeb3d3c180789dd66a6"}, - {file = "numpy-1.21.6-cp38-cp38-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4aa48afdce4660b0076a00d80afa54e8a97cd49f457d68a4342d188a09451c1a"}, - {file = "numpy-1.21.6-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:d6a96eef20f639e6a97d23e57dd0c1b1069a7b4fd7027482a4c5c451cd7732f4"}, - {file = "numpy-1.21.6-cp38-cp38-win32.whl", hash = "sha256:5c3c8def4230e1b959671eb959083661b4a0d2e9af93ee339c7dada6759a9470"}, - {file = "numpy-1.21.6-cp38-cp38-win_amd64.whl", hash = "sha256:bf2ec4b75d0e9356edea834d1de42b31fe11f726a81dfb2c2112bc1eaa508fcf"}, - {file = "numpy-1.21.6-cp39-cp39-macosx_10_9_universal2.whl", hash = "sha256:4391bd07606be175aafd267ef9bea87cf1b8210c787666ce82073b05f202add1"}, - {file = "numpy-1.21.6-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:67f21981ba2f9d7ba9ade60c9e8cbaa8cf8e9ae51673934480e45cf55e953673"}, - {file = "numpy-1.21.6-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:ee5ec40fdd06d62fe5d4084bef4fd50fd4bb6bfd2bf519365f569dc470163ab0"}, - {file = "numpy-1.21.6-cp39-cp39-manylinux_2_12_i686.manylinux2010_i686.whl", hash = "sha256:1dbe1c91269f880e364526649a52eff93ac30035507ae980d2fed33aaee633ac"}, - {file = "numpy-1.21.6-cp39-cp39-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:d9caa9d5e682102453d96a0ee10c7241b72859b01a941a397fd965f23b3e016b"}, - {file = "numpy-1.21.6-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:58459d3bad03343ac4b1b42ed14d571b8743dc80ccbf27444f266729df1d6f5b"}, - {file = "numpy-1.21.6-cp39-cp39-win32.whl", hash = "sha256:7f5ae4f304257569ef3b948810816bc87c9146e8c446053539947eedeaa32786"}, - {file = "numpy-1.21.6-cp39-cp39-win_amd64.whl", hash = "sha256:e31f0bb5928b793169b87e3d1e070f2342b22d5245c755e2b81caa29756246c3"}, - {file = "numpy-1.21.6-pp37-pypy37_pp73-manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:dd1c8f6bd65d07d3810b90d02eba7997e32abbdf1277a481d698969e921a3be0"}, - {file = "numpy-1.21.6.zip", hash = "sha256:ecb55251139706669fdec2ff073c98ef8e9a84473e51e716211b41aa0f18e656"}, - {file = "numpy-1.24.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:3c1104d3c036fb81ab923f507536daedc718d0ad5a8707c6061cdfd6d184e570"}, - {file = "numpy-1.24.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:202de8f38fc4a45a3eea4b63e2f376e5f2dc64ef0fa692838e31a808520efaf7"}, - {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:8535303847b89aa6b0f00aa1dc62867b5a32923e4d1681a35b5eef2d9591a463"}, - {file = "numpy-1.24.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2d926b52ba1367f9acb76b0df6ed21f0b16a1ad87c6720a1121674e5cf63e2b6"}, - {file = "numpy-1.24.3-cp310-cp310-win32.whl", hash = "sha256:f21c442fdd2805e91799fbe044a7b999b8571bb0ab0f7850d0cb9641a687092b"}, - {file = "numpy-1.24.3-cp310-cp310-win_amd64.whl", hash = "sha256:ab5f23af8c16022663a652d3b25dcdc272ac3f83c3af4c02eb8b824e6b3ab9d7"}, - {file = "numpy-1.24.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:9a7721ec204d3a237225db3e194c25268faf92e19338a35f3a224469cb6039a3"}, - {file = "numpy-1.24.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:d6cc757de514c00b24ae8cf5c876af2a7c3df189028d68c0cb4eaa9cd5afc2bf"}, - {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76e3f4e85fc5d4fd311f6e9b794d0c00e7002ec122be271f2019d63376f1d385"}, - {file = "numpy-1.24.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1d3c026f57ceaad42f8231305d4653d5f05dc6332a730ae5c0bea3513de0950"}, - {file = "numpy-1.24.3-cp311-cp311-win32.whl", hash = "sha256:c91c4afd8abc3908e00a44b2672718905b8611503f7ff87390cc0ac3423fb096"}, - {file = "numpy-1.24.3-cp311-cp311-win_amd64.whl", hash = "sha256:5342cf6aad47943286afa6f1609cad9b4266a05e7f2ec408e2cf7aea7ff69d80"}, - {file = "numpy-1.24.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:7776ea65423ca6a15255ba1872d82d207bd1e09f6d0894ee4a64678dd2204078"}, - {file = "numpy-1.24.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:ae8d0be48d1b6ed82588934aaaa179875e7dc4f3d84da18d7eae6eb3f06c242c"}, - {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ecde0f8adef7dfdec993fd54b0f78183051b6580f606111a6d789cd14c61ea0c"}, - {file = "numpy-1.24.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:4749e053a29364d3452c034827102ee100986903263e89884922ef01a0a6fd2f"}, - {file = "numpy-1.24.3-cp38-cp38-win32.whl", hash = "sha256:d933fabd8f6a319e8530d0de4fcc2e6a61917e0b0c271fded460032db42a0fe4"}, - {file = "numpy-1.24.3-cp38-cp38-win_amd64.whl", hash = "sha256:56e48aec79ae238f6e4395886b5eaed058abb7231fb3361ddd7bfdf4eed54289"}, - {file = "numpy-1.24.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4719d5aefb5189f50887773699eaf94e7d1e02bf36c1a9d353d9f46703758ca4"}, - {file = "numpy-1.24.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0ec87a7084caa559c36e0a2309e4ecb1baa03b687201d0a847c8b0ed476a7187"}, - {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ea8282b9bcfe2b5e7d491d0bf7f3e2da29700cec05b49e64d6246923329f2b02"}, - {file = "numpy-1.24.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:210461d87fb02a84ef243cac5e814aad2b7f4be953b32cb53327bb49fd77fbb4"}, - {file = "numpy-1.24.3-cp39-cp39-win32.whl", hash = "sha256:784c6da1a07818491b0ffd63c6bbe5a33deaa0e25a20e1b3ea20cf0e43f8046c"}, - {file = "numpy-1.24.3-cp39-cp39-win_amd64.whl", hash = "sha256:d5036197ecae68d7f491fcdb4df90082b0d4960ca6599ba2659957aafced7c17"}, - {file = "numpy-1.24.3-pp38-pypy38_pp73-macosx_10_9_x86_64.whl", hash = "sha256:352ee00c7f8387b44d19f4cada524586f07379c0d49270f87233983bc5087ca0"}, - {file = "numpy-1.24.3-pp38-pypy38_pp73-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1a7d6acc2e7524c9955e5c903160aa4ea083736fde7e91276b0e5d98e6332812"}, - {file = "numpy-1.24.3-pp38-pypy38_pp73-win_amd64.whl", hash = "sha256:35400e6a8d102fd07c71ed7dcadd9eb62ee9a6e84ec159bd48c28235bbb0f8e4"}, - {file = "numpy-1.24.3.tar.gz", hash = "sha256:ab344f1bf21f140adab8e47fdbc7c35a477dc01408791f8ba00d018dd0bc5155"}, -] -oauthlib = [ - {file = "oauthlib-3.2.2-py3-none-any.whl", hash = "sha256:8139f29aac13e25d502680e9e19963e83f16838d48a0d71c287fe40e7067fbca"}, - {file = "oauthlib-3.2.2.tar.gz", hash = "sha256:9859c40929662bec5d64f34d01c99e093149682a3f38915dc0655d5a633dd918"}, -] -openpyxl = [ - {file = "openpyxl-3.1.2-py2.py3-none-any.whl", hash = "sha256:f91456ead12ab3c6c2e9491cf33ba6d08357d802192379bb482f1033ade496f5"}, - {file = "openpyxl-3.1.2.tar.gz", hash = "sha256:a6f5977418eff3b2d5500d54d9db50c8277a368436f4e4f8ddb1be3422870184"}, -] -packaging = [ - {file = "packaging-23.1-py3-none-any.whl", hash = "sha256:994793af429502c4ea2ebf6bf664629d07c1a9fe974af92966e4b8d2df7edc61"}, - {file = "packaging-23.1.tar.gz", hash = "sha256:a392980d2b6cffa644431898be54b0045151319d1e7ec34f0cfed48767dd334f"}, -] -pandas = [ - {file = "pandas-1.3.5-cp310-cp310-macosx_10_9_universal2.whl", hash = "sha256:62d5b5ce965bae78f12c1c0df0d387899dd4211ec0bdc52822373f13a3a022b9"}, - {file = "pandas-1.3.5-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:adfeb11be2d54f275142c8ba9bf67acee771b7186a5745249c7d5a06c670136b"}, - {file = "pandas-1.3.5-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:60a8c055d58873ad81cae290d974d13dd479b82cbb975c3e1fa2cf1920715296"}, - {file = "pandas-1.3.5-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fd541ab09e1f80a2a1760032d665f6e032d8e44055d602d65eeea6e6e85498cb"}, - {file = "pandas-1.3.5-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2651d75b9a167cc8cc572cf787ab512d16e316ae00ba81874b560586fa1325e0"}, - {file = "pandas-1.3.5-cp310-cp310-win_amd64.whl", hash = "sha256:aaf183a615ad790801fa3cf2fa450e5b6d23a54684fe386f7e3208f8b9bfbef6"}, - {file = "pandas-1.3.5-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:344295811e67f8200de2390093aeb3c8309f5648951b684d8db7eee7d1c81fb7"}, - {file = "pandas-1.3.5-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:552020bf83b7f9033b57cbae65589c01e7ef1544416122da0c79140c93288f56"}, - {file = "pandas-1.3.5-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5cce0c6bbeb266b0e39e35176ee615ce3585233092f685b6a82362523e59e5b4"}, - {file = "pandas-1.3.5-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:7d28a3c65463fd0d0ba8bbb7696b23073efee0510783340a44b08f5e96ffce0c"}, - {file = "pandas-1.3.5-cp37-cp37m-win32.whl", hash = "sha256:a62949c626dd0ef7de11de34b44c6475db76995c2064e2d99c6498c3dba7fe58"}, - {file = "pandas-1.3.5-cp37-cp37m-win_amd64.whl", hash = "sha256:8025750767e138320b15ca16d70d5cdc1886e8f9cc56652d89735c016cd8aea6"}, - {file = "pandas-1.3.5-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:fe95bae4e2d579812865db2212bb733144e34d0c6785c0685329e5b60fcb85dd"}, - {file = "pandas-1.3.5-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5f261553a1e9c65b7a310302b9dbac31cf0049a51695c14ebe04e4bfd4a96f02"}, - {file = "pandas-1.3.5-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8b6dbec5f3e6d5dc80dcfee250e0a2a652b3f28663492f7dab9a24416a48ac39"}, - {file = "pandas-1.3.5-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d3bc49af96cd6285030a64779de5b3688633a07eb75c124b0747134a63f4c05f"}, - {file = "pandas-1.3.5-cp38-cp38-win32.whl", hash = "sha256:b6b87b2fb39e6383ca28e2829cddef1d9fc9e27e55ad91ca9c435572cdba51bf"}, - {file = "pandas-1.3.5-cp38-cp38-win_amd64.whl", hash = "sha256:a395692046fd8ce1edb4c6295c35184ae0c2bbe787ecbe384251da609e27edcb"}, - {file = "pandas-1.3.5-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:bd971a3f08b745a75a86c00b97f3007c2ea175951286cdda6abe543e687e5f2f"}, - {file = "pandas-1.3.5-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:37f06b59e5bc05711a518aa10beaec10942188dccb48918bb5ae602ccbc9f1a0"}, - {file = "pandas-1.3.5-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2c21778a688d3712d35710501f8001cdbf96eb70a7c587a3d5613573299fdca6"}, - {file = "pandas-1.3.5-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3345343206546545bc26a05b4602b6a24385b5ec7c75cb6059599e3d56831da2"}, - {file = "pandas-1.3.5-cp39-cp39-win32.whl", hash = "sha256:c69406a2808ba6cf580c2255bcf260b3f214d2664a3a4197d0e640f573b46fd3"}, - {file = "pandas-1.3.5-cp39-cp39-win_amd64.whl", hash = "sha256:32e1a26d5ade11b547721a72f9bfc4bd113396947606e00d5b4a5b79b3dcb006"}, - {file = "pandas-1.3.5.tar.gz", hash = "sha256:1e4285f5de1012de20ca46b188ccf33521bff61ba5c5ebd78b4fb28e5416a9f1"}, -] -pathspec = [ - {file = "pathspec-0.11.1-py3-none-any.whl", hash = "sha256:d8af70af76652554bd134c22b3e8a1cc46ed7d91edcdd721ef1a0c51a84a5293"}, - {file = "pathspec-0.11.1.tar.gz", hash = "sha256:2798de800fa92780e33acca925945e9a19a133b715067cf165b8866c15a31687"}, -] -platformdirs = [ - {file = "platformdirs-3.5.0-py3-none-any.whl", hash = "sha256:47692bc24c1958e8b0f13dd727307cff1db103fca36399f457da8e05f222fdc4"}, - {file = "platformdirs-3.5.0.tar.gz", hash = "sha256:7954a68d0ba23558d753f73437c55f89027cf8f5108c19844d4b82e5af396335"}, -] -pluggy = [ - {file = "pluggy-1.0.0-py2.py3-none-any.whl", hash = "sha256:74134bbf457f031a36d68416e1509f34bd5ccc019f0bcc952c7b909d06b37bd3"}, - {file = "pluggy-1.0.0.tar.gz", hash = "sha256:4224373bacce55f955a878bf9cfa763c1e360858e330072059e10bad68531159"}, -] -pyarrow = [ - {file = "pyarrow-12.0.0-cp310-cp310-macosx_10_14_x86_64.whl", hash = "sha256:3b97649c8a9a09e1d8dc76513054f1331bd9ece78ee39365e6bf6bc7503c1e94"}, - {file = "pyarrow-12.0.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:bc4ea634dacb03936f50fcf59574a8e727f90c17c24527e488d8ceb52ae284de"}, - {file = "pyarrow-12.0.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:1d568acfca3faa565d663e53ee34173be8e23a95f78f2abfdad198010ec8f745"}, - {file = "pyarrow-12.0.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1b50bb9a82dca38a002d7cbd802a16b1af0f8c50ed2ec94a319f5f2afc047ee9"}, - {file = "pyarrow-12.0.0-cp310-cp310-win_amd64.whl", hash = "sha256:3d1733b1ea086b3c101427d0e57e2be3eb964686e83c2363862a887bb5c41fa8"}, - {file = "pyarrow-12.0.0-cp311-cp311-macosx_10_14_x86_64.whl", hash = "sha256:a7cd32fe77f967fe08228bc100433273020e58dd6caced12627bcc0a7675a513"}, - {file = "pyarrow-12.0.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:92fb031e6777847f5c9b01eaa5aa0c9033e853ee80117dce895f116d8b0c3ca3"}, - {file = "pyarrow-12.0.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:280289ebfd4ac3570f6b776515baa01e4dcbf17122c401e4b7170a27c4be63fd"}, - {file = "pyarrow-12.0.0-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:272f147d4f8387bec95f17bb58dcfc7bc7278bb93e01cb7b08a0e93a8921e18e"}, - {file = "pyarrow-12.0.0-cp311-cp311-win_amd64.whl", hash = "sha256:0846ace49998825eda4722f8d7f83fa05601c832549c9087ea49d6d5397d8cec"}, - {file = "pyarrow-12.0.0-cp37-cp37m-macosx_10_14_x86_64.whl", hash = "sha256:993287136369aca60005ee7d64130f9466489c4f7425f5c284315b0a5401ccd9"}, - {file = "pyarrow-12.0.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:7a7b6a765ee4f88efd7d8348d9a1f804487d60799d0428b6ddf3344eaef37282"}, - {file = "pyarrow-12.0.0-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a1c4fce253d5bdc8d62f11cfa3da5b0b34b562c04ce84abb8bd7447e63c2b327"}, - {file = "pyarrow-12.0.0-cp37-cp37m-win_amd64.whl", hash = "sha256:e6be4d85707fc8e7a221c8ab86a40449ce62559ce25c94321df7c8500245888f"}, - {file = "pyarrow-12.0.0-cp38-cp38-macosx_10_14_x86_64.whl", hash = "sha256:ea830d9f66bfb82d30b5794642f83dd0e4a718846462d22328981e9eb149cba8"}, - {file = "pyarrow-12.0.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7b5b9f60d9ef756db59bec8d90e4576b7df57861e6a3d6a8bf99538f68ca15b3"}, - {file = "pyarrow-12.0.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:b99e559d27db36ad3a33868a475f03e3129430fc065accc839ef4daa12c6dab6"}, - {file = "pyarrow-12.0.0-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:5b0810864a593b89877120972d1f7af1d1c9389876dbed92b962ed81492d3ffc"}, - {file = "pyarrow-12.0.0-cp38-cp38-win_amd64.whl", hash = "sha256:23a77d97f4d101ddfe81b9c2ee03a177f0e590a7e68af15eafa06e8f3cf05976"}, - {file = "pyarrow-12.0.0-cp39-cp39-macosx_10_14_x86_64.whl", hash = "sha256:2cc63e746221cddb9001f7281dee95fd658085dd5b717b076950e1ccc607059c"}, - {file = "pyarrow-12.0.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:d8c26912607e26c2991826bbaf3cf2b9c8c3e17566598c193b492f058b40d3a4"}, - {file = "pyarrow-12.0.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:0d8b90efc290e99a81d06015f3a46601c259ecc81ffb6d8ce288c91bd1b868c9"}, - {file = "pyarrow-12.0.0-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2466be046b81863be24db370dffd30a2e7894b4f9823fb60ef0a733c31ac6256"}, - {file = "pyarrow-12.0.0-cp39-cp39-win_amd64.whl", hash = "sha256:0e36425b1c1cbf5447718b3f1751bf86c58f2b3ad299f996cd9b1aa040967656"}, - {file = "pyarrow-12.0.0.tar.gz", hash = "sha256:19c812d303610ab5d664b7b1de4051ae23565f9f94d04cbea9e50569746ae1ee"}, -] -pylint = [ - {file = "pylint-2.13.9-py3-none-any.whl", hash = "sha256:705c620d388035bdd9ff8b44c5bcdd235bfb49d276d488dd2c8ff1736aa42526"}, - {file = "pylint-2.13.9.tar.gz", hash = "sha256:095567c96e19e6f57b5b907e67d265ff535e588fe26b12b5ebe1fc5645b2c731"}, -] -pytest = [ - {file = "pytest-7.3.1-py3-none-any.whl", hash = "sha256:3799fa815351fea3a5e96ac7e503a96fa51cc9942c3753cda7651b93c1cfa362"}, - {file = "pytest-7.3.1.tar.gz", hash = "sha256:434afafd78b1d78ed0addf160ad2b77a30d35d4bdf8af234fe621919d9ed15e3"}, -] -python-dateutil = [ - {file = "python-dateutil-2.8.2.tar.gz", hash = "sha256:0123cacc1627ae19ddf3c27a5de5bd67ee4586fbdd6440d9748f8abb483d3e86"}, - {file = "python_dateutil-2.8.2-py2.py3-none-any.whl", hash = "sha256:961d03dc3453ebbc59dbdea9e4e11c5651520a876d0f4db161e8674aae935da9"}, -] -pytz = [ - {file = "pytz-2023.3-py2.py3-none-any.whl", hash = "sha256:a151b3abb88eda1d4e34a9814df37de2a80e301e68ba0fd856fb9b46bfbbbffb"}, - {file = "pytz-2023.3.tar.gz", hash = "sha256:1d8ce29db189191fb55338ee6d0387d82ab59f3d00eac103412d64e0ebd0c588"}, -] -requests = [ - {file = "requests-2.30.0-py3-none-any.whl", hash = "sha256:10e94cc4f3121ee6da529d358cdaeaff2f1c409cd377dbc72b825852f2f7e294"}, - {file = "requests-2.30.0.tar.gz", hash = "sha256:239d7d4458afcb28a692cdd298d87542235f4ca8d36d03a15bfc128a6559a2f4"}, -] -setuptools = [ - {file = "setuptools-67.7.2-py3-none-any.whl", hash = "sha256:23aaf86b85ca52ceb801d32703f12d77517b2556af839621c641fca11287952b"}, - {file = "setuptools-67.7.2.tar.gz", hash = "sha256:f104fa03692a2602fa0fec6c6a9e63b6c8a968de13e17c026957dd1f53d80990"}, -] -six = [ - {file = "six-1.16.0-py2.py3-none-any.whl", hash = "sha256:8abb2f1d86890a2dfb989f9a77cfcfd3e47c2a354b01111771326f8aa26e0254"}, - {file = "six-1.16.0.tar.gz", hash = "sha256:1e61c37477a1626458e36f7b1d82aa5c9b094fa4802892072e49de9c60c4c926"}, -] -sqlalchemy = [ - {file = "SQLAlchemy-1.4.48-cp27-cp27m-macosx_10_14_x86_64.whl", hash = "sha256:4bac3aa3c3d8bc7408097e6fe8bf983caa6e9491c5d2e2488cfcfd8106f13b6a"}, - {file = "SQLAlchemy-1.4.48-cp27-cp27m-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:dbcae0e528d755f4522cad5842f0942e54b578d79f21a692c44d91352ea6d64e"}, - {file = "SQLAlchemy-1.4.48-cp27-cp27m-win32.whl", hash = "sha256:cbbe8b8bffb199b225d2fe3804421b7b43a0d49983f81dc654d0431d2f855543"}, - {file = "SQLAlchemy-1.4.48-cp27-cp27m-win_amd64.whl", hash = "sha256:627e04a5d54bd50628fc8734d5fc6df2a1aa5962f219c44aad50b00a6cdcf965"}, - {file = "SQLAlchemy-1.4.48-cp27-cp27mu-manylinux_2_5_x86_64.manylinux1_x86_64.whl", hash = "sha256:9af1db7a287ef86e0f5cd990b38da6bd9328de739d17e8864f1817710da2d217"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-macosx_11_0_x86_64.whl", hash = "sha256:ce7915eecc9c14a93b73f4e1c9d779ca43e955b43ddf1e21df154184f39748e5"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:5381ddd09a99638f429f4cbe1b71b025bed318f6a7b23e11d65f3eed5e181c33"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:87609f6d4e81a941a17e61a4c19fee57f795e96f834c4f0a30cee725fc3f81d9"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fb0808ad34167f394fea21bd4587fc62f3bd81bba232a1e7fbdfa17e6cfa7cd7"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-win32.whl", hash = "sha256:d53cd8bc582da5c1c8c86b6acc4ef42e20985c57d0ebc906445989df566c5603"}, - {file = "SQLAlchemy-1.4.48-cp310-cp310-win_amd64.whl", hash = "sha256:4355e5915844afdc5cf22ec29fba1010166e35dd94a21305f49020022167556b"}, - {file = "SQLAlchemy-1.4.48-cp311-cp311-macosx_10_9_universal2.whl", hash = "sha256:066c2b0413e8cb980e6d46bf9d35ca83be81c20af688fedaef01450b06e4aa5e"}, - {file = "SQLAlchemy-1.4.48-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:c99bf13e07140601d111a7c6f1fc1519914dd4e5228315bbda255e08412f61a4"}, - {file = "SQLAlchemy-1.4.48-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2ee26276f12614d47cc07bc85490a70f559cba965fb178b1c45d46ffa8d73fda"}, - {file = "SQLAlchemy-1.4.48-cp311-cp311-win32.whl", hash = "sha256:49c312bcff4728bffc6fb5e5318b8020ed5c8b958a06800f91859fe9633ca20e"}, - {file = "SQLAlchemy-1.4.48-cp311-cp311-win_amd64.whl", hash = "sha256:cef2e2abc06eab187a533ec3e1067a71d7bbec69e582401afdf6d8cad4ba3515"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-macosx_10_14_x86_64.whl", hash = "sha256:3509159e050bd6d24189ec7af373359f07aed690db91909c131e5068176c5a5d"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:2fc2ab4d9f6d9218a5caa4121bdcf1125303482a1cdcfcdbd8567be8518969c0"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e1ddbbcef9bcedaa370c03771ebec7e39e3944782bef49e69430383c376a250b"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:6f82d8efea1ca92b24f51d3aea1a82897ed2409868a0af04247c8c1e4fef5890"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-win32.whl", hash = "sha256:e3e98d4907805b07743b583a99ecc58bf8807ecb6985576d82d5e8ae103b5272"}, - {file = "SQLAlchemy-1.4.48-cp36-cp36m-win_amd64.whl", hash = "sha256:25887b4f716e085a1c5162f130b852f84e18d2633942c8ca40dfb8519367c14f"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-macosx_10_15_x86_64.whl", hash = "sha256:0817c181271b0ce5df1aa20949f0a9e2426830fed5ecdcc8db449618f12c2730"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:fe1dd2562313dd9fe1778ed56739ad5d9aae10f9f43d9f4cf81d65b0c85168bb"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:68413aead943883b341b2b77acd7a7fe2377c34d82e64d1840860247cec7ff7c"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:fbde5642104ac6e95f96e8ad6d18d9382aa20672008cf26068fe36f3004491df"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-win32.whl", hash = "sha256:11c6b1de720f816c22d6ad3bbfa2f026f89c7b78a5c4ffafb220e0183956a92a"}, - {file = "SQLAlchemy-1.4.48-cp37-cp37m-win_amd64.whl", hash = "sha256:eb5464ee8d4bb6549d368b578e9529d3c43265007193597ddca71c1bae6174e6"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-macosx_10_15_x86_64.whl", hash = "sha256:92e6133cf337c42bfee03ca08c62ba0f2d9695618c8abc14a564f47503157be9"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:44d29a3fc6d9c45962476b470a81983dd8add6ad26fdbfae6d463b509d5adcda"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:005e942b451cad5285015481ae4e557ff4154dde327840ba91b9ac379be3b6ce"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9c8cfe951ed074ba5e708ed29c45397a95c4143255b0d022c7c8331a75ae61f3"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-win32.whl", hash = "sha256:2b9af65cc58726129d8414fc1a1a650dcdd594ba12e9c97909f1f57d48e393d3"}, - {file = "SQLAlchemy-1.4.48-cp38-cp38-win_amd64.whl", hash = "sha256:2b562e9d1e59be7833edf28b0968f156683d57cabd2137d8121806f38a9d58f4"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-macosx_11_0_x86_64.whl", hash = "sha256:a1fc046756cf2a37d7277c93278566ddf8be135c6a58397b4c940abf837011f4"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:9d9b55252d2ca42a09bcd10a697fa041e696def9dfab0b78c0aaea1485551a08"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6dab89874e72a9ab5462997846d4c760cdb957958be27b03b49cf0de5e5c327c"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:1fd8b5ee5a3acc4371f820934b36f8109ce604ee73cc668c724abb054cebcb6e"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-win32.whl", hash = "sha256:eee09350fd538e29cfe3a496ec6f148504d2da40dbf52adefb0d2f8e4d38ccc4"}, - {file = "SQLAlchemy-1.4.48-cp39-cp39-win_amd64.whl", hash = "sha256:7ad2b0f6520ed5038e795cc2852eb5c1f20fa6831d73301ced4aafbe3a10e1f6"}, - {file = "SQLAlchemy-1.4.48.tar.gz", hash = "sha256:b47bc287096d989a0838ce96f7d8e966914a24da877ed41a7531d44b55cdb8df"}, -] -thrift = [ - {file = "thrift-0.16.0.tar.gz", hash = "sha256:2b5b6488fcded21f9d312aa23c9ff6a0195d0f6ae26ddbd5ad9e3e25dfc14408"}, -] -tomli = [ - {file = "tomli-2.0.1-py3-none-any.whl", hash = "sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc"}, - {file = "tomli-2.0.1.tar.gz", hash = "sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f"}, -] -typed-ast = [ - {file = "typed_ast-1.5.4-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:669dd0c4167f6f2cd9f57041e03c3c2ebf9063d0757dc89f79ba1daa2bfca9d4"}, - {file = "typed_ast-1.5.4-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:211260621ab1cd7324e0798d6be953d00b74e0428382991adfddb352252f1d62"}, - {file = "typed_ast-1.5.4-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:267e3f78697a6c00c689c03db4876dd1efdfea2f251a5ad6555e82a26847b4ac"}, - {file = "typed_ast-1.5.4-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:c542eeda69212fa10a7ada75e668876fdec5f856cd3d06829e6aa64ad17c8dfe"}, - {file = "typed_ast-1.5.4-cp310-cp310-win_amd64.whl", hash = "sha256:a9916d2bb8865f973824fb47436fa45e1ebf2efd920f2b9f99342cb7fab93f72"}, - {file = "typed_ast-1.5.4-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:79b1e0869db7c830ba6a981d58711c88b6677506e648496b1f64ac7d15633aec"}, - {file = "typed_ast-1.5.4-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a94d55d142c9265f4ea46fab70977a1944ecae359ae867397757d836ea5a3f47"}, - {file = "typed_ast-1.5.4-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:183afdf0ec5b1b211724dfef3d2cad2d767cbefac291f24d69b00546c1837fb6"}, - {file = "typed_ast-1.5.4-cp36-cp36m-win_amd64.whl", hash = "sha256:639c5f0b21776605dd6c9dbe592d5228f021404dafd377e2b7ac046b0349b1a1"}, - {file = "typed_ast-1.5.4-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:cf4afcfac006ece570e32d6fa90ab74a17245b83dfd6655a6f68568098345ff6"}, - {file = "typed_ast-1.5.4-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ed855bbe3eb3715fca349c80174cfcfd699c2f9de574d40527b8429acae23a66"}, - {file = "typed_ast-1.5.4-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:6778e1b2f81dfc7bc58e4b259363b83d2e509a65198e85d5700dfae4c6c8ff1c"}, - {file = "typed_ast-1.5.4-cp37-cp37m-win_amd64.whl", hash = "sha256:0261195c2062caf107831e92a76764c81227dae162c4f75192c0d489faf751a2"}, - {file = "typed_ast-1.5.4-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:2efae9db7a8c05ad5547d522e7dbe62c83d838d3906a3716d1478b6c1d61388d"}, - {file = "typed_ast-1.5.4-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:7d5d014b7daa8b0bf2eaef684295acae12b036d79f54178b92a2b6a56f92278f"}, - {file = "typed_ast-1.5.4-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:370788a63915e82fd6f212865a596a0fefcbb7d408bbbb13dea723d971ed8bdc"}, - {file = "typed_ast-1.5.4-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4e964b4ff86550a7a7d56345c7864b18f403f5bd7380edf44a3c1fb4ee7ac6c6"}, - {file = "typed_ast-1.5.4-cp38-cp38-win_amd64.whl", hash = "sha256:683407d92dc953c8a7347119596f0b0e6c55eb98ebebd9b23437501b28dcbb8e"}, - {file = "typed_ast-1.5.4-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:4879da6c9b73443f97e731b617184a596ac1235fe91f98d279a7af36c796da35"}, - {file = "typed_ast-1.5.4-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:3e123d878ba170397916557d31c8f589951e353cc95fb7f24f6bb69adc1a8a97"}, - {file = "typed_ast-1.5.4-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:ebd9d7f80ccf7a82ac5f88c521115cc55d84e35bf8b446fcd7836eb6b98929a3"}, - {file = "typed_ast-1.5.4-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:98f80dee3c03455e92796b58b98ff6ca0b2a6f652120c263efdba4d6c5e58f72"}, - {file = "typed_ast-1.5.4-cp39-cp39-win_amd64.whl", hash = "sha256:0fdbcf2fef0ca421a3f5912555804296f0b0960f0418c440f5d6d3abb549f3e1"}, - {file = "typed_ast-1.5.4.tar.gz", hash = "sha256:39e21ceb7388e4bb37f4c679d72707ed46c2fbf2a5609b8b8ebc4b067d977df2"}, -] -typing-extensions = [ - {file = "typing_extensions-4.5.0-py3-none-any.whl", hash = "sha256:fb33085c39dd998ac16d1431ebc293a8b3eedd00fd4a32de0ff79002c19511b4"}, - {file = "typing_extensions-4.5.0.tar.gz", hash = "sha256:5cb5f4a79139d699607b3ef622a1dedafa84e115ab0024e0d9c044a9479ca7cb"}, -] -urllib3 = [ - {file = "urllib3-2.0.2-py3-none-any.whl", hash = "sha256:d055c2f9d38dc53c808f6fdc8eab7360b6fdbbde02340ed25cfbcd817c62469e"}, - {file = "urllib3-2.0.2.tar.gz", hash = "sha256:61717a1095d7e155cdb737ac7bb2f4324a858a1e2e6466f6d03ff630ca68d3cc"}, -] -wrapt = [ - {file = "wrapt-1.15.0-cp27-cp27m-macosx_10_9_x86_64.whl", hash = "sha256:ca1cccf838cd28d5a0883b342474c630ac48cac5df0ee6eacc9c7290f76b11c1"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_i686.whl", hash = "sha256:e826aadda3cae59295b95343db8f3d965fb31059da7de01ee8d1c40a60398b29"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux1_x86_64.whl", hash = "sha256:5fc8e02f5984a55d2c653f5fea93531e9836abbd84342c1d1e17abc4a15084c2"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_i686.whl", hash = "sha256:96e25c8603a155559231c19c0349245eeb4ac0096fe3c1d0be5c47e075bd4f46"}, - {file = "wrapt-1.15.0-cp27-cp27m-manylinux2010_x86_64.whl", hash = "sha256:40737a081d7497efea35ab9304b829b857f21558acfc7b3272f908d33b0d9d4c"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_i686.whl", hash = "sha256:f87ec75864c37c4c6cb908d282e1969e79763e0d9becdfe9fe5473b7bb1e5f09"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux1_x86_64.whl", hash = "sha256:1286eb30261894e4c70d124d44b7fd07825340869945c79d05bda53a40caa079"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_i686.whl", hash = "sha256:493d389a2b63c88ad56cdc35d0fa5752daac56ca755805b1b0c530f785767d5e"}, - {file = "wrapt-1.15.0-cp27-cp27mu-manylinux2010_x86_64.whl", hash = "sha256:58d7a75d731e8c63614222bcb21dd992b4ab01a399f1f09dd82af17bbfc2368a"}, - {file = "wrapt-1.15.0-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:21f6d9a0d5b3a207cdf7acf8e58d7d13d463e639f0c7e01d82cdb671e6cb7923"}, - {file = "wrapt-1.15.0-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:ce42618f67741d4697684e501ef02f29e758a123aa2d669e2d964ff734ee00ee"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:41d07d029dd4157ae27beab04d22b8e261eddfc6ecd64ff7000b10dc8b3a5727"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:54accd4b8bc202966bafafd16e69da9d5640ff92389d33d28555c5fd4f25ccb7"}, - {file = "wrapt-1.15.0-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:2fbfbca668dd15b744418265a9607baa970c347eefd0db6a518aaf0cfbd153c0"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:76e9c727a874b4856d11a32fb0b389afc61ce8aaf281ada613713ddeadd1cfec"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:e20076a211cd6f9b44a6be58f7eeafa7ab5720eb796975d0c03f05b47d89eb90"}, - {file = "wrapt-1.15.0-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:a74d56552ddbde46c246b5b89199cb3fd182f9c346c784e1a93e4dc3f5ec9975"}, - {file = "wrapt-1.15.0-cp310-cp310-win32.whl", hash = "sha256:26458da5653aa5b3d8dc8b24192f574a58984c749401f98fff994d41d3f08da1"}, - {file = "wrapt-1.15.0-cp310-cp310-win_amd64.whl", hash = "sha256:75760a47c06b5974aa5e01949bf7e66d2af4d08cb8c1d6516af5e39595397f5e"}, - {file = "wrapt-1.15.0-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:ba1711cda2d30634a7e452fc79eabcadaffedf241ff206db2ee93dd2c89a60e7"}, - {file = "wrapt-1.15.0-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:56374914b132c702aa9aa9959c550004b8847148f95e1b824772d453ac204a72"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:a89ce3fd220ff144bd9d54da333ec0de0399b52c9ac3d2ce34b569cf1a5748fb"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:3bbe623731d03b186b3d6b0d6f51865bf598587c38d6f7b0be2e27414f7f214e"}, - {file = "wrapt-1.15.0-cp311-cp311-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:3abbe948c3cbde2689370a262a8d04e32ec2dd4f27103669a45c6929bcdbfe7c"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:b67b819628e3b748fd3c2192c15fb951f549d0f47c0449af0764d7647302fda3"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:7eebcdbe3677e58dd4c0e03b4f2cfa346ed4049687d839adad68cc38bb559c92"}, - {file = "wrapt-1.15.0-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:74934ebd71950e3db69960a7da29204f89624dde411afbfb3b4858c1409b1e98"}, - {file = "wrapt-1.15.0-cp311-cp311-win32.whl", hash = "sha256:bd84395aab8e4d36263cd1b9308cd504f6cf713b7d6d3ce25ea55670baec5416"}, - {file = "wrapt-1.15.0-cp311-cp311-win_amd64.whl", hash = "sha256:a487f72a25904e2b4bbc0817ce7a8de94363bd7e79890510174da9d901c38705"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_i686.whl", hash = "sha256:4ff0d20f2e670800d3ed2b220d40984162089a6e2c9646fdb09b85e6f9a8fc29"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux1_x86_64.whl", hash = "sha256:9ed6aa0726b9b60911f4aed8ec5b8dd7bf3491476015819f56473ffaef8959bd"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_i686.whl", hash = "sha256:896689fddba4f23ef7c718279e42f8834041a21342d95e56922e1c10c0cc7afb"}, - {file = "wrapt-1.15.0-cp35-cp35m-manylinux2010_x86_64.whl", hash = "sha256:75669d77bb2c071333417617a235324a1618dba66f82a750362eccbe5b61d248"}, - {file = "wrapt-1.15.0-cp35-cp35m-win32.whl", hash = "sha256:fbec11614dba0424ca72f4e8ba3c420dba07b4a7c206c8c8e4e73f2e98f4c559"}, - {file = "wrapt-1.15.0-cp35-cp35m-win_amd64.whl", hash = "sha256:fd69666217b62fa5d7c6aa88e507493a34dec4fa20c5bd925e4bc12fce586639"}, - {file = "wrapt-1.15.0-cp36-cp36m-macosx_10_9_x86_64.whl", hash = "sha256:b0724f05c396b0a4c36a3226c31648385deb6a65d8992644c12a4963c70326ba"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:bbeccb1aa40ab88cd29e6c7d8585582c99548f55f9b2581dfc5ba68c59a85752"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:38adf7198f8f154502883242f9fe7333ab05a5b02de7d83aa2d88ea621f13364"}, - {file = "wrapt-1.15.0-cp36-cp36m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:578383d740457fa790fdf85e6d346fda1416a40549fe8db08e5e9bd281c6a475"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_aarch64.whl", hash = "sha256:a4cbb9ff5795cd66f0066bdf5947f170f5d63a9274f99bdbca02fd973adcf2a8"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_i686.whl", hash = "sha256:af5bd9ccb188f6a5fdda9f1f09d9f4c86cc8a539bd48a0bfdc97723970348418"}, - {file = "wrapt-1.15.0-cp36-cp36m-musllinux_1_1_x86_64.whl", hash = "sha256:b56d5519e470d3f2fe4aa7585f0632b060d532d0696c5bdfb5e8319e1d0f69a2"}, - {file = "wrapt-1.15.0-cp36-cp36m-win32.whl", hash = "sha256:77d4c1b881076c3ba173484dfa53d3582c1c8ff1f914c6461ab70c8428b796c1"}, - {file = "wrapt-1.15.0-cp36-cp36m-win_amd64.whl", hash = "sha256:077ff0d1f9d9e4ce6476c1a924a3332452c1406e59d90a2cf24aeb29eeac9420"}, - {file = "wrapt-1.15.0-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:5c5aa28df055697d7c37d2099a7bc09f559d5053c3349b1ad0c39000e611d317"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3a8564f283394634a7a7054b7983e47dbf39c07712d7b177b37e03f2467a024e"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:780c82a41dc493b62fc5884fb1d3a3b81106642c5c5c78d6a0d4cbe96d62ba7e"}, - {file = "wrapt-1.15.0-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:e169e957c33576f47e21864cf3fc9ff47c223a4ebca8960079b8bd36cb014fd0"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:b02f21c1e2074943312d03d243ac4388319f2456576b2c6023041c4d57cd7019"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:f2e69b3ed24544b0d3dbe2c5c0ba5153ce50dcebb576fdc4696d52aa22db6034"}, - {file = "wrapt-1.15.0-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:d787272ed958a05b2c86311d3a4135d3c2aeea4fc655705f074130aa57d71653"}, - {file = "wrapt-1.15.0-cp37-cp37m-win32.whl", hash = "sha256:02fce1852f755f44f95af51f69d22e45080102e9d00258053b79367d07af39c0"}, - {file = "wrapt-1.15.0-cp37-cp37m-win_amd64.whl", hash = "sha256:abd52a09d03adf9c763d706df707c343293d5d106aea53483e0ec8d9e310ad5e"}, - {file = "wrapt-1.15.0-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:cdb4f085756c96a3af04e6eca7f08b1345e94b53af8921b25c72f096e704e145"}, - {file = "wrapt-1.15.0-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:230ae493696a371f1dbffaad3dafbb742a4d27a0afd2b1aecebe52b740167e7f"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:63424c681923b9f3bfbc5e3205aafe790904053d42ddcc08542181a30a7a51bd"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d6bcbfc99f55655c3d93feb7ef3800bd5bbe963a755687cbf1f490a71fb7794b"}, - {file = "wrapt-1.15.0-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c99f4309f5145b93eca6e35ac1a988f0dc0a7ccf9ccdcd78d3c0adf57224e62f"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:b130fe77361d6771ecf5a219d8e0817d61b236b7d8b37cc045172e574ed219e6"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:96177eb5645b1c6985f5c11d03fc2dbda9ad24ec0f3a46dcce91445747e15094"}, - {file = "wrapt-1.15.0-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:d5fe3e099cf07d0fb5a1e23d399e5d4d1ca3e6dfcbe5c8570ccff3e9208274f7"}, - {file = "wrapt-1.15.0-cp38-cp38-win32.whl", hash = "sha256:abd8f36c99512755b8456047b7be10372fca271bf1467a1caa88db991e7c421b"}, - {file = "wrapt-1.15.0-cp38-cp38-win_amd64.whl", hash = "sha256:b06fa97478a5f478fb05e1980980a7cdf2712015493b44d0c87606c1513ed5b1"}, - {file = "wrapt-1.15.0-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2e51de54d4fb8fb50d6ee8327f9828306a959ae394d3e01a1ba8b2f937747d86"}, - {file = "wrapt-1.15.0-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:0970ddb69bba00670e58955f8019bec4a42d1785db3faa043c33d81de2bf843c"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76407ab327158c510f44ded207e2f76b657303e17cb7a572ffe2f5a8a48aa04d"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd525e0e52a5ff16653a3fc9e3dd827981917d34996600bbc34c05d048ca35cc"}, - {file = "wrapt-1.15.0-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:9d37ac69edc5614b90516807de32d08cb8e7b12260a285ee330955604ed9dd29"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:078e2a1a86544e644a68422f881c48b84fef6d18f8c7a957ffd3f2e0a74a0d4a"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:2cf56d0e237280baed46f0b5316661da892565ff58309d4d2ed7dba763d984b8"}, - {file = "wrapt-1.15.0-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:7dc0713bf81287a00516ef43137273b23ee414fe41a3c14be10dd95ed98a2df9"}, - {file = "wrapt-1.15.0-cp39-cp39-win32.whl", hash = "sha256:46ed616d5fb42f98630ed70c3529541408166c22cdfd4540b88d5f21006b0eff"}, - {file = "wrapt-1.15.0-cp39-cp39-win_amd64.whl", hash = "sha256:eef4d64c650f33347c1f9266fa5ae001440b232ad9b98f1f43dfe7a79435c0a6"}, - {file = "wrapt-1.15.0-py3-none-any.whl", hash = "sha256:64b1df0f83706b4ef4cfb4fb0e4c2669100fd7ecacfb59e091fad300d4e04640"}, - {file = "wrapt-1.15.0.tar.gz", hash = "sha256:d06730c6aed78cee4126234cf2d071e01b44b915e725a6cb439a879ec9754a3a"}, -] -zipp = [ - {file = "zipp-3.15.0-py3-none-any.whl", hash = "sha256:48904fc76a60e542af151aded95726c1a5c34ed43ab4134b597665c86d7ad556"}, - {file = "zipp-3.15.0.tar.gz", hash = "sha256:112929ad649da941c23de50f356a2b5570c954b65150642bccdd66bf194d224b"}, -] +lock-version = "2.0" +python-versions = "^3.8.0" +content-hash = "31066a85f646d0009d6fe9ffc833a64fcb4b6923c2e7f2652e7aa8540acba298" diff --git a/pyproject.toml b/pyproject.toml index e93dcd1b..69d0c274 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,37 +1,43 @@ [tool.poetry] name = "databricks-sql-connector" -version = "2.5.2" +version = "3.4.0" description = "Databricks SQL Connector for Python" authors = ["Databricks "] license = "Apache-2.0" readme = "README.md" -packages = [{include = "databricks", from = "src"}] +packages = [{ include = "databricks", from = "src" }] include = ["CHANGELOG.md"] [tool.poetry.dependencies] -python = "^3.7.1" -thrift = "^0.16.0" -pandas = "^1.2.5" -pyarrow = [ - {version = ">=6.0.0", python = ">=3.7,<3.11"}, - {version = ">=10.0.1", python = ">=3.11"} +python = "^3.8.0" +thrift = ">=0.16.0,<0.21.0" +pandas = [ + { version = ">=1.2.5,<2.3.0", python = ">=3.8" } ] +pyarrow = ">=14.0.1,<17" + lz4 = "^4.0.2" -requests="^2.18.1" -oauthlib="^3.1.0" +requests = "^2.18.1" +oauthlib = "^3.1.0" numpy = [ - {version = ">=1.16.6", python = ">=3.7,<3.11"}, - {version = ">=1.23.4", python = ">=3.11"} + { version = "^1.16.6", python = ">=3.8,<3.11" }, + { version = "^1.23.4", python = ">=3.11" }, ] -sqlalchemy = "^1.3.24" +sqlalchemy = { version = ">=2.0.21", optional = true } openpyxl = "^3.0.10" -alembic = "^1.0.11" +alembic = { version = "^1.0.11", optional = true } +urllib3 = ">=1.26" + +[tool.poetry.extras] +sqlalchemy = ["sqlalchemy"] +alembic = ["sqlalchemy", "alembic"] [tool.poetry.dev-dependencies] pytest = "^7.1.2" -mypy = "^0.950" +mypy = "^1.10.1" pylint = ">=2.12.0" black = "^22.3.0" +pytest-dotenv = "^0.5.2" [tool.poetry.urls] "Homepage" = "https://github.com/databricks/databricks-sql-python" @@ -50,3 +56,11 @@ exclude = ['ttypes\.py$', 'TCLIService\.py$'] [tool.black] exclude = '/(\.eggs|\.git|\.hg|\.mypy_cache|\.nox|\.tox|\.venv|\.svn|_build|buck-out|build|dist|thrift_api)/' + +[tool.pytest.ini_options] +markers = {"reviewed" = "Test case has been reviewed by Databricks"} +minversion = "6.0" +log_cli = "false" +log_cli_level = "INFO" +testpaths = ["tests", "src/databricks/sqlalchemy/test_local"] +env_files = ["test.env"] diff --git a/src/databricks/__init__.py b/src/databricks/__init__.py index 2c691f3a..40d3f2e8 100644 --- a/src/databricks/__init__.py +++ b/src/databricks/__init__.py @@ -1,4 +1,7 @@ -# https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages -# This file should only contain the following line. Otherwise other sub-packages databricks.* namespace -# may not be importable. +# See: https://packaging.python.org/guides/packaging-namespace-packages/#pkgutil-style-namespace-packages +# +# This file must only contain the following line, or other packages in the databricks.* namespace +# may not be importable. The contents of this file must be byte-for-byte equivalent across all packages. +# If they are not, parallel package installation may lead to clobbered and invalid files. +# Also see https://github.com/databricks/databricks-sdk-py/issues/343. __path__ = __import__("pkgutil").extend_path(__path__, __name__) diff --git a/src/databricks/sql/__init__.py b/src/databricks/sql/__init__.py index fdfb3fb6..c0fdc2f1 100644 --- a/src/databricks/sql/__init__.py +++ b/src/databricks/sql/__init__.py @@ -5,7 +5,47 @@ # PEP 249 module globals apilevel = "2.0" threadsafety = 1 # Threads may share the module, but not connections. -paramstyle = "pyformat" # Python extended format codes, e.g. ...WHERE name=%(name)s + +paramstyle = "named" + +import re + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + # Use this import purely for type annotations, a la https://mypy.readthedocs.io/en/latest/runtime_troubles.html#import-cycles + from .client import Connection + + +class RedactUrlQueryParamsFilter(logging.Filter): + pattern = re.compile(r"(\?|&)([\w-]+)=([^&]+)") + mask = r"\1\2=" + + def __init__(self): + super().__init__() + + def redact(self, string): + return re.sub(self.pattern, self.mask, str(string)) + + def filter(self, record): + record.msg = self.redact(str(record.msg)) + if isinstance(record.args, dict): + for k in record.args.keys(): + record.args[k] = ( + self.redact(record.args[k]) + if isinstance(record.arg[k], str) + else record.args[k] + ) + else: + record.args = tuple( + (self.redact(arg) if isinstance(arg, str) else arg) + for arg in record.args + ) + + return True + + +logging.getLogger("urllib3.connectionpool").addFilter(RedactUrlQueryParamsFilter()) class DBAPITypeObject(object): @@ -28,7 +68,7 @@ def __repr__(self): DATE = DBAPITypeObject("date") ROWID = DBAPITypeObject() -__version__ = "2.5.2" +__version__ = "3.4.0" USER_AGENT_NAME = "PyDatabricksSqlConnector" # These two functions are pyhive legacy @@ -44,7 +84,7 @@ def TimestampFromTicks(ticks): return Timestamp(*time.localtime(ticks)[:6]) -def connect(server_hostname, http_path, access_token=None, **kwargs): +def connect(server_hostname, http_path, access_token=None, **kwargs) -> "Connection": from .client import Connection return Connection(server_hostname, http_path, access_token, **kwargs) diff --git a/src/databricks/sql/auth/auth.py b/src/databricks/sql/auth/auth.py old mode 100644 new mode 100755 index b56d8f7f..347934ee --- a/src/databricks/sql/auth/auth.py +++ b/src/databricks/sql/auth/auth.py @@ -1,19 +1,18 @@ from enum import Enum -from typing import List +from typing import Optional, List from databricks.sql.auth.authenticators import ( AuthProvider, AccessTokenAuthProvider, - BasicAuthProvider, ExternalAuthProvider, DatabricksOAuthProvider, ) -from databricks.sql.experimental.oauth_persistence import OAuthPersistence class AuthType(Enum): DATABRICKS_OAUTH = "databricks-oauth" - # other supported types (access_token, user/pass) can be inferred + AZURE_OAUTH = "azure-oauth" + # other supported types (access_token) can be inferred # we can add more types as needed later @@ -21,21 +20,17 @@ class ClientContext: def __init__( self, hostname: str, - username: str = None, - password: str = None, - access_token: str = None, - auth_type: str = None, - oauth_scopes: List[str] = None, - oauth_client_id: str = None, - oauth_redirect_port_range: List[int] = None, - use_cert_as_auth: str = None, - tls_client_cert_file: str = None, + access_token: Optional[str] = None, + auth_type: Optional[str] = None, + oauth_scopes: Optional[List[str]] = None, + oauth_client_id: Optional[str] = None, + oauth_redirect_port_range: Optional[List[int]] = None, + use_cert_as_auth: Optional[str] = None, + tls_client_cert_file: Optional[str] = None, oauth_persistence=None, credentials_provider=None, ): self.hostname = hostname - self.username = username - self.password = password self.access_token = access_token self.auth_type = auth_type self.oauth_scopes = oauth_scopes @@ -50,7 +45,7 @@ def __init__( def get_auth_provider(cfg: ClientContext): if cfg.credentials_provider: return ExternalAuthProvider(cfg.credentials_provider) - if cfg.auth_type == AuthType.DATABRICKS_OAUTH.value: + if cfg.auth_type in [AuthType.DATABRICKS_OAUTH.value, AuthType.AZURE_OAUTH.value]: assert cfg.oauth_redirect_port_range is not None assert cfg.oauth_client_id is not None assert cfg.oauth_scopes is not None @@ -61,21 +56,35 @@ def get_auth_provider(cfg: ClientContext): cfg.oauth_redirect_port_range, cfg.oauth_client_id, cfg.oauth_scopes, + cfg.auth_type, ) elif cfg.access_token is not None: return AccessTokenAuthProvider(cfg.access_token) - elif cfg.username is not None and cfg.password is not None: - return BasicAuthProvider(cfg.username, cfg.password) elif cfg.use_cert_as_auth and cfg.tls_client_cert_file: # no op authenticator. authentication is performed using ssl certificate outside of headers return AuthProvider() else: - raise RuntimeError("No valid authentication settings!") + if ( + cfg.oauth_redirect_port_range is not None + and cfg.oauth_client_id is not None + and cfg.oauth_scopes is not None + ): + return DatabricksOAuthProvider( + cfg.hostname, + cfg.oauth_persistence, + cfg.oauth_redirect_port_range, + cfg.oauth_client_id, + cfg.oauth_scopes, + ) + else: + raise RuntimeError("No valid authentication settings!") PYSQL_OAUTH_SCOPES = ["sql", "offline_access"] PYSQL_OAUTH_CLIENT_ID = "databricks-sql-python" +PYSQL_OAUTH_AZURE_CLIENT_ID = "96eecda7-19ea-49cc-abb5-240097d554f5" PYSQL_OAUTH_REDIRECT_PORT_RANGE = list(range(8020, 8025)) +PYSQL_OAUTH_AZURE_REDIRECT_PORT_RANGE = [8030] def normalize_host_name(hostname: str): @@ -84,20 +93,36 @@ def normalize_host_name(hostname: str): return f"{maybe_scheme}{hostname}{maybe_trailing_slash}" +def get_client_id_and_redirect_port(use_azure_auth: bool): + return ( + (PYSQL_OAUTH_CLIENT_ID, PYSQL_OAUTH_REDIRECT_PORT_RANGE) + if not use_azure_auth + else (PYSQL_OAUTH_AZURE_CLIENT_ID, PYSQL_OAUTH_AZURE_REDIRECT_PORT_RANGE) + ) + + def get_python_sql_connector_auth_provider(hostname: str, **kwargs): + auth_type = kwargs.get("auth_type") + (client_id, redirect_port_range) = get_client_id_and_redirect_port( + auth_type == AuthType.AZURE_OAUTH.value + ) + if kwargs.get("username") or kwargs.get("password"): + raise ValueError( + "Username/password authentication is no longer supported. " + "Please use OAuth or access token instead." + ) + cfg = ClientContext( hostname=normalize_host_name(hostname), - auth_type=kwargs.get("auth_type"), + auth_type=auth_type, access_token=kwargs.get("access_token"), - username=kwargs.get("_username"), - password=kwargs.get("_password"), use_cert_as_auth=kwargs.get("_use_cert_as_auth"), tls_client_cert_file=kwargs.get("_tls_client_cert_file"), oauth_scopes=PYSQL_OAUTH_SCOPES, - oauth_client_id=kwargs.get("oauth_client_id") or PYSQL_OAUTH_CLIENT_ID, + oauth_client_id=kwargs.get("oauth_client_id") or client_id, oauth_redirect_port_range=[kwargs["oauth_redirect_port"]] if kwargs.get("oauth_client_id") and kwargs.get("oauth_redirect_port") - else PYSQL_OAUTH_REDIRECT_PORT_RANGE, + else redirect_port_range, oauth_persistence=kwargs.get("experimental_oauth_persistence"), credentials_provider=kwargs.get("credentials_provider"), ) diff --git a/src/databricks/sql/auth/authenticators.py b/src/databricks/sql/auth/authenticators.py index eb368e1e..64eb91bb 100644 --- a/src/databricks/sql/auth/authenticators.py +++ b/src/databricks/sql/auth/authenticators.py @@ -4,6 +4,7 @@ from typing import Callable, Dict, List from databricks.sql.auth.oauth import OAuthManager +from databricks.sql.auth.endpoint import get_oauth_endpoints, infer_cloud_from_host # Private API: this is an evolving interface and it will change in the future. # Please must not depend on it in your applications. @@ -17,6 +18,7 @@ def add_headers(self, request_headers: Dict[str, str]): HeaderFactory = Callable[[], Dict[str, str]] + # In order to keep compatibility with SDK class CredentialsProvider(abc.ABC): """CredentialsProvider is the protocol (call-side interface) @@ -41,21 +43,6 @@ def add_headers(self, request_headers: Dict[str, str]): request_headers["Authorization"] = self.__authorization_header_value -# Private API: this is an evolving interface and it will change in the future. -# Please must not depend on it in your applications. -class BasicAuthProvider(AuthProvider): - def __init__(self, username: str, password: str): - auth_credentials = f"{username}:{password}".encode("UTF-8") - auth_credentials_base64 = base64.standard_b64encode(auth_credentials).decode( - "UTF-8" - ) - - self.__authorization_header_value = f"Basic {auth_credentials_base64}" - - def add_headers(self, request_headers: Dict[str, str]): - request_headers["Authorization"] = self.__authorization_header_value - - # Private API: this is an evolving interface and it will change in the future. # Please must not depend on it in your applications. class DatabricksOAuthProvider(AuthProvider): @@ -68,13 +55,25 @@ def __init__( redirect_port_range: List[int], client_id: str, scopes: List[str], + auth_type: str = "databricks-oauth", ): try: + idp_endpoint = get_oauth_endpoints(hostname, auth_type == "azure-oauth") + if not idp_endpoint: + raise NotImplementedError( + f"OAuth is not supported for host ${hostname}" + ) + + # Convert to the corresponding scopes in the corresponding IdP + cloud_scopes = idp_endpoint.get_scopes_mapping(scopes) + self.oauth_manager = OAuthManager( - port_range=redirect_port_range, client_id=client_id + port_range=redirect_port_range, + client_id=client_id, + idp_endpoint=idp_endpoint, ) self._hostname = hostname - self._scopes_as_str = DatabricksOAuthProvider.SCOPE_DELIM.join(scopes) + self._scopes_as_str = DatabricksOAuthProvider.SCOPE_DELIM.join(cloud_scopes) self._oauth_persistence = oauth_persistence self._client_id = client_id self._access_token = None diff --git a/src/databricks/sql/auth/endpoint.py b/src/databricks/sql/auth/endpoint.py new file mode 100644 index 00000000..5cb26ae3 --- /dev/null +++ b/src/databricks/sql/auth/endpoint.py @@ -0,0 +1,140 @@ +# +# It implements all the cloud specific OAuth configuration/metadata +# +# Azure: It uses Databricks internal IdP or Azure AD +# AWS: It uses Databricks internal IdP +# GCP: It uses Databricks internal IdP +# +from abc import ABC, abstractmethod +from enum import Enum +from typing import Optional, List +import os + +OIDC_REDIRECTOR_PATH = "oidc" + + +class OAuthScope: + OFFLINE_ACCESS = "offline_access" + SQL = "sql" + + +class CloudType(Enum): + AWS = "aws" + AZURE = "azure" + GCP = "gcp" + + +DATABRICKS_AWS_DOMAINS = [ + ".cloud.databricks.com", + ".cloud.databricks.us", + ".dev.databricks.com", +] + +DATABRICKS_AZURE_DOMAINS = [ + ".azuredatabricks.net", + ".databricks.azure.cn", + ".databricks.azure.us", +] +DATABRICKS_GCP_DOMAINS = [".gcp.databricks.com"] + +# Domain supported by Databricks InHouse OAuth +DATABRICKS_OAUTH_AZURE_DOMAINS = [".azuredatabricks.net"] + + +# Infer cloud type from Databricks SQL instance hostname +def infer_cloud_from_host(hostname: str) -> Optional[CloudType]: + # normalize + host = hostname.lower().replace("https://", "").split("/")[0] + + if any(e for e in DATABRICKS_AZURE_DOMAINS if host.endswith(e)): + return CloudType.AZURE + elif any(e for e in DATABRICKS_AWS_DOMAINS if host.endswith(e)): + return CloudType.AWS + elif any(e for e in DATABRICKS_GCP_DOMAINS if host.endswith(e)): + return CloudType.GCP + else: + return None + + +def is_supported_databricks_oauth_host(hostname: str) -> bool: + host = hostname.lower().replace("https://", "").split("/")[0] + domains = ( + DATABRICKS_AWS_DOMAINS + DATABRICKS_GCP_DOMAINS + DATABRICKS_OAUTH_AZURE_DOMAINS + ) + return any(e for e in domains if host.endswith(e)) + + +def get_databricks_oidc_url(hostname: str): + maybe_scheme = "https://" if not hostname.startswith("https://") else "" + maybe_trailing_slash = "/" if not hostname.endswith("/") else "" + return f"{maybe_scheme}{hostname}{maybe_trailing_slash}{OIDC_REDIRECTOR_PATH}" + + +class OAuthEndpointCollection(ABC): + @abstractmethod + def get_scopes_mapping(self, scopes: List[str]) -> List[str]: + raise NotImplementedError() + + # Endpoint for oauth2 authorization e.g https://idp.example.com/oauth2/v2.0/authorize + @abstractmethod + def get_authorization_url(self, hostname: str) -> str: + raise NotImplementedError() + + # Endpoint for well-known openid configuration e.g https://idp.example.com/oauth2/.well-known/openid-configuration + @abstractmethod + def get_openid_config_url(self, hostname: str) -> str: + raise NotImplementedError() + + +class AzureOAuthEndpointCollection(OAuthEndpointCollection): + DATATRICKS_AZURE_APP = "2ff814a6-3304-4ab8-85cb-cd0e6f879c1d" + + def get_scopes_mapping(self, scopes: List[str]) -> List[str]: + # There is no corresponding scopes in Azure, instead, access control will be delegated to Databricks + tenant_id = os.getenv( + "DATABRICKS_AZURE_TENANT_ID", + AzureOAuthEndpointCollection.DATATRICKS_AZURE_APP, + ) + azure_scope = f"{tenant_id}/user_impersonation" + mapped_scopes = [azure_scope] + if OAuthScope.OFFLINE_ACCESS in scopes: + mapped_scopes.append(OAuthScope.OFFLINE_ACCESS) + return mapped_scopes + + def get_authorization_url(self, hostname: str): + # We need get account specific url, which can be redirected by databricks unified oidc endpoint + return f"{get_databricks_oidc_url(hostname)}/oauth2/v2.0/authorize" + + def get_openid_config_url(self, hostname: str): + return "https://login.microsoftonline.com/organizations/v2.0/.well-known/openid-configuration" + + +class InHouseOAuthEndpointCollection(OAuthEndpointCollection): + def get_scopes_mapping(self, scopes: List[str]) -> List[str]: + # No scope mapping in AWS + return scopes.copy() + + def get_authorization_url(self, hostname: str): + idp_url = get_databricks_oidc_url(hostname) + return f"{idp_url}/oauth2/v2.0/authorize" + + def get_openid_config_url(self, hostname: str): + idp_url = get_databricks_oidc_url(hostname) + return f"{idp_url}/.well-known/oauth-authorization-server" + + +def get_oauth_endpoints( + hostname: str, use_azure_auth: bool +) -> Optional[OAuthEndpointCollection]: + cloud = infer_cloud_from_host(hostname) + + if cloud in [CloudType.AWS, CloudType.GCP]: + return InHouseOAuthEndpointCollection() + elif cloud == CloudType.AZURE: + return ( + InHouseOAuthEndpointCollection() + if is_supported_databricks_oauth_host(hostname) and not use_azure_auth + else AzureOAuthEndpointCollection() + ) + else: + return None diff --git a/src/databricks/sql/auth/oauth.py b/src/databricks/sql/auth/oauth.py index 0f49aa88..806df08f 100644 --- a/src/databricks/sql/auth/oauth.py +++ b/src/databricks/sql/auth/oauth.py @@ -14,17 +14,38 @@ from requests.exceptions import RequestException from databricks.sql.auth.oauth_http_handler import OAuthHttpSingleRequestHandler +from databricks.sql.auth.endpoint import OAuthEndpointCollection logger = logging.getLogger(__name__) -class OAuthManager: - OIDC_REDIRECTOR_PATH = "oidc" +class IgnoreNetrcAuth(requests.auth.AuthBase): + """This auth method is a no-op. + + We use it to force requestslib to not use .netrc to write auth headers + when making .post() requests to the oauth token endpoints, since these + don't require authentication. + + In cases where .netrc is outdated or corrupt, these requests will fail. + + See issue #121 + """ - def __init__(self, port_range: List[int], client_id: str): + def __call__(self, r): + return r + + +class OAuthManager: + def __init__( + self, + port_range: List[int], + client_id: str, + idp_endpoint: OAuthEndpointCollection, + ): self.port_range = port_range self.client_id = client_id self.redirect_port = None + self.idp_endpoint = idp_endpoint @staticmethod def __token_urlsafe(nbytes=32): @@ -34,14 +55,14 @@ def __token_urlsafe(nbytes=32): def __get_redirect_url(redirect_port: int): return f"http://localhost:{redirect_port}" - @staticmethod - def __fetch_well_known_config(idp_url: str): - known_config_url = f"{idp_url}/.well-known/oauth-authorization-server" + def __fetch_well_known_config(self, hostname: str): + known_config_url = self.idp_endpoint.get_openid_config_url(hostname) + try: - response = requests.get(url=known_config_url) + response = requests.get(url=known_config_url, auth=IgnoreNetrcAuth()) except RequestException as e: logger.error( - f"Unable to fetch OAuth configuration from {idp_url}.\n" + f"Unable to fetch OAuth configuration from {known_config_url}.\n" "Verify it is a valid workspace URL and that OAuth is " "enabled on this account." ) @@ -50,7 +71,7 @@ def __fetch_well_known_config(idp_url: str): if response.status_code != 200: msg = ( f"Received status {response.status_code} OAuth configuration from " - f"{idp_url}.\n Verify it is a valid workspace URL and " + f"{known_config_url}.\n Verify it is a valid workspace URL and " "that OAuth is enabled on this account." ) logger.error(msg) @@ -59,18 +80,12 @@ def __fetch_well_known_config(idp_url: str): return response.json() except requests.exceptions.JSONDecodeError as e: logger.error( - f"Unable to decode OAuth configuration from {idp_url}.\n" + f"Unable to decode OAuth configuration from {known_config_url}.\n" "Verify it is a valid workspace URL and that OAuth is " "enabled on this account." ) raise e - @staticmethod - def __get_idp_url(host: str): - maybe_scheme = "https://" if not host.startswith("https://") else "" - maybe_trailing_slash = "/" if not host.endswith("/") else "" - return f"{maybe_scheme}{host}{maybe_trailing_slash}{OAuthManager.OIDC_REDIRECTOR_PATH}" - @staticmethod def __get_challenge(): verifier_string = OAuthManager.__token_urlsafe(32) @@ -110,7 +125,7 @@ def __get_authorization_code(self, client, auth_url, scope, state, challenge): logger.info(f"Port {port} is in use") last_error = e except Exception as e: - logger.error("unexpected error", e) + logger.error("unexpected error: %s", e) if self.redirect_port is None: logger.error( f"Tried all the ports {self.port_range} for oauth redirect, but can't find free port" @@ -150,12 +165,13 @@ def __send_token_request(token_request_url, data): "Accept": "application/json", "Content-Type": "application/x-www-form-urlencoded", } - response = requests.post(url=token_request_url, data=data, headers=headers) + response = requests.post( + url=token_request_url, data=data, headers=headers, auth=IgnoreNetrcAuth() + ) return response.json() def __send_refresh_token_request(self, hostname, refresh_token): - idp_url = OAuthManager.__get_idp_url(hostname) - oauth_config = OAuthManager.__fetch_well_known_config(idp_url) + oauth_config = self.__fetch_well_known_config(hostname) token_request_url = oauth_config["token_endpoint"] client = oauthlib.oauth2.WebApplicationClient(self.client_id) token_request_body = client.prepare_refresh_body( @@ -215,14 +231,15 @@ def check_and_refresh_access_token( return fresh_access_token, fresh_refresh_token, True def get_tokens(self, hostname: str, scope=None): - idp_url = self.__get_idp_url(hostname) - oauth_config = self.__fetch_well_known_config(idp_url) + oauth_config = self.__fetch_well_known_config(hostname) # We are going to override oauth_config["authorization_endpoint"] use the # /oidc redirector on the hostname, which may inject additional parameters. - auth_url = f"{hostname}oidc/v1/authorize" + auth_url = self.idp_endpoint.get_authorization_url(hostname) + state = OAuthManager.__token_urlsafe(16) (verifier, challenge) = OAuthManager.__get_challenge() client = oauthlib.oauth2.WebApplicationClient(self.client_id) + try: auth_response = self.__get_authorization_code( client, auth_url, scope, state, challenge diff --git a/src/databricks/sql/auth/retry.py b/src/databricks/sql/auth/retry.py new file mode 100755 index 00000000..0c6547cb --- /dev/null +++ b/src/databricks/sql/auth/retry.py @@ -0,0 +1,436 @@ +import logging +import time +import typing +from enum import Enum +from typing import List, Optional, Tuple, Union + +# We only use this import for type hinting +try: + # If urllib3~=2.0 is installed + from urllib3 import BaseHTTPResponse +except ImportError: + # If urllib3~=1.0 is installed + from urllib3 import HTTPResponse as BaseHTTPResponse +from urllib3 import Retry +from urllib3.util.retry import RequestHistory + +from databricks.sql.exc import ( + CursorAlreadyClosedError, + MaxRetryDurationError, + NonRecoverableNetworkError, + OperationalError, + SessionAlreadyClosedError, + UnsafeToRetryError, +) + +logger = logging.getLogger(__name__) + + +class CommandType(Enum): + EXECUTE_STATEMENT = "ExecuteStatement" + CLOSE_SESSION = "CloseSession" + CLOSE_OPERATION = "CloseOperation" + GET_OPERATION_STATUS = "GetOperationStatus" + OTHER = "Other" + + @classmethod + def get(cls, value: str): + value_name_map = {i.value: i.name for i in cls} + valid_command = value_name_map.get(value, False) + if valid_command: + return getattr(cls, str(valid_command)) + else: + return cls.OTHER + + +class DatabricksRetryPolicy(Retry): + """ + Implements our v3 retry policy by extending urllib3's robust default retry behaviour. + + Retry logic varies based on the overall wall-clock request time and Thrift CommandType + being issued. ThriftBackend starts a timer and sets the current CommandType prior to + initiating a network request. See `self.should_retry()` for details about what we do + and do not retry. + + :param delay_min: + Float of seconds for the minimum delay between retries. This is an alias for urllib3's + `backoff_factor`. + + :param delay_max: + Float of seconds for the maximum delay between retries. + + :param stop_after_attempts_count: + Integer maximum number of attempts that will be retried. This is an alias for urllib3's + `total`. + + :param stop_after_attempts_duration: + Float of maximum number of seconds within which a request may be retried starting from + the beginning of the first request. + + :param delay_default: + Float of seconds the connector will wait between sucessive GetOperationStatus + requests. This parameter is not used to retry failed network requests. We include + it in this class to keep all retry behaviour encapsulated in this file. + + :param force_dangerous_codes: + List of integer HTTP status codes that the connector will retry, even for dangerous + commands like ExecuteStatement. This is passed to urllib3 by extending its status_forcelist + + :param urllib3_kwargs: + Dictionary of arguments that are passed to Retry.__init__. Any setting of Retry() that + Databricks does not override or extend may be modified here. + """ + + def __init__( + self, + delay_min: float, + delay_max: float, + stop_after_attempts_count: int, + stop_after_attempts_duration: float, + delay_default: float, + force_dangerous_codes: List[int], + urllib3_kwargs: dict = {}, + ): + # These values do not change from one command to the next + self.delay_max = delay_max + self.delay_min = delay_min + self.stop_after_attempts_count = stop_after_attempts_count + self.stop_after_attempts_duration = stop_after_attempts_duration + self._delay_default = delay_default + self.force_dangerous_codes = force_dangerous_codes + + # the urllib3 kwargs are a mix of configuration (some of which we override) + # and counters like `total` or `connect` which may change between successive retries + # we only care about urllib3 kwargs that we alias, override, or add to in some way + + # the length of _history increases as retries are performed + _history: Optional[Tuple[RequestHistory, ...]] = urllib3_kwargs.get("history") + + if not _history: + # no attempts were made so we can retry the current command as many times as specified + # by the user + _attempts_remaining = self.stop_after_attempts_count + else: + # at least one of our attempts has been consumed, and urllib3 will have set a total + # `total` is a counter that begins equal to self.stop_after_attempts_count and is + # decremented after each unsuccessful request. When `total` is zero, urllib3 raises a + # MaxRetryError + _total: int = urllib3_kwargs.pop("total") + _attempts_remaining = _total + + _urllib_kwargs_we_care_about = dict( + total=_attempts_remaining, + respect_retry_after_header=True, + backoff_factor=self.delay_min, + allowed_methods=["POST"], + status_forcelist=[429, 503, *self.force_dangerous_codes], + ) + + urllib3_kwargs.update(**_urllib_kwargs_we_care_about) + + super().__init__( + **urllib3_kwargs, + ) + + @classmethod + def __private_init__( + cls, retry_start_time: float, command_type: Optional[CommandType], **init_kwargs + ): + """ + Returns a new instance of DatabricksRetryPolicy with the _retry_start_time and _command_type + properties already set. This method should only be called by DatabricksRetryPolicy itself between + successive Retry attempts. + + :param retry_start_time: + Float unix timestamp. Used to monitor the overall request duration across successive + retries. Never set this value directly. Use self.start_retry_timer() instead. Users + never set this value. It is set by ThriftBackend immediately before issuing a network + request. + + :param command_type: + CommandType of the current request being retried. Used to modify retry behaviour based + on the type of Thrift command being issued. See self.should_retry() for details. Users + never set this value directly. It is set by ThriftBackend immediately before issuing + a network request. + + :param init_kwargs: + A dictionary of parameters that will be passed to __init__ in the new object + """ + + new_object = cls(**init_kwargs) + new_object._retry_start_time = retry_start_time + new_object.command_type = command_type + return new_object + + def new( + self, **urllib3_incremented_counters: typing.Any + ) -> "DatabricksRetryPolicy": + """This method is responsible for passing the entire Retry state to its next iteration. + + urllib3 calls Retry.new() between successive requests as part of its `.increment()` method + as shown below: + + ```python + new_retry = self.new( + total=total, + connect=connect, + read=read, + redirect=redirect, + status=status_count, + other=other, + history=history, + ) + ``` + + The arguments it passes to `.new()` (total, connect, read, etc.) are those modified by `.increment()`. + + Since self.__init__ has a different signature than Retry.__init__ , we implement our own `self.new()` + to pipe our Databricks-specific state while preserving the super-class's behaviour. + + """ + + # These arguments will match the function signature for self.__init__ + databricks_init_params = dict( + delay_min=self.delay_min, + delay_max=self.delay_max, + stop_after_attempts_count=self.stop_after_attempts_count, + stop_after_attempts_duration=self.stop_after_attempts_duration, + delay_default=self.delay_default, + force_dangerous_codes=self.force_dangerous_codes, + urllib3_kwargs={}, + ) + + # Gather urllib3's current retry state _before_ increment was called + # These arguments match the function signature for Retry.__init__ + # Note: if we update urllib3 we may need to add/remove arguments from this dict + urllib3_init_params = dict( + total=self.total, + connect=self.connect, + read=self.read, + redirect=self.redirect, + status=self.status, + other=self.other, + allowed_methods=self.allowed_methods, + status_forcelist=self.status_forcelist, + backoff_factor=self.backoff_factor, + raise_on_redirect=self.raise_on_redirect, + raise_on_status=self.raise_on_status, + history=self.history, + remove_headers_on_redirect=self.remove_headers_on_redirect, + respect_retry_after_header=self.respect_retry_after_header, + ) + + # Update urllib3's current state to reflect the incremented counters + urllib3_init_params.update(**urllib3_incremented_counters) + + # Include urllib3's current state in our __init__ params + databricks_init_params["urllib3_kwargs"].update(**urllib3_init_params) # type: ignore[attr-defined] + + return type(self).__private_init__( + retry_start_time=self._retry_start_time, + command_type=self.command_type, + **databricks_init_params, + ) + + @property + def command_type(self) -> Optional[CommandType]: + return self._command_type + + @command_type.setter + def command_type(self, value: CommandType) -> None: + self._command_type = value + + @property + def delay_default(self) -> float: + """Time in seconds the connector will wait between requests polling a GetOperationStatus Request + + This property is never read by urllib3 for the purpose of retries. It's stored in this class + to keep all retry logic in one place. + + This property is only set by __init__ and cannot be modified afterward. + """ + return self._delay_default + + def start_retry_timer(self) -> None: + """Timer is used to monitor the overall time across successive requests + + Should only be called by ThriftBackend before sending a Thrift command""" + self._retry_start_time = time.time() + + def check_timer_duration(self) -> float: + """Return time in seconds since the timer was started""" + + if self._retry_start_time is None: + raise OperationalError( + "Cannot check retry timer. Timer was not started for this request." + ) + else: + return time.time() - self._retry_start_time + + def check_proposed_wait(self, proposed_wait: Union[int, float]) -> None: + """Raise an exception if the proposed wait would exceed the configured max_attempts_duration""" + + proposed_overall_time = self.check_timer_duration() + proposed_wait + if proposed_overall_time > self.stop_after_attempts_duration: + raise MaxRetryDurationError( + f"Retry request would exceed Retry policy max retry duration of {self.stop_after_attempts_duration} seconds" + ) + + def sleep_for_retry(self, response: BaseHTTPResponse) -> bool: + """Sleeps for the duration specified in the response Retry-After header, if present + + A MaxRetryDurationError will be raised if doing so would exceed self.max_attempts_duration + + This method is only called by urllib3 internals. + """ + retry_after = self.get_retry_after(response) + if retry_after: + backoff = self.get_backoff_time() + proposed_wait = max(backoff, retry_after) + self.check_proposed_wait(proposed_wait) + time.sleep(proposed_wait) + return True + + return False + + def get_backoff_time(self) -> float: + """Calls urllib3's built-in get_backoff_time. + + Never returns a value larger than self.delay_max + A MaxRetryDurationError will be raised if the calculated backoff would exceed self.max_attempts_duration + + Note: within urllib3, a backoff is only calculated in cases where a Retry-After header is not present + in the previous unsuccessful request and `self.respect_retry_after_header` is True (which is always true) + """ + + proposed_backoff = super().get_backoff_time() + proposed_backoff = min(proposed_backoff, self.delay_max) + self.check_proposed_wait(proposed_backoff) + + return proposed_backoff + + def should_retry(self, method: str, status_code: int) -> Tuple[bool, str]: + """This method encapsulates the connector's approach to retries. + + We always retry a request unless one of these conditions is met: + + 1. The request received a 200 (Success) status code + Because the request succeeded . + 2. The request received a 501 (Not Implemented) status code + Because this request can never succeed. + 3. The request received a 404 (Not Found) code and the request CommandType + was GetOperationStatus, CloseSession or CloseOperation. This code indicates + that the command, session or cursor was already closed. Further retries will + always return the same code. + 4. The request CommandType was ExecuteStatement and the HTTP code does not + appear in the default status_forcelist or force_dangerous_codes list. By + default, this means ExecuteStatement is only retried for codes 429 and 503. + This limit prevents automatically retrying non-idempotent commands that could + be destructive. + 5. The request received a 401 response, because this can never succeed. + 6. The request received a 403 response, because this can never succeed. + + + Q: What about OSErrors and Redirects? + A: urllib3 automatically retries in both scenarios + + Returns True if the request should be retried. Returns False or raises an exception + if a retry would violate the configured policy. + """ + + # Request succeeded. Don't retry. + if status_code == 200: + return False, "200 codes are not retried" + + if status_code == 401: + raise NonRecoverableNetworkError( + "Received 401 - UNAUTHORIZED. Confirm your authentication credentials." + ) + + if status_code == 403: + raise NonRecoverableNetworkError( + "Received 403 - FORBIDDEN. Confirm your authentication credentials." + ) + + # Request failed and server said NotImplemented. This isn't recoverable. Don't retry. + if status_code == 501: + raise NonRecoverableNetworkError("Received code 501 from server.") + + # Request failed and this method is not retryable. We only retry POST requests. + if not self._is_method_retryable(method): + return False, "Only POST requests are retried" + + # Request failed with 404 and was a GetOperationStatus. This is not recoverable. Don't retry. + if status_code == 404 and self.command_type == CommandType.GET_OPERATION_STATUS: + return ( + False, + "GetOperationStatus received 404 code from Databricks. Operation was canceled.", + ) + + # Request failed with 404 because CloseSession returns 404 if you repeat the request. + if ( + status_code == 404 + and self.command_type == CommandType.CLOSE_SESSION + and len(self.history) > 0 + ): + raise SessionAlreadyClosedError( + "CloseSession received 404 code from Databricks. Session is already closed." + ) + + # Request failed with 404 because CloseOperation returns 404 if you repeat the request. + if ( + status_code == 404 + and self.command_type == CommandType.CLOSE_OPERATION + and len(self.history) > 0 + ): + raise CursorAlreadyClosedError( + "CloseOperation received 404 code from Databricks. Cursor is already closed." + ) + + # Request failed, was an ExecuteStatement and the command may have reached the server + if ( + self.command_type == CommandType.EXECUTE_STATEMENT + and status_code not in self.status_forcelist + and status_code not in self.force_dangerous_codes + ): + raise UnsafeToRetryError( + "ExecuteStatement command can only be retried for codes 429 and 503" + ) + + # Request failed with a dangerous code, was an ExecuteStatement, but user forced retries for this + # dangerous code. Note that these lines _are not required_ to make these requests retry. They would + # retry automatically. This code is included only so that we can log the exact reason for the retry. + # This gives users signal that their _retry_dangerous_codes setting actually did something. + if ( + self.command_type == CommandType.EXECUTE_STATEMENT + and status_code in self.force_dangerous_codes + ): + return ( + True, + f"Request failed with dangerous code {status_code} that is one of the configured _retry_dangerous_codes.", + ) + + # None of the above conditions applied. Eagerly retry. + logger.debug( + f"This request should be retried: {self.command_type and self.command_type.value}" + ) + return ( + True, + "Failed requests are retried by default per configured DatabricksRetryPolicy", + ) + + def is_retry( + self, method: str, status_code: int, has_retry_after: bool = False + ) -> bool: + """ + Called by urllib3 when determining whether or not to retry + + Logs a debug message if the request will be retried + """ + + should_retry, msg = self.should_retry(method, status_code) + + if should_retry: + logger.debug(msg) + + return should_retry diff --git a/src/databricks/sql/auth/thrift_http_client.py b/src/databricks/sql/auth/thrift_http_client.py index a924ea63..6273ab28 100644 --- a/src/databricks/sql/auth/thrift_http_client.py +++ b/src/databricks/sql/auth/thrift_http_client.py @@ -1,10 +1,20 @@ +import base64 import logging -from typing import Dict - +import urllib.parse +from typing import Dict, Union, Optional +import six import thrift -import urllib.parse, six, base64 +import ssl +import warnings +from http.client import HTTPResponse +from io import BytesIO + +from urllib3 import HTTPConnectionPool, HTTPSConnectionPool, ProxyManager +from urllib3.util import make_headers +from databricks.sql.auth.retry import CommandType, DatabricksRetryPolicy +from databricks.sql.types import SSLOptions logger = logging.getLogger(__name__) @@ -16,34 +26,193 @@ def __init__( uri_or_host, port=None, path=None, - cafile=None, - cert_file=None, - key_file=None, - ssl_context=None, + ssl_options: Optional[SSLOptions] = None, + max_connections: int = 1, + retry_policy: Union[DatabricksRetryPolicy, int] = 0, ): - super().__init__( - uri_or_host, port, path, cafile, cert_file, key_file, ssl_context - ) + self._ssl_options = ssl_options + + if port is not None: + warnings.warn( + "Please use the THttpClient('http{s}://host:port/path') constructor", + DeprecationWarning, + stacklevel=2, + ) + self.host = uri_or_host + self.port = port + assert path + self.path = path + self.scheme = "http" + else: + parsed = urllib.parse.urlsplit(uri_or_host) + self.scheme = parsed.scheme + assert self.scheme in ("http", "https") + if self.scheme == "https": + if self._ssl_options is not None: + # TODO: Not sure if those options are used anywhere - need to double-check + self.certfile = self._ssl_options.tls_client_cert_file + self.keyfile = self._ssl_options.tls_client_cert_key_file + self.context = self._ssl_options.create_ssl_context() + self.port = parsed.port + self.host = parsed.hostname + self.path = parsed.path + if parsed.query: + self.path += "?%s" % parsed.query + try: + proxy = urllib.request.getproxies()[self.scheme] + except KeyError: + proxy = None + else: + if urllib.request.proxy_bypass(self.host): + proxy = None + if proxy: + parsed = urllib.parse.urlparse(proxy) + + # realhost and realport are the host and port of the actual request + self.realhost = self.host + self.realport = self.port + + # this is passed to ProxyManager + self.proxy_uri: str = proxy + self.host = parsed.hostname + self.port = parsed.port + self.proxy_auth = self.basic_proxy_auth_headers(parsed) + else: + self.realhost = self.realport = self.proxy_auth = None + + self.max_connections = max_connections + + # If retry_policy == 0 then urllib3 will not retry automatically + # this falls back to the pre-v3 behaviour where thrift_backend.py handles retry logic + self.retry_policy = retry_policy + + self.__wbuf = BytesIO() + self.__resp: Union[None, HTTPResponse] = None + self.__timeout = None + self.__custom_headers = None + self.__auth_provider = auth_provider def setCustomHeaders(self, headers: Dict[str, str]): self._headers = headers super().setCustomHeaders(headers) + def startRetryTimer(self): + """Notify DatabricksRetryPolicy of the request start time + + This is used to enforce the retry_stop_after_attempts_duration + """ + self.retry_policy and self.retry_policy.start_retry_timer() + + def open(self): + + # self.__pool replaces the self.__http used by the original THttpClient + _pool_kwargs = {"maxsize": self.max_connections} + + if self.scheme == "http": + pool_class = HTTPConnectionPool + elif self.scheme == "https": + pool_class = HTTPSConnectionPool + _pool_kwargs.update( + { + "cert_reqs": ssl.CERT_REQUIRED + if self._ssl_options.tls_verify + else ssl.CERT_NONE, + "ca_certs": self._ssl_options.tls_trusted_ca_file, + "cert_file": self._ssl_options.tls_client_cert_file, + "key_file": self._ssl_options.tls_client_cert_key_file, + "key_password": self._ssl_options.tls_client_cert_key_password, + } + ) + + if self.using_proxy(): + proxy_manager = ProxyManager( + self.proxy_uri, + num_pools=1, + proxy_headers=self.proxy_auth, + ) + self.__pool = proxy_manager.connection_from_host( + host=self.realhost, + port=self.realport, + scheme=self.scheme, + pool_kwargs=_pool_kwargs, + ) + else: + self.__pool = pool_class(self.host, self.port, **_pool_kwargs) + + def close(self): + self.__resp and self.__resp.drain_conn() + self.__resp and self.__resp.release_conn() + self.__resp = None + + def read(self, sz): + return self.__resp.read(sz) + + def isOpen(self): + return self.__resp is not None + def flush(self): + + # Pull data out of buffer that will be sent in this request + data = self.__wbuf.getvalue() + self.__wbuf = BytesIO() + + # Header handling + headers = dict(self._headers) self.__auth_provider.add_headers(headers) self._headers = headers self.setCustomHeaders(self._headers) - super().flush() + + # Note: we don't set User-Agent explicitly in this class because PySQL + # should always provide one. Unlike the original THttpClient class, our version + # doesn't define a default User-Agent and so should raise an exception if one + # isn't provided. + assert self.__custom_headers and "User-Agent" in self.__custom_headers + + headers = { + "Content-Type": "application/x-thrift", + "Content-Length": str(len(data)), + } + + if self.using_proxy() and self.scheme == "http" and self.proxy_auth is not None: + headers.update(self.proxy_auth) + + if self.__custom_headers: + custom_headers = {key: val for key, val in self.__custom_headers.items()} + headers.update(**custom_headers) + + # HTTP request + self.__resp = self.__pool.request( + "POST", + url=self.path, + body=data, + headers=headers, + preload_content=False, + timeout=self.__timeout, + retries=self.retry_policy, + ) + + # Get reply to flush the request + self.code = self.__resp.status + self.message = self.__resp.reason + self.headers = self.__resp.headers @staticmethod - def basic_proxy_auth_header(proxy): + def basic_proxy_auth_headers(proxy): if proxy is None or not proxy.username: return None ap = "%s:%s" % ( urllib.parse.unquote(proxy.username), urllib.parse.unquote(proxy.password), ) - cr = base64.b64encode(ap.encode()).strip() - return "Basic " + six.ensure_str(cr) + return make_headers(proxy_basic_auth=ap) + + def set_retry_command_type(self, value: CommandType): + """Pass the provided CommandType to the retry policy""" + if isinstance(self.retry_policy, DatabricksRetryPolicy): + self.retry_policy.command_type = value + else: + logger.warning( + "DatabricksRetryPolicy is currently bypassed. The CommandType cannot be set." + ) diff --git a/src/databricks/sql/client.py b/src/databricks/sql/client.py old mode 100644 new mode 100755 index 722ed778..4df67a08 --- a/src/databricks/sql/client.py +++ b/src/databricks/sql/client.py @@ -1,25 +1,61 @@ -from typing import Dict, Tuple, List, Optional, Any, Union +from typing import Dict, Tuple, List, Optional, Any, Union, Sequence import pandas -import pyarrow +try: + import pyarrow +except ImportError: + pyarrow = None import requests import json import os +import decimal +from uuid import UUID from databricks.sql import __version__ from databricks.sql import * -from databricks.sql.exc import OperationalError +from databricks.sql.exc import ( + OperationalError, + SessionAlreadyClosedError, + CursorAlreadyClosedError, +) +from databricks.sql.thrift_api.TCLIService import ttypes from databricks.sql.thrift_backend import ThriftBackend -from databricks.sql.utils import ExecuteResponse, ParamEscaper, inject_parameters -from databricks.sql.types import Row +from databricks.sql.utils import ( + ExecuteResponse, + ParamEscaper, + inject_parameters, + transform_paramstyle, + ColumnTable, + ColumnQueue +) +from databricks.sql.parameters.native import ( + DbsqlParameterBase, + TDbsqlParameter, + TParameterDict, + TParameterSequence, + TParameterCollection, + ParameterStructure, + dbsql_parameter_from_primitive, + ParameterApproach, +) + + +from databricks.sql.types import Row, SSLOptions from databricks.sql.auth.auth import get_python_sql_connector_auth_provider from databricks.sql.experimental.oauth_persistence import OAuthPersistence +from databricks.sql.thrift_api.TCLIService.ttypes import ( + TSparkParameter, +) + + logger = logging.getLogger(__name__) -DEFAULT_RESULT_BUFFER_SIZE_BYTES = 10485760 +DEFAULT_RESULT_BUFFER_SIZE_BYTES = 104857600 DEFAULT_ARRAY_SIZE = 100000 +NO_NATIVE_PARAMS: List = [] + class Connection: def __init__( @@ -28,9 +64,10 @@ def __init__( http_path: str, access_token: Optional[str] = None, http_headers: Optional[List[Tuple[str, str]]] = None, - session_configuration: Dict[str, Any] = None, + session_configuration: Optional[Dict[str, Any]] = None, catalog: Optional[str] = None, schema: Optional[str] = None, + _use_arrow_native_complex_types: Optional[bool] = True, **kwargs, ) -> None: """ @@ -44,11 +81,13 @@ def __init__( Http Bearer access token, e.g. Databricks Personal Access Token. Unless if you use auth_type=`databricks-oauth` you need to pass `access_token. Examples: + ``` connection = sql.connect( server_hostname='dbc-12345.staging.cloud.databricks.com', http_path='sql/protocolv1/o/6789/12abc567', access_token='dabpi12345678' ) + ``` :param http_headers: An optional list of (k, v) pairs that will be set as Http headers on every request :param session_configuration: An optional dictionary of Spark session parameters. Defaults to None. Execute the SQL command `SET -v` to get a full list of available commands. @@ -56,12 +95,15 @@ def __init__( :param schema: An optional initial schema to use. Requires DBR version 9.0+ Other Parameters: - auth_type: `str`, optional - `databricks-oauth` : to use oauth with fine-grained permission scopes, set to `databricks-oauth`. - This is currently in private preview for Databricks accounts on AWS. - This supports User to Machine OAuth authentication for Databricks on AWS with - any IDP configured. This is only for interactive python applications and open a browser window. - Note this is beta (private preview) + use_inline_params: `boolean` | str, optional (default is False) + When True, parameterized calls to cursor.execute() will try to render parameter values inline with the + query text instead of using native bound parameters supported in DBR 14.1 and above. This connector will attempt to + sanitise parameterized inputs to prevent SQL injection. The inline parameter approach is maintained for + legacy purposes and will be deprecated in a future release. When this parameter is `True` you will see + a warning log message. To suppress this log message, set `use_inline_params="silent"`. + auth_type: `str`, optional (default is databricks-oauth if neither `access_token` nor `tls_client_cert_file` is set) + `databricks-oauth` : to use Databricks OAuth with fine-grained permission scopes, set to `databricks-oauth`. + `azure-oauth` : to use Microsoft Entra ID OAuth flow, set to `azure-oauth`. oauth_client_id: `str`, optional custom oauth client_id. If not specified, it will use the built-in client_id of databricks-sql-python. @@ -72,9 +114,9 @@ def __init__( experimental_oauth_persistence: configures preferred storage for persisting oauth tokens. This has to be a class implementing `OAuthPersistence`. - When `auth_type` is set to `databricks-oauth` without persisting the oauth token in a persistence storage - the oauth tokens will only be maintained in memory and if the python process restarts the end user - will have to login again. + When `auth_type` is set to `databricks-oauth` or `azure-oauth` without persisting the oauth token in a + persistence storage the oauth tokens will only be maintained in memory and if the python process + restarts the end user will have to login again. Note this is beta (private preview) For persisting the oauth token in a prod environment you should subclass and implement OAuthPersistence @@ -103,6 +145,7 @@ def read(self) -> Optional[OAuthToken]: own implementation of OAuthPersistence. Examples: + ``` # for development only from databricks.sql.experimental.oauth_persistence import DevOnlyFilePersistence @@ -112,30 +155,37 @@ def read(self) -> Optional[OAuthToken]: auth_type="databricks-oauth", experimental_oauth_persistence=DevOnlyFilePersistence("~/dev-oauth.json") ) - - + ``` + :param _use_arrow_native_complex_types: `bool`, optional + Controls whether a complex type field value is returned as a string or as a native Arrow type. Defaults to True. + When True: + MAP is returned as List[Tuple[str, Any]] + STRUCT is returned as Dict[str, Any] + ARRAY is returned as numpy.ndarray + When False, complex types are returned as a strings. These are generally deserializable as JSON. """ # Internal arguments in **kwargs: # _user_agent_entry # Tag to add to User-Agent header. For use by partners. - # _username, _password - # Username and password Basic authentication (no official support) # _use_cert_as_auth - # Use a TLS cert instead of a token or username / password (internal use only) + # Use a TLS cert instead of a token # _enable_ssl # Connect over HTTP instead of HTTPS # _port # Which port to connect to # _skip_routing_headers: # Don't set routing headers if set to True (for use when connecting directly to server) + # _tls_no_verify + # Set to True (Boolean) to completely disable SSL verification. # _tls_verify_hostname # Set to False (Boolean) to disable SSL hostname verification, but check certificate. # _tls_trusted_ca_file # Set to the path of the file containing trusted CA certificates for server certificate # verification. If not provide, uses system truststore. - # _tls_client_cert_file, _tls_client_cert_key_file + # _tls_client_cert_file, _tls_client_cert_key_file, _tls_client_cert_key_password # Set client SSL certificate. + # See https://docs.python.org/3/library/ssl.html#ssl.SSLContext.load_cert_chain # _retry_stop_after_attempts_count # The maximum number of attempts during a request retry sequence (defaults to 24) # _socket_timeout @@ -144,15 +194,14 @@ def read(self) -> Optional[OAuthToken]: # _disable_pandas # In case the deserialisation through pandas causes any issues, it can be disabled with # this flag. - # _use_arrow_native_complex_types - # DBR will return native Arrow types for structs, arrays and maps instead of Arrow strings - # (True by default) # _use_arrow_native_decimals # Databricks runtime will return native Arrow types for decimals instead of Arrow strings # (True by default) # _use_arrow_native_timestamps # Databricks runtime will return native Arrow types for timestamps instead of Arrow strings # (True by default) + # use_cloud_fetch + # Enable use of cloud fetch to extract large query results in parallel via cloud storage if access_token: access_token_kv = {"access_token": access_token} @@ -177,23 +226,72 @@ def read(self) -> Optional[OAuthToken]: base_headers = [("User-Agent", useragent_header)] + self._ssl_options = SSLOptions( + # Double negation is generally a bad thing, but we have to keep backward compatibility + tls_verify=not kwargs.get( + "_tls_no_verify", False + ), # by default - verify cert and host + tls_verify_hostname=kwargs.get("_tls_verify_hostname", True), + tls_trusted_ca_file=kwargs.get("_tls_trusted_ca_file"), + tls_client_cert_file=kwargs.get("_tls_client_cert_file"), + tls_client_cert_key_file=kwargs.get("_tls_client_cert_key_file"), + tls_client_cert_key_password=kwargs.get("_tls_client_cert_key_password"), + ) + self.thrift_backend = ThriftBackend( self.host, self.port, http_path, (http_headers or []) + base_headers, auth_provider, + ssl_options=self._ssl_options, + _use_arrow_native_complex_types=_use_arrow_native_complex_types, **kwargs, ) - self._session_handle = self.thrift_backend.open_session( + self._open_session_resp = self.thrift_backend.open_session( session_configuration, catalog, schema ) + self._session_handle = self._open_session_resp.sessionHandle + self.protocol_version = self.get_protocol_version(self._open_session_resp) + self.use_cloud_fetch = kwargs.get("use_cloud_fetch", True) self.open = True - logger.info("Successfully opened session " + str(self.get_session_id())) + logger.info("Successfully opened session " + str(self.get_session_id_hex())) self._cursors = [] # type: List[Cursor] - def __enter__(self): + self.use_inline_params = self._set_use_inline_params_with_warning( + kwargs.get("use_inline_params", False) + ) + + def _set_use_inline_params_with_warning(self, value: Union[bool, str]): + """Valid values are True, False, and "silent" + + False: Use native parameters + True: Use inline parameters and log a warning + "silent": Use inline parameters and don't log a warning + """ + + if value is False: + return False + + if value not in [True, "silent"]: + raise ValueError( + f"Invalid value for use_inline_params: {value}. " + + 'Valid values are True, False, and "silent"' + ) + + if value is True: + logger.warning( + "Parameterised queries executed with this client will use the inline parameter approach." + "This approach will be deprecated in a future release. Consider using native parameters." + "Learn more: https://github.com/databricks/databricks-sql-python/tree/main/docs/parameters.md" + 'To suppress this warning, set use_inline_params="silent"' + ) + + return value + + # The ideal return type for this method is perhaps Self, but that was not added until 3.11, and we support pre-3.11 pythons, currently. + def __enter__(self) -> "Connection": return self def __exit__(self, exc_type, exc_value, traceback): @@ -203,7 +301,7 @@ def __del__(self): if self.open: logger.debug( "Closing unclosed connection for session " - "{}".format(self.get_session_id()) + "{}".format(self.get_session_id_hex()) ) try: self._close(close_cursors=False) @@ -214,6 +312,33 @@ def __del__(self): def get_session_id(self): return self.thrift_backend.handle_to_id(self._session_handle) + @staticmethod + def get_protocol_version(openSessionResp): + """ + Since the sessionHandle will sometimes have a serverProtocolVersion, it takes + precedence over the serverProtocolVersion defined in the OpenSessionResponse. + """ + if ( + openSessionResp.sessionHandle + and hasattr(openSessionResp.sessionHandle, "serverProtocolVersion") + and openSessionResp.sessionHandle.serverProtocolVersion + ): + return openSessionResp.sessionHandle.serverProtocolVersion + return openSessionResp.serverProtocolVersion + + @staticmethod + def server_parameterized_queries_enabled(protocolVersion): + if ( + protocolVersion + and protocolVersion >= ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V8 + ): + return True + else: + return False + + def get_session_id_hex(self): + return self.thrift_backend.handle_to_hex_id(self._session_handle) + def cursor( self, arraysize: int = DEFAULT_ARRAY_SIZE, @@ -244,7 +369,28 @@ def _close(self, close_cursors=True) -> None: if close_cursors: for cursor in self._cursors: cursor.close() - self.thrift_backend.close_session(self._session_handle) + + logger.info(f"Closing session {self.get_session_id_hex()}") + if not self.open: + logger.debug("Session appears to have been closed already") + + try: + self.thrift_backend.close_session(self._session_handle) + except RequestError as e: + if isinstance(e.args[1], SessionAlreadyClosedError): + logger.info("Session was closed by a prior request") + except DatabaseError as e: + if "Invalid SessionHandle" in str(e): + logger.warning( + f"Attempted to close session that was already closed: {e}" + ) + else: + logger.warning( + f"Attempt to close session raised an exception at the server: {e}" + ) + except Exception as e: + logger.error(f"Attempt to close session raised a local exception: {e}") + self.open = False def commit(self): @@ -283,7 +429,8 @@ def __init__( self.escaper = ParamEscaper() self.lastrowid = None - def __enter__(self): + # The ideal return type for this method is perhaps Self, but that was not added until 3.11, and we support pre-3.11 pythons, currently. + def __enter__(self) -> "Cursor": return self def __exit__(self, exc_type, exc_value, traceback): @@ -296,6 +443,132 @@ def __iter__(self): else: raise Error("There is no active result set") + def _determine_parameter_approach( + self, params: Optional[TParameterCollection] + ) -> ParameterApproach: + """Encapsulates the logic for choosing whether to send parameters in native vs inline mode + + If params is None then ParameterApproach.NONE is returned. + If self.use_inline_params is True then inline mode is used. + If self.use_inline_params is False, then check if the server supports them and proceed. + Else raise an exception. + + Returns a ParameterApproach enumeration or raises an exception + + If inline approach is used when the server supports native approach, a warning is logged + """ + + if params is None: + return ParameterApproach.NONE + + if self.connection.use_inline_params: + return ParameterApproach.INLINE + + else: + return ParameterApproach.NATIVE + + def _all_dbsql_parameters_are_named(self, params: List[TDbsqlParameter]) -> bool: + """Return True if all members of the list have a non-null .name attribute""" + return all([i.name is not None for i in params]) + + def _normalize_tparametersequence( + self, params: TParameterSequence + ) -> List[TDbsqlParameter]: + """Retains the same order as the input list.""" + + output: List[TDbsqlParameter] = [] + for p in params: + if isinstance(p, DbsqlParameterBase): + output.append(p) + else: + output.append(dbsql_parameter_from_primitive(value=p)) + + return output + + def _normalize_tparameterdict( + self, params: TParameterDict + ) -> List[TDbsqlParameter]: + return [ + dbsql_parameter_from_primitive(value=value, name=name) + for name, value in params.items() + ] + + def _normalize_tparametercollection( + self, params: Optional[TParameterCollection] + ) -> List[TDbsqlParameter]: + if params is None: + return [] + if isinstance(params, dict): + return self._normalize_tparameterdict(params) + if isinstance(params, Sequence): + return self._normalize_tparametersequence(list(params)) + + def _determine_parameter_structure( + self, + parameters: List[TDbsqlParameter], + ) -> ParameterStructure: + all_named = self._all_dbsql_parameters_are_named(parameters) + if all_named: + return ParameterStructure.NAMED + else: + return ParameterStructure.POSITIONAL + + def _prepare_inline_parameters( + self, stmt: str, params: Optional[Union[Sequence, Dict[str, Any]]] + ) -> Tuple[str, List]: + """Return a statement and list of native parameters to be passed to thrift_backend for execution + + :stmt: + A string SQL query containing parameter markers of PEP-249 paramstyle `pyformat`. + For example `%(param)s`. + + :params: + An iterable of parameter values to be rendered inline. If passed as a Dict, the keys + must match the names of the markers included in :stmt:. If passed as a List, its length + must equal the count of parameter markers in :stmt:. + + Returns a tuple of: + stmt: the passed statement with the param markers replaced by literal rendered values + params: an empty list representing the native parameters to be passed with this query. + The list is always empty because native parameters are never used under the inline approach + """ + + escaped_values = self.escaper.escape_args(params) + rendered_statement = inject_parameters(stmt, escaped_values) + + return rendered_statement, NO_NATIVE_PARAMS + + def _prepare_native_parameters( + self, + stmt: str, + params: List[TDbsqlParameter], + param_structure: ParameterStructure, + ) -> Tuple[str, List[TSparkParameter]]: + """Return a statement and a list of native parameters to be passed to thrift_backend for execution + + :stmt: + A string SQL query containing parameter markers of PEP-249 paramstyle `named`. + For example `:param`. + + :params: + An iterable of parameter values to be sent natively. If passed as a Dict, the keys + must match the names of the markers included in :stmt:. If passed as a List, its length + must equal the count of parameter markers in :stmt:. In list form, any member of the list + can be wrapped in a DbsqlParameter class. + + Returns a tuple of: + stmt: the passed statement` with the param markers replaced by literal rendered values + params: a list of TSparkParameters that will be passed in native mode + """ + + stmt = stmt + output = [ + p.as_tspark_param(named=param_structure == ParameterStructure.NAMED) + for p in params + ] + + return stmt, output + def _close_and_clear_active_result_set(self): try: if self.active_result_set: @@ -355,12 +628,15 @@ def _handle_staging_operation( "Local file operations are restricted to paths within the configured staging_allowed_local_path" ) - # TODO: Experiment with DBR sending real headers. - # The specification says headers will be in JSON format but the current null value is actually an empty list [] + # May be real headers, or could be json string + headers = ( + json.loads(row.headers) if isinstance(row.headers, str) else row.headers + ) + handler_args = { "presigned_url": row.presignedUrl, "local_file": abs_localFile, - "headers": json.loads(row.headers or "{}"), + "headers": dict(headers) or {}, } logger.debug( @@ -383,7 +659,7 @@ def _handle_staging_operation( ) def _handle_staging_put( - self, presigned_url: str, local_file: str, headers: dict = None + self, presigned_url: str, local_file: str, headers: Optional[dict] = None ): """Make an HTTP PUT request @@ -398,7 +674,7 @@ def _handle_staging_put( # fmt: off # Design borrowed from: https://stackoverflow.com/a/2342589/5093960 - + OK = requests.codes.ok # 200 CREATED = requests.codes.created # 201 ACCEPTED = requests.codes.accepted # 202 @@ -418,7 +694,7 @@ def _handle_staging_put( ) def _handle_staging_get( - self, local_file: str, presigned_url: str, headers: dict = None + self, local_file: str, presigned_url: str, headers: Optional[dict] = None ): """Make an HTTP GET request, create a local file with the received data @@ -440,7 +716,9 @@ def _handle_staging_get( with open(local_file, "wb") as fp: fp.write(r.content) - def _handle_staging_remove(self, presigned_url: str, headers: dict = None): + def _handle_staging_remove( + self, presigned_url: str, headers: Optional[dict] = None + ): """Make an HTTP DELETE request to the presigned_url""" r = requests.delete(url=presigned_url, headers=headers) @@ -451,31 +729,72 @@ def _handle_staging_remove(self, presigned_url: str, headers: dict = None): ) def execute( - self, operation: str, parameters: Optional[Dict[str, str]] = None + self, + operation: str, + parameters: Optional[TParameterCollection] = None, ) -> "Cursor": """ Execute a query and wait for execution to complete. - Parameters should be given in extended param format style: %(...). - For example: - operation = "SELECT * FROM table WHERE field = %(some_value)s" - parameters = {"some_value": "foo"} - Will result in the query "SELECT * FROM table WHERE field = 'foo' being sent to the server + + The parameterisation behaviour of this method depends on which parameter approach is used: + - With INLINE mode, parameters are rendered inline with the query text + - With NATIVE mode (default), parameters are sent to the server separately for binding + + This behaviour is controlled by the `use_inline_params` argument passed when building a connection. + + The paramstyle for these approaches is different: + + If the connection was instantiated with use_inline_params=False (default), then parameters + should be given in PEP-249 `named` paramstyle like :param_name. Parameters passed by positionally + are indicated using a `?` in the query text. + + If the connection was instantiated with use_inline_params=True, then parameters + should be given in PEP-249 `pyformat` paramstyle like %(param_name)s. Parameters passed by positionally + are indicated using a `%s` marker in the query. Note: this approach is not recommended as it can break + your SQL query syntax and will be removed in a future release. + + ```python + inline_operation = "SELECT * FROM table WHERE field = %(some_value)s" + native_operation = "SELECT * FROM table WHERE field = :some_value" + parameters = {"some_value": "foo"} + ``` + + Both will result in the query equivalent to "SELECT * FROM table WHERE field = 'foo' + being sent to the server + :returns self """ - if parameters is not None: - operation = inject_parameters( - operation, self.escaper.escape_args(parameters) + + param_approach = self._determine_parameter_approach(parameters) + if param_approach == ParameterApproach.NONE: + prepared_params = NO_NATIVE_PARAMS + prepared_operation = operation + + elif param_approach == ParameterApproach.INLINE: + prepared_operation, prepared_params = self._prepare_inline_parameters( + operation, parameters + ) + elif param_approach == ParameterApproach.NATIVE: + normalized_parameters = self._normalize_tparametercollection(parameters) + param_structure = self._determine_parameter_structure(normalized_parameters) + transformed_operation = transform_paramstyle( + operation, normalized_parameters, param_structure + ) + prepared_operation, prepared_params = self._prepare_native_parameters( + transformed_operation, normalized_parameters, param_structure ) self._check_not_closed() self._close_and_clear_active_result_set() execute_response = self.thrift_backend.execute_command( - operation=operation, + operation=prepared_operation, session_handle=self.connection._session_handle, max_rows=self.arraysize, max_bytes=self.buffer_size_bytes, lz4_compression=self.connection.lz4_compression, cursor=self, + use_cloud_fetch=self.connection.use_cloud_fetch, + parameters=prepared_params, ) self.active_result_set = ResultSet( self.connection, @@ -494,8 +813,10 @@ def execute( def executemany(self, operation, seq_of_parameters): """ - Prepare a database operation (query or command) and then execute it against all parameter - sequences or mappings found in the sequence ``seq_of_parameters``. + Execute the operation once for every set of passed in parameters. + + This will issue N sequential request to the database where N is the length of the provided sequence. + No optimizations of the query (like batching) will be performed. Only the final result set is retained. @@ -561,7 +882,7 @@ def tables( catalog_name: Optional[str] = None, schema_name: Optional[str] = None, table_name: Optional[str] = None, - table_types: List[str] = None, + table_types: Optional[List[str]] = None, ) -> "Cursor": """ Get tables corresponding to the catalog_name, schema_name and table_name. @@ -675,14 +996,14 @@ def fetchmany(self, size: int) -> List[Row]: else: raise Error("There is no active result set") - def fetchall_arrow(self) -> pyarrow.Table: + def fetchall_arrow(self) -> "pyarrow.Table": self._check_not_closed() if self.active_result_set: return self.active_result_set.fetchall_arrow() else: raise Error("There is no active result set") - def fetchmany_arrow(self, size) -> pyarrow.Table: + def fetchmany_arrow(self, size) -> "pyarrow.Table": self._check_not_closed() if self.active_result_set: return self.active_result_set.fetchmany_arrow(size) @@ -707,9 +1028,22 @@ def cancel(self) -> None: def close(self) -> None: """Close cursor""" self.open = False + self.active_op_handle = None if self.active_result_set: self._close_and_clear_active_result_set() + @property + def query_id(self) -> Optional[str]: + """ + This attribute is an identifier of last executed query. + + This attribute will be ``None`` if the cursor has not had an operation + invoked via the execute method yet, or if cursor was closed. + """ + if self.active_op_handle is not None: + return str(UUID(bytes=self.active_op_handle.operationId.guid)) + return None + @property def description(self) -> Optional[List[Tuple]]: """ @@ -801,6 +1135,7 @@ def __iter__(self): break def _fill_results_buffer(self): + # At initialization or if the server does not have cloud fetch result links available results, has_more_rows = self.thrift_backend.fetch_results( op_handle=self.command_id, max_rows=self.arraysize, @@ -813,6 +1148,18 @@ def _fill_results_buffer(self): self.results = results self.has_more_rows = has_more_rows + def _convert_columnar_table(self, table): + column_names = [c[0] for c in self.description] + ResultRow = Row(*column_names) + result = [] + for row_index in range(table.num_rows): + curr_row = [] + for col_index in range(table.num_columns): + curr_row.append(table.get_item(col_index, row_index)) + result.append(ResultRow(*curr_row)) + + return result + def _convert_arrow_table(self, table): column_names = [c[0] for c in self.description] ResultRow = Row(*column_names) @@ -848,14 +1195,14 @@ def _convert_arrow_table(self, table): timestamp_as_object=True, ) - res = df.to_numpy(na_value=None) + res = df.to_numpy(na_value=None, dtype="object") return [ResultRow(*v) for v in res] @property def rownumber(self): return self._next_row_index - def fetchmany_arrow(self, size: int) -> pyarrow.Table: + def fetchmany_arrow(self, size: int) -> "pyarrow.Table": """ Fetch the next set of rows of a query result, returning a PyArrow table. @@ -880,7 +1227,46 @@ def fetchmany_arrow(self, size: int) -> pyarrow.Table: return results - def fetchall_arrow(self) -> pyarrow.Table: + def merge_columnar(self, result1, result2): + """ + Function to merge / combining the columnar results into a single result + :param result1: + :param result2: + :return: + """ + + if result1.column_names != result2.column_names: + raise ValueError("The columns in the results don't match") + + merged_result = [result1.column_table[i] + result2.column_table[i] for i in range(result1.num_columns)] + return ColumnTable(merged_result, result1.column_names) + + def fetchmany_columnar(self, size: int): + """ + Fetch the next set of rows of a query result, returning a Columnar Table. + An empty sequence is returned when no more rows are available. + """ + if size < 0: + raise ValueError("size argument for fetchmany is %s but must be >= 0", size) + + results = self.results.next_n_rows(size) + n_remaining_rows = size - results.num_rows + self._next_row_index += results.num_rows + + while ( + n_remaining_rows > 0 + and not self.has_been_closed_server_side + and self.has_more_rows + ): + self._fill_results_buffer() + partial_results = self.results.next_n_rows(n_remaining_rows) + results = self.merge_columnar(results, partial_results) + n_remaining_rows -= partial_results.num_rows + self._next_row_index += partial_results.num_rows + + return results + + def fetchall_arrow(self) -> "pyarrow.Table": """Fetch all (remaining) rows of a query result, returning them as a PyArrow table.""" results = self.results.remaining_rows() self._next_row_index += results.num_rows @@ -893,12 +1279,30 @@ def fetchall_arrow(self) -> pyarrow.Table: return results + def fetchall_columnar(self): + """Fetch all (remaining) rows of a query result, returning them as a Columnar table.""" + results = self.results.remaining_rows() + self._next_row_index += results.num_rows + + while not self.has_been_closed_server_side and self.has_more_rows: + self._fill_results_buffer() + partial_results = self.results.remaining_rows() + results = self.merge_columnar(results, partial_results) + self._next_row_index += partial_results.num_rows + + return results + def fetchone(self) -> Optional[Row]: """ Fetch the next row of a query result set, returning a single sequence, or None when no more data is available. """ - res = self._convert_arrow_table(self.fetchmany_arrow(1)) + + if isinstance(self.results, ColumnQueue): + res = self._convert_columnar_table(self.fetchmany_columnar(1)) + else: + res = self._convert_arrow_table(self.fetchmany_arrow(1)) + if len(res) > 0: return res[0] else: @@ -908,7 +1312,10 @@ def fetchall(self) -> List[Row]: """ Fetch all (remaining) rows of a query result, returning them as a list of rows. """ - return self._convert_arrow_table(self.fetchall_arrow()) + if isinstance(self.results, ColumnQueue): + return self._convert_columnar_table(self.fetchall_columnar()) + else: + return self._convert_arrow_table(self.fetchall_arrow()) def fetchmany(self, size: int) -> List[Row]: """ @@ -916,7 +1323,10 @@ def fetchmany(self, size: int) -> List[Row]: An empty sequence is returned when no more rows are available. """ - return self._convert_arrow_table(self.fetchmany_arrow(size)) + if isinstance(self.results, ColumnQueue): + return self._convert_columnar_table(self.fetchmany_columnar(size)) + else: + return self._convert_arrow_table(self.fetchmany_arrow(size)) def close(self) -> None: """ @@ -932,6 +1342,9 @@ def close(self) -> None: and self.connection.open ): self.thrift_backend.close_command(self.command_id) + except RequestError as e: + if isinstance(e.args[1], CursorAlreadyClosedError): + logger.info("Operation was canceled by a prior request") finally: self.has_been_closed_server_side = True self.op_state = self.thrift_backend.CLOSED_OP_STATE diff --git a/src/databricks/sql/cloudfetch/download_manager.py b/src/databricks/sql/cloudfetch/download_manager.py new file mode 100644 index 00000000..7e96cd32 --- /dev/null +++ b/src/databricks/sql/cloudfetch/download_manager.py @@ -0,0 +1,108 @@ +import logging + +from concurrent.futures import ThreadPoolExecutor, Future +from typing import List, Union + +from databricks.sql.cloudfetch.downloader import ( + ResultSetDownloadHandler, + DownloadableResultSettings, + DownloadedFile, +) +from databricks.sql.types import SSLOptions + +from databricks.sql.thrift_api.TCLIService.ttypes import TSparkArrowResultLink + +logger = logging.getLogger(__name__) + + +class ResultFileDownloadManager: + def __init__( + self, + links: List[TSparkArrowResultLink], + max_download_threads: int, + lz4_compressed: bool, + ssl_options: SSLOptions, + ): + self._pending_links: List[TSparkArrowResultLink] = [] + for link in links: + if link.rowCount <= 0: + continue + logger.debug( + "ResultFileDownloadManager: adding file link, start offset {}, row count: {}".format( + link.startRowOffset, link.rowCount + ) + ) + self._pending_links.append(link) + + self._download_tasks: List[Future[DownloadedFile]] = [] + self._max_download_threads: int = max_download_threads + self._thread_pool = ThreadPoolExecutor(max_workers=self._max_download_threads) + + self._downloadable_result_settings = DownloadableResultSettings(lz4_compressed) + self._ssl_options = ssl_options + + def get_next_downloaded_file( + self, next_row_offset: int + ) -> Union[DownloadedFile, None]: + """ + Get next file that starts at given offset. + + This function gets the next downloaded file in which its rows start at the specified next_row_offset + in relation to the full result. File downloads are scheduled if not already, and once the correct + download handler is located, the function waits for the download status and returns the resulting file. + If there are no more downloads, a download was not successful, or the correct file could not be located, + this function shuts down the thread pool and returns None. + + Args: + next_row_offset (int): The offset of the starting row of the next file we want data from. + """ + + # Make sure the download queue is always full + self._schedule_downloads() + + # No more files to download from this batch of links + if len(self._download_tasks) == 0: + self._shutdown_manager() + return None + + task = self._download_tasks.pop(0) + # Future's `result()` method will wait for the call to complete, and return + # the value returned by the call. If the call throws an exception - `result()` + # will throw the same exception + file = task.result() + if (next_row_offset < file.start_row_offset) or ( + next_row_offset > file.start_row_offset + file.row_count + ): + logger.debug( + "ResultFileDownloadManager: file does not contain row {}, start {}, row count {}".format( + next_row_offset, file.start_row_offset, file.row_count + ) + ) + + return file + + def _schedule_downloads(self): + """ + While download queue has a capacity, peek pending links and submit them to thread pool. + """ + logger.debug("ResultFileDownloadManager: schedule downloads") + while (len(self._download_tasks) < self._max_download_threads) and ( + len(self._pending_links) > 0 + ): + link = self._pending_links.pop(0) + logger.debug( + "- start: {}, row count: {}".format(link.startRowOffset, link.rowCount) + ) + handler = ResultSetDownloadHandler( + settings=self._downloadable_result_settings, + link=link, + ssl_options=self._ssl_options, + ) + task = self._thread_pool.submit(handler.run) + self._download_tasks.append(task) + + def _shutdown_manager(self): + # Clear download handlers and shutdown the thread pool + self._pending_links = [] + self._download_tasks = [] + self._thread_pool.shutdown(wait=False) diff --git a/src/databricks/sql/cloudfetch/downloader.py b/src/databricks/sql/cloudfetch/downloader.py new file mode 100644 index 00000000..03c70054 --- /dev/null +++ b/src/databricks/sql/cloudfetch/downloader.py @@ -0,0 +1,175 @@ +import logging +from dataclasses import dataclass + +import requests +from requests.adapters import HTTPAdapter, Retry +import lz4.frame +import time + +from databricks.sql.thrift_api.TCLIService.ttypes import TSparkArrowResultLink +from databricks.sql.exc import Error +from databricks.sql.types import SSLOptions + +logger = logging.getLogger(__name__) + +# TODO: Ideally, we should use a common retry policy (DatabricksRetryPolicy) for all the requests across the library. +# But DatabricksRetryPolicy should be updated first - currently it can work only with Thrift requests +retryPolicy = Retry( + total=5, # max retry attempts + backoff_factor=1, # min delay, 1 second + # TODO: `backoff_max` is supported since `urllib3` v2.0.0, but we allow >= 1.26. + # The default value (120 seconds) used since v1.26 looks reasonable enough + # backoff_max=60, # max delay, 60 seconds + # retry all status codes below 100, 429 (Too Many Requests), and all codes above 500, + # excluding 501 Not implemented + status_forcelist=[*range(0, 101), 429, 500, *range(502, 1000)], +) + + +@dataclass +class DownloadedFile: + """ + Class for the result file and metadata. + + Attributes: + file_bytes (bytes): Downloaded file in bytes. + start_row_offset (int): The offset of the starting row in relation to the full result. + row_count (int): Number of rows the file represents in the result. + """ + + file_bytes: bytes + start_row_offset: int + row_count: int + + +@dataclass +class DownloadableResultSettings: + """ + Class for settings common to each download handler. + + Attributes: + is_lz4_compressed (bool): Whether file is expected to be lz4 compressed. + link_expiry_buffer_secs (int): Time in seconds to prevent download of a link before it expires. Default 0 secs. + download_timeout (int): Timeout for download requests. Default 60 secs. + max_consecutive_file_download_retries (int): Number of consecutive download retries before shutting down. + """ + + is_lz4_compressed: bool + link_expiry_buffer_secs: int = 0 + download_timeout: int = 60 + max_consecutive_file_download_retries: int = 0 + + +class ResultSetDownloadHandler: + def __init__( + self, + settings: DownloadableResultSettings, + link: TSparkArrowResultLink, + ssl_options: SSLOptions, + ): + self.settings = settings + self.link = link + self._ssl_options = ssl_options + + def run(self) -> DownloadedFile: + """ + Download the file described in the cloud fetch link. + + This function checks if the link has or is expiring, gets the file via a requests session, decompresses the + file, and signals to waiting threads that the download is finished and whether it was successful. + """ + + logger.debug( + "ResultSetDownloadHandler: starting file download, offset {}, row count {}".format( + self.link.startRowOffset, self.link.rowCount + ) + ) + + # Check if link is already expired or is expiring + ResultSetDownloadHandler._validate_link( + self.link, self.settings.link_expiry_buffer_secs + ) + + session = requests.Session() + session.mount("http://", HTTPAdapter(max_retries=retryPolicy)) + session.mount("https://", HTTPAdapter(max_retries=retryPolicy)) + + try: + # Get the file via HTTP request + response = session.get( + self.link.fileLink, + timeout=self.settings.download_timeout, + verify=self._ssl_options.tls_verify, + # TODO: Pass cert from `self._ssl_options` + ) + response.raise_for_status() + + # Save (and decompress if needed) the downloaded file + compressed_data = response.content + decompressed_data = ( + ResultSetDownloadHandler._decompress_data(compressed_data) + if self.settings.is_lz4_compressed + else compressed_data + ) + + # The size of the downloaded file should match the size specified from TSparkArrowResultLink + if len(decompressed_data) != self.link.bytesNum: + logger.debug( + "ResultSetDownloadHandler: downloaded file size {} does not match the expected value {}".format( + len(decompressed_data), self.link.bytesNum + ) + ) + + logger.debug( + "ResultSetDownloadHandler: successfully downloaded file, offset {}, row count {}".format( + self.link.startRowOffset, self.link.rowCount + ) + ) + + return DownloadedFile( + decompressed_data, + self.link.startRowOffset, + self.link.rowCount, + ) + finally: + if session: + session.close() + + @staticmethod + def _validate_link(link: TSparkArrowResultLink, expiry_buffer_secs: int): + """ + Check if a link has expired or will expire. + + Expiry buffer can be set to avoid downloading files that has not expired yet when the function is called, + but may expire before the file has fully downloaded. + """ + current_time = int(time.time()) + if ( + link.expiryTime <= current_time + or link.expiryTime - current_time <= expiry_buffer_secs + ): + raise Error("CloudFetch link has expired") + + @staticmethod + def _decompress_data(compressed_data: bytes) -> bytes: + """ + Decompress lz4 frame compressed data. + + Decompresses data that has been lz4 compressed, either via the whole frame or by series of chunks. + """ + uncompressed_data, bytes_read = lz4.frame.decompress( + compressed_data, return_bytes_read=True + ) + # The last cloud fetch file of the entire result is commonly punctuated by frequent end-of-frame markers. + # Full frame decompression above will short-circuit, so chunking is necessary + if bytes_read < len(compressed_data): + d_context = lz4.frame.create_decompression_context() + start = 0 + uncompressed_data = bytearray() + while start < len(compressed_data): + data, num_bytes, is_end = lz4.frame.decompress_chunk( + d_context, compressed_data[start:] + ) + uncompressed_data += data + start += num_bytes + return uncompressed_data diff --git a/src/databricks/sql/exc.py b/src/databricks/sql/exc.py index bb1e203e..3b27283a 100644 --- a/src/databricks/sql/exc.py +++ b/src/databricks/sql/exc.py @@ -93,3 +93,25 @@ class RequestError(OperationalError): """ pass + + +class MaxRetryDurationError(RequestError): + """Thrown if the next HTTP request retry would exceed the configured + stop_after_attempts_duration + """ + + +class NonRecoverableNetworkError(RequestError): + """Thrown if an HTTP code 501 is received""" + + +class UnsafeToRetryError(RequestError): + """Thrown if ExecuteStatement request receives a code other than 200, 429, or 503""" + + +class SessionAlreadyClosedError(RequestError): + """Thrown if CloseSession receives a code 404. ThriftBackend should gracefully proceed as this is expected.""" + + +class CursorAlreadyClosedError(RequestError): + """Thrown if CancelOperation receives a code 404. ThriftBackend should gracefully proceed as this is expected.""" diff --git a/src/databricks/sql/experimental/oauth_persistence.py b/src/databricks/sql/experimental/oauth_persistence.py index bd0066d9..13a96612 100644 --- a/src/databricks/sql/experimental/oauth_persistence.py +++ b/src/databricks/sql/experimental/oauth_persistence.py @@ -27,6 +27,17 @@ def read(self, hostname: str) -> Optional[OAuthToken]: pass +class OAuthPersistenceCache(OAuthPersistence): + def __init__(self): + self.tokens = {} + + def persist(self, hostname: str, oauth_token: OAuthToken): + self.tokens[hostname] = oauth_token + + def read(self, hostname: str) -> Optional[OAuthToken]: + return self.tokens.get(hostname) + + # Note this is only intended to be used for development class DevOnlyFilePersistence(OAuthPersistence): def __init__(self, file_path): diff --git a/src/databricks/sql/parameters/__init__.py b/src/databricks/sql/parameters/__init__.py new file mode 100644 index 00000000..3c39cf2b --- /dev/null +++ b/src/databricks/sql/parameters/__init__.py @@ -0,0 +1,15 @@ +from databricks.sql.parameters.native import ( + IntegerParameter, + StringParameter, + BigIntegerParameter, + BooleanParameter, + DateParameter, + DoubleParameter, + FloatParameter, + VoidParameter, + SmallIntParameter, + TimestampParameter, + TimestampNTZParameter, + TinyIntParameter, + DecimalParameter, +) diff --git a/src/databricks/sql/parameters/native.py b/src/databricks/sql/parameters/native.py new file mode 100644 index 00000000..8a436355 --- /dev/null +++ b/src/databricks/sql/parameters/native.py @@ -0,0 +1,606 @@ +import datetime +import decimal +from enum import Enum, auto +from typing import Optional, Sequence + +from databricks.sql.exc import NotSupportedError +from databricks.sql.thrift_api.TCLIService.ttypes import ( + TSparkParameter, + TSparkParameterValue, +) + +import datetime +import decimal +from enum import Enum, auto +from typing import Dict, List, Union + + +class ParameterApproach(Enum): + INLINE = 1 + NATIVE = 2 + NONE = 3 + + +class ParameterStructure(Enum): + NAMED = 1 + POSITIONAL = 2 + NONE = 3 + + +class DatabricksSupportedType(Enum): + """Enumerate every supported Databricks SQL type shown here: + + https://docs.databricks.com/en/sql/language-manual/sql-ref-datatypes.html + """ + + BIGINT = auto() + BINARY = auto() + BOOLEAN = auto() + DATE = auto() + DECIMAL = auto() + DOUBLE = auto() + FLOAT = auto() + INT = auto() + INTERVAL = auto() + VOID = auto() + SMALLINT = auto() + STRING = auto() + TIMESTAMP = auto() + TIMESTAMP_NTZ = auto() + TINYINT = auto() + ARRAY = auto() + MAP = auto() + STRUCT = auto() + + +TAllowedParameterValue = Union[ + str, int, float, datetime.datetime, datetime.date, bool, decimal.Decimal, None +] + + +class DbsqlParameterBase: + """Parent class for IntegerParameter, DecimalParameter etc.. + + Each each instance that extends this base class should be capable of generating a TSparkParameter + It should know how to generate a cast expression based off its DatabricksSupportedType. + + By default the cast expression should render the string value of it's `value` and the literal + name of its Databricks Supported Type + + Interface should be: + + from databricks.sql.parameters import DecimalParameter + param = DecimalParameter(value, scale=None, precision=None) + cursor.execute("SELECT ?",[param]) + + Or + + from databricks.sql.parameters import IntegerParameter + param = IntegerParameter(42) + cursor.execute("SELECT ?", [param]) + """ + + CAST_EXPR: str + name: Optional[str] + + def as_tspark_param(self, named: bool) -> TSparkParameter: + """Returns a TSparkParameter object that can be passed to the DBR thrift server.""" + + tsp = TSparkParameter(value=self._tspark_param_value(), type=self._cast_expr()) + + if named: + tsp.name = self.name + tsp.ordinal = False + elif not named: + tsp.ordinal = True + return tsp + + def _tspark_param_value(self): + return TSparkParameterValue(stringValue=str(self.value)) + + def _cast_expr(self): + return self.CAST_EXPR + + def __str__(self): + return f"{self.__class__}(name={self.name}, value={self.value})" + + def __repr__(self): + return self.__str__() + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + +class IntegerParameter(DbsqlParameterBase): + """Wrap a Python `int` that will be bound to a Databricks SQL INT column.""" + + def __init__(self, value: int, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to an INT. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.INT.name + + +class StringParameter(DbsqlParameterBase): + """Wrap a Python `str` that will be bound to a Databricks SQL STRING column.""" + + def __init__(self, value: str, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to a STRING. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.STRING.name + + +class BigIntegerParameter(DbsqlParameterBase): + """Wrap a Python `int` that will be bound to a Databricks SQL BIGINT column.""" + + def __init__(self, value: int, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to a BIGINT. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.BIGINT.name + + +class BooleanParameter(DbsqlParameterBase): + """Wrap a Python `bool` that will be bound to a Databricks SQL BOOLEAN column.""" + + def __init__(self, value: bool, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to a BOOLEAN. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.BOOLEAN.name + + +class DateParameter(DbsqlParameterBase): + """Wrap a Python `date` that will be bound to a Databricks SQL DATE column.""" + + def __init__(self, value: datetime.date, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to a DATE. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.DATE.name + + +class DoubleParameter(DbsqlParameterBase): + """Wrap a Python `float` that will be bound to a Databricks SQL DOUBLE column.""" + + def __init__(self, value: float, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to a DOUBLE. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.DOUBLE.name + + +class FloatParameter(DbsqlParameterBase): + """Wrap a Python `float` that will be bound to a Databricks SQL FLOAT column.""" + + def __init__(self, value: float, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to a FLOAT. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.FLOAT.name + + +class VoidParameter(DbsqlParameterBase): + """Wrap a Python `None` that will be bound to a Databricks SQL VOID type.""" + + def __init__(self, value: None, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to a VOID. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.VOID.name + + def _tspark_param_value(self): + """For Void types, the TSparkParameter.value should be a Python NoneType""" + return None + + +class SmallIntParameter(DbsqlParameterBase): + """Wrap a Python `int` that will be bound to a Databricks SQL SMALLINT type.""" + + def __init__(self, value: int, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to a SMALLINT. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.SMALLINT.name + + +class TimestampParameter(DbsqlParameterBase): + """Wrap a Python `datetime` that will be bound to a Databricks SQL TIMESTAMP type.""" + + def __init__(self, value: datetime.datetime, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to a TIMESTAMP. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.TIMESTAMP.name + + +class TimestampNTZParameter(DbsqlParameterBase): + """Wrap a Python `datetime` that will be bound to a Databricks SQL TIMESTAMP_NTZ type.""" + + def __init__(self, value: datetime.datetime, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to a TIMESTAMP_NTZ. + If it contains a timezone, that info will be lost. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.TIMESTAMP_NTZ.name + + +class TinyIntParameter(DbsqlParameterBase): + """Wrap a Python `int` that will be bound to a Databricks SQL TINYINT type.""" + + def __init__(self, value: int, name: Optional[str] = None): + """ + :value: + The value to bind for this parameter. This will be casted to a TINYINT. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + """ + self.value = value + self.name = name + + CAST_EXPR = DatabricksSupportedType.TINYINT.name + + +class DecimalParameter(DbsqlParameterBase): + """Wrap a Python `Decimal` that will be bound to a Databricks SQL DECIMAL type.""" + + CAST_EXPR = "DECIMAL({},{})" + + def __init__( + self, + value: decimal.Decimal, + name: Optional[str] = None, + scale: Optional[int] = None, + precision: Optional[int] = None, + ): + """ + If set, `scale` and `precision` must both be set. If neither is set, the value + will be casted to the smallest possible DECIMAL type that can contain it. + + :value: + The value to bind for this parameter. This will be casted to a DECIMAL. + :name: + If None, your query must contain a `?` marker. Like: + + ```sql + SELECT * FROM table WHERE field = ? + ``` + If not None, your query should contain a named parameter marker. Like: + ```sql + SELECT * FROM table WHERE field = :my_param + ``` + + The `name` argument to this function would be `my_param`. + :scale: + The maximum precision (total number of digits) of the number between 1 and 38. + :precision: + The number of digits to the right of the decimal point. + """ + self.value: decimal.Decimal = value + self.name = name + self.scale = scale + self.precision = precision + + if not self.valid_scale_and_precision(): + raise ValueError( + "DecimalParameter requires both or none of scale and precision to be set" + ) + + def valid_scale_and_precision(self): + if (self.scale is None and self.precision is None) or ( + isinstance(self.scale, int) and isinstance(self.precision, int) + ): + return True + else: + return False + + def _cast_expr(self): + if self.scale and self.precision: + return self.CAST_EXPR.format(self.scale, self.precision) + else: + return self.calculate_decimal_cast_string(self.value) + + def calculate_decimal_cast_string(self, input: decimal.Decimal) -> str: + """Returns the smallest SQL cast argument that can contain the passed decimal + + Example: + Input: Decimal("1234.5678") + Output: DECIMAL(8,4) + """ + + string_decimal = str(input) + + if string_decimal.startswith("0."): + # This decimal is less than 1 + overall = after = len(string_decimal) - 2 + elif "." not in string_decimal: + # This decimal has no fractional component + overall = len(string_decimal) + after = 0 + else: + # This decimal has both whole and fractional parts + parts = string_decimal.split(".") + parts_lengths = [len(i) for i in parts] + before, after = parts_lengths[:2] + overall = before + after + + return self.CAST_EXPR.format(overall, after) + + +def dbsql_parameter_from_int(value: int, name: Optional[str] = None): + """Returns IntegerParameter unless the passed int() requires a BIGINT. + + Note: TinyIntegerParameter is never inferred here because it is a rarely used type and clauses like LIMIT and OFFSET + cannot accept TINYINT bound parameter values. + """ + if -128 <= value <= 127: + # If DBR is ever updated to permit TINYINT values passed to LIMIT and OFFSET + # then we can change this line to return TinyIntParameter + return IntegerParameter(value=value, name=name) + elif -2147483648 <= value <= 2147483647: + return IntegerParameter(value=value, name=name) + else: + return BigIntegerParameter(value=value, name=name) + + +def dbsql_parameter_from_primitive( + value: TAllowedParameterValue, name: Optional[str] = None +) -> "TDbsqlParameter": + """Returns a DbsqlParameter subclass given an inferrable value + + This is a convenience function that can be used to create a DbsqlParameter subclass + without having to explicitly import a subclass of DbsqlParameter. + """ + + # This series of type checks are required for mypy not to raise + # havoc. We can't use TYPE_INFERRENCE_MAP because mypy doesn't trust + # its logic + + if type(value) is int: + return dbsql_parameter_from_int(value, name=name) + elif type(value) is str: + return StringParameter(value=value, name=name) + elif type(value) is float: + return FloatParameter(value=value, name=name) + elif type(value) is datetime.datetime: + return TimestampParameter(value=value, name=name) + elif type(value) is datetime.date: + return DateParameter(value=value, name=name) + elif type(value) is bool: + return BooleanParameter(value=value, name=name) + elif type(value) is decimal.Decimal: + return DecimalParameter(value=value, name=name) + elif value is None: + return VoidParameter(value=value, name=name) + + else: + raise NotSupportedError( + f"Could not infer parameter type from value: {value} - {type(value)} \n" + "Please specify the type explicitly." + ) + + +TDbsqlParameter = Union[ + IntegerParameter, + StringParameter, + BigIntegerParameter, + BooleanParameter, + DateParameter, + DoubleParameter, + FloatParameter, + VoidParameter, + SmallIntParameter, + TimestampParameter, + TimestampNTZParameter, + TinyIntParameter, + DecimalParameter, +] + + +TParameterSequence = Sequence[Union[TDbsqlParameter, TAllowedParameterValue]] +TParameterDict = Dict[str, TAllowedParameterValue] +TParameterCollection = Union[TParameterSequence, TParameterDict] + + +_all__ = [ + "IntegerParameter", + "StringParameter", + "BigIntegerParameter", + "BooleanParameter", + "DateParameter", + "DoubleParameter", + "FloatParameter", + "VoidParameter", + "SmallIntParameter", + "TimestampParameter", + "TimestampNTZParameter", + "TinyIntParameter", + "DecimalParameter", +] diff --git a/src/databricks/sql/parameters/py.typed b/src/databricks/sql/parameters/py.typed new file mode 100644 index 00000000..e69de29b diff --git a/src/databricks/sql/py.typed b/src/databricks/sql/py.typed new file mode 100755 index 00000000..e69de29b diff --git a/src/databricks/sql/thrift_api/TCLIService/TCLIService-remote b/src/databricks/sql/thrift_api/TCLIService/TCLIService-remote index 5271f955..552b21e5 100755 --- a/src/databricks/sql/thrift_api/TCLIService/TCLIService-remote +++ b/src/databricks/sql/thrift_api/TCLIService/TCLIService-remote @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Autogenerated by Thrift Compiler (0.17.0) +# Autogenerated by Thrift Compiler (0.19.0) # # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING # @@ -45,7 +45,6 @@ if len(sys.argv) <= 1 or sys.argv[1] == '--help': print(' TGetDelegationTokenResp GetDelegationToken(TGetDelegationTokenReq req)') print(' TCancelDelegationTokenResp CancelDelegationToken(TCancelDelegationTokenReq req)') print(' TRenewDelegationTokenResp RenewDelegationToken(TRenewDelegationTokenReq req)') - print(' TDBSqlGetLoadInformationResp GetLoadInformation(TDBSqlGetLoadInformationReq req)') print('') sys.exit(0) @@ -251,12 +250,6 @@ elif cmd == 'RenewDelegationToken': sys.exit(1) pp.pprint(client.RenewDelegationToken(eval(args[0]),)) -elif cmd == 'GetLoadInformation': - if len(args) != 1: - print('GetLoadInformation requires 1 args') - sys.exit(1) - pp.pprint(client.GetLoadInformation(eval(args[0]),)) - else: print('Unrecognized method %s' % cmd) sys.exit(1) diff --git a/src/databricks/sql/thrift_api/TCLIService/TCLIService.py b/src/databricks/sql/thrift_api/TCLIService/TCLIService.py index 6dde6702..071e78a9 100644 --- a/src/databricks/sql/thrift_api/TCLIService/TCLIService.py +++ b/src/databricks/sql/thrift_api/TCLIService/TCLIService.py @@ -1,5 +1,5 @@ # -# Autogenerated by Thrift Compiler (0.17.0) +# Autogenerated by Thrift Compiler (0.19.0) # # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING # @@ -187,14 +187,6 @@ def RenewDelegationToken(self, req): """ pass - def GetLoadInformation(self, req): - """ - Parameters: - - req - - """ - pass - class Client(Iface): def __init__(self, iprot, oprot=None): @@ -875,38 +867,6 @@ def recv_RenewDelegationToken(self): return result.success raise TApplicationException(TApplicationException.MISSING_RESULT, "RenewDelegationToken failed: unknown result") - def GetLoadInformation(self, req): - """ - Parameters: - - req - - """ - self.send_GetLoadInformation(req) - return self.recv_GetLoadInformation() - - def send_GetLoadInformation(self, req): - self._oprot.writeMessageBegin('GetLoadInformation', TMessageType.CALL, self._seqid) - args = GetLoadInformation_args() - args.req = req - args.write(self._oprot) - self._oprot.writeMessageEnd() - self._oprot.trans.flush() - - def recv_GetLoadInformation(self): - iprot = self._iprot - (fname, mtype, rseqid) = iprot.readMessageBegin() - if mtype == TMessageType.EXCEPTION: - x = TApplicationException() - x.read(iprot) - iprot.readMessageEnd() - raise x - result = GetLoadInformation_result() - result.read(iprot) - iprot.readMessageEnd() - if result.success is not None: - return result.success - raise TApplicationException(TApplicationException.MISSING_RESULT, "GetLoadInformation failed: unknown result") - class Processor(Iface, TProcessor): def __init__(self, handler): @@ -933,7 +893,6 @@ def __init__(self, handler): self._processMap["GetDelegationToken"] = Processor.process_GetDelegationToken self._processMap["CancelDelegationToken"] = Processor.process_CancelDelegationToken self._processMap["RenewDelegationToken"] = Processor.process_RenewDelegationToken - self._processMap["GetLoadInformation"] = Processor.process_GetLoadInformation self._on_message_begin = None def on_message_begin(self, func): @@ -1439,29 +1398,6 @@ def process_RenewDelegationToken(self, seqid, iprot, oprot): oprot.writeMessageEnd() oprot.trans.flush() - def process_GetLoadInformation(self, seqid, iprot, oprot): - args = GetLoadInformation_args() - args.read(iprot) - iprot.readMessageEnd() - result = GetLoadInformation_result() - try: - result.success = self._handler.GetLoadInformation(args.req) - msg_type = TMessageType.REPLY - except TTransport.TTransportException: - raise - except TApplicationException as ex: - logging.exception('TApplication exception in handler') - msg_type = TMessageType.EXCEPTION - result = ex - except Exception: - logging.exception('Unexpected exception in handler') - msg_type = TMessageType.EXCEPTION - result = TApplicationException(TApplicationException.INTERNAL_ERROR, 'Internal error') - oprot.writeMessageBegin("GetLoadInformation", msg_type, seqid) - result.write(oprot) - oprot.writeMessageEnd() - oprot.trans.flush() - # HELPER FUNCTIONS AND STRUCTURES @@ -4088,130 +4024,5 @@ def __ne__(self, other): RenewDelegationToken_result.thrift_spec = ( (0, TType.STRUCT, 'success', [TRenewDelegationTokenResp, None], None, ), # 0 ) - - -class GetLoadInformation_args(object): - """ - Attributes: - - req - - """ - - - def __init__(self, req=None,): - self.req = req - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRUCT: - self.req = TDBSqlGetLoadInformationReq() - self.req.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('GetLoadInformation_args') - if self.req is not None: - oprot.writeFieldBegin('req', TType.STRUCT, 1) - self.req.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(GetLoadInformation_args) -GetLoadInformation_args.thrift_spec = ( - None, # 0 - (1, TType.STRUCT, 'req', [TDBSqlGetLoadInformationReq, None], None, ), # 1 -) - - -class GetLoadInformation_result(object): - """ - Attributes: - - success - - """ - - - def __init__(self, success=None,): - self.success = success - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 0: - if ftype == TType.STRUCT: - self.success = TDBSqlGetLoadInformationResp() - self.success.read(iprot) - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('GetLoadInformation_result') - if self.success is not None: - oprot.writeFieldBegin('success', TType.STRUCT, 0) - self.success.write(oprot) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) -all_structs.append(GetLoadInformation_result) -GetLoadInformation_result.thrift_spec = ( - (0, TType.STRUCT, 'success', [TDBSqlGetLoadInformationResp, None], None, ), # 0 -) fix_spec(all_structs) del all_structs diff --git a/src/databricks/sql/thrift_api/TCLIService/constants.py b/src/databricks/sql/thrift_api/TCLIService/constants.py index 66dfc322..2cdf2f41 100644 --- a/src/databricks/sql/thrift_api/TCLIService/constants.py +++ b/src/databricks/sql/thrift_api/TCLIService/constants.py @@ -1,5 +1,5 @@ # -# Autogenerated by Thrift Compiler (0.17.0) +# Autogenerated by Thrift Compiler (0.19.0) # # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING # diff --git a/src/databricks/sql/thrift_api/TCLIService/ttypes.py b/src/databricks/sql/thrift_api/TCLIService/ttypes.py index 07ecfe48..16abbc2e 100644 --- a/src/databricks/sql/thrift_api/TCLIService/ttypes.py +++ b/src/databricks/sql/thrift_api/TCLIService/ttypes.py @@ -1,5 +1,5 @@ # -# Autogenerated by Thrift Compiler (0.17.0) +# Autogenerated by Thrift Compiler (0.19.0) # # DO NOT EDIT UNLESS YOU ARE SURE THAT YOU KNOW WHAT YOU ARE DOING # @@ -36,6 +36,7 @@ class TProtocolVersion(object): SPARK_CLI_SERVICE_PROTOCOL_V5 = 42245 SPARK_CLI_SERVICE_PROTOCOL_V6 = 42246 SPARK_CLI_SERVICE_PROTOCOL_V7 = 42247 + SPARK_CLI_SERVICE_PROTOCOL_V8 = 42248 _VALUES_TO_NAMES = { -7: "__HIVE_JDBC_WORKAROUND", @@ -57,6 +58,7 @@ class TProtocolVersion(object): 42245: "SPARK_CLI_SERVICE_PROTOCOL_V5", 42246: "SPARK_CLI_SERVICE_PROTOCOL_V6", 42247: "SPARK_CLI_SERVICE_PROTOCOL_V7", + 42248: "SPARK_CLI_SERVICE_PROTOCOL_V8", } _NAMES_TO_VALUES = { @@ -79,6 +81,7 @@ class TProtocolVersion(object): "SPARK_CLI_SERVICE_PROTOCOL_V5": 42245, "SPARK_CLI_SERVICE_PROTOCOL_V6": 42246, "SPARK_CLI_SERVICE_PROTOCOL_V7": 42247, + "SPARK_CLI_SERVICE_PROTOCOL_V8": 42248, } @@ -178,6 +181,39 @@ class TSparkRowSetType(object): } +class TDBSqlCompressionCodec(object): + NONE = 0 + LZ4_FRAME = 1 + LZ4_BLOCK = 2 + + _VALUES_TO_NAMES = { + 0: "NONE", + 1: "LZ4_FRAME", + 2: "LZ4_BLOCK", + } + + _NAMES_TO_VALUES = { + "NONE": 0, + "LZ4_FRAME": 1, + "LZ4_BLOCK": 2, + } + + +class TDBSqlArrowLayout(object): + ARROW_BATCH = 0 + ARROW_STREAMING = 1 + + _VALUES_TO_NAMES = { + 0: "ARROW_BATCH", + 1: "ARROW_STREAMING", + } + + _NAMES_TO_VALUES = { + "ARROW_BATCH": 0, + "ARROW_STREAMING": 1, + } + + class TOperationIdempotencyType(object): UNKNOWN = 0 NON_IDEMPOTENT = 1 @@ -475,6 +511,21 @@ class TResultPersistenceMode(object): } +class TDBSqlCloseOperationReason(object): + NONE = 0 + COMMAND_INACTIVITY_TIMEOUT = 1 + + _VALUES_TO_NAMES = { + 0: "NONE", + 1: "COMMAND_INACTIVITY_TIMEOUT", + } + + _NAMES_TO_VALUES = { + "NONE": 0, + "COMMAND_INACTIVITY_TIMEOUT": 1, + } + + class TCacheLookupResult(object): CACHE_INELIGIBLE = 0 LOCAL_CACHE_HIT = 1 @@ -529,6 +580,18 @@ class TCloudFetchDisabledReason(object): } +class TDBSqlManifestFileFormat(object): + THRIFT_GET_RESULT_SET_METADATA_RESP = 0 + + _VALUES_TO_NAMES = { + 0: "THRIFT_GET_RESULT_SET_METADATA_RESP", + } + + _NAMES_TO_VALUES = { + "THRIFT_GET_RESULT_SET_METADATA_RESP": 0, + } + + class TFetchOrientation(object): FETCH_NEXT = 0 FETCH_PRIOR = 1 @@ -556,6 +619,27 @@ class TFetchOrientation(object): } +class TDBSqlFetchDisposition(object): + DISPOSITION_UNSPECIFIED = 0 + DISPOSITION_INLINE = 1 + DISPOSITION_EXTERNAL_LINKS = 2 + DISPOSITION_INTERNAL_DBFS = 3 + + _VALUES_TO_NAMES = { + 0: "DISPOSITION_UNSPECIFIED", + 1: "DISPOSITION_INLINE", + 2: "DISPOSITION_EXTERNAL_LINKS", + 3: "DISPOSITION_INTERNAL_DBFS", + } + + _NAMES_TO_VALUES = { + "DISPOSITION_UNSPECIFIED": 0, + "DISPOSITION_INLINE": 1, + "DISPOSITION_EXTERNAL_LINKS": 2, + "DISPOSITION_INTERNAL_DBFS": 3, + } + + class TJobExecutionStatus(object): IN_PROGRESS = 0 COMPLETE = 1 @@ -712,7 +796,7 @@ def __ne__(self, other): return not (self == other) -class TPrimitiveTypeEntry(object): +class TTAllowedParameterValueEntry(object): """ Attributes: - type @@ -754,7 +838,7 @@ def write(self, oprot): if oprot._fast_encode is not None and self.thrift_spec is not None: oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) return - oprot.writeStructBegin('TPrimitiveTypeEntry') + oprot.writeStructBegin('TTAllowedParameterValueEntry') if self.type is not None: oprot.writeFieldBegin('type', TType.I32, 1) oprot.writeI32(self.type) @@ -1143,7 +1227,7 @@ def read(self, iprot): break if fid == 1: if ftype == TType.STRUCT: - self.primitiveEntry = TPrimitiveTypeEntry() + self.primitiveEntry = TTAllowedParameterValueEntry() self.primitiveEntry.read(iprot) else: iprot.skip(ftype) @@ -2841,6 +2925,270 @@ def __ne__(self, other): return not (self == other) +class TDBSqlJsonArrayFormat(object): + """ + Attributes: + - compressionCodec + + """ + + + def __init__(self, compressionCodec=None,): + self.compressionCodec = compressionCodec + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.compressionCodec = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TDBSqlJsonArrayFormat') + if self.compressionCodec is not None: + oprot.writeFieldBegin('compressionCodec', TType.I32, 1) + oprot.writeI32(self.compressionCodec) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TDBSqlCsvFormat(object): + """ + Attributes: + - compressionCodec + + """ + + + def __init__(self, compressionCodec=None,): + self.compressionCodec = compressionCodec + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.compressionCodec = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TDBSqlCsvFormat') + if self.compressionCodec is not None: + oprot.writeFieldBegin('compressionCodec', TType.I32, 1) + oprot.writeI32(self.compressionCodec) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TDBSqlArrowFormat(object): + """ + Attributes: + - arrowLayout + - compressionCodec + + """ + + + def __init__(self, arrowLayout=None, compressionCodec=None,): + self.arrowLayout = arrowLayout + self.compressionCodec = compressionCodec + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.arrowLayout = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.I32: + self.compressionCodec = iprot.readI32() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TDBSqlArrowFormat') + if self.arrowLayout is not None: + oprot.writeFieldBegin('arrowLayout', TType.I32, 1) + oprot.writeI32(self.arrowLayout) + oprot.writeFieldEnd() + if self.compressionCodec is not None: + oprot.writeFieldBegin('compressionCodec', TType.I32, 2) + oprot.writeI32(self.compressionCodec) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TDBSqlResultFormat(object): + """ + Attributes: + - arrowFormat + - csvFormat + - jsonArrayFormat + + """ + + + def __init__(self, arrowFormat=None, csvFormat=None, jsonArrayFormat=None,): + self.arrowFormat = arrowFormat + self.csvFormat = csvFormat + self.jsonArrayFormat = jsonArrayFormat + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRUCT: + self.arrowFormat = TDBSqlArrowFormat() + self.arrowFormat.read(iprot) + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.csvFormat = TDBSqlCsvFormat() + self.csvFormat.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRUCT: + self.jsonArrayFormat = TDBSqlJsonArrayFormat() + self.jsonArrayFormat.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TDBSqlResultFormat') + if self.arrowFormat is not None: + oprot.writeFieldBegin('arrowFormat', TType.STRUCT, 1) + self.arrowFormat.write(oprot) + oprot.writeFieldEnd() + if self.csvFormat is not None: + oprot.writeFieldBegin('csvFormat', TType.STRUCT, 2) + self.csvFormat.write(oprot) + oprot.writeFieldEnd() + if self.jsonArrayFormat is not None: + oprot.writeFieldBegin('jsonArrayFormat', TType.STRUCT, 3) + self.jsonArrayFormat.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + class TSparkArrowBatch(object): """ Attributes: @@ -2921,16 +3269,18 @@ class TSparkArrowResultLink(object): - startRowOffset - rowCount - bytesNum + - httpHeaders """ - def __init__(self, fileLink=None, expiryTime=None, startRowOffset=None, rowCount=None, bytesNum=None,): + def __init__(self, fileLink=None, expiryTime=None, startRowOffset=None, rowCount=None, bytesNum=None, httpHeaders=None,): self.fileLink = fileLink self.expiryTime = expiryTime self.startRowOffset = startRowOffset self.rowCount = rowCount self.bytesNum = bytesNum + self.httpHeaders = httpHeaders def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -2966,6 +3316,17 @@ def read(self, iprot): self.bytesNum = iprot.readI64() else: iprot.skip(ftype) + elif fid == 6: + if ftype == TType.MAP: + self.httpHeaders = {} + (_ktype105, _vtype106, _size104) = iprot.readMapBegin() + for _i108 in range(_size104): + _key109 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val110 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.httpHeaders[_key109] = _val110 + iprot.readMapEnd() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -2996,6 +3357,14 @@ def write(self, oprot): oprot.writeFieldBegin('bytesNum', TType.I64, 5) oprot.writeI64(self.bytesNum) oprot.writeFieldEnd() + if self.httpHeaders is not None: + oprot.writeFieldBegin('httpHeaders', TType.MAP, 6) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.httpHeaders)) + for kiter111, viter112 in self.httpHeaders.items(): + oprot.writeString(kiter111.encode('utf-8') if sys.version_info[0] == 2 else kiter111) + oprot.writeString(viter112.encode('utf-8') if sys.version_info[0] == 2 else viter112) + oprot.writeMapEnd() + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -3032,16 +3401,22 @@ class TDBSqlCloudResultFile(object): - rowCount - uncompressedBytes - compressedBytes + - fileLink + - linkExpiryTime + - httpHeaders """ - def __init__(self, filePath=None, startRowOffset=None, rowCount=None, uncompressedBytes=None, compressedBytes=None,): + def __init__(self, filePath=None, startRowOffset=None, rowCount=None, uncompressedBytes=None, compressedBytes=None, fileLink=None, linkExpiryTime=None, httpHeaders=None,): self.filePath = filePath self.startRowOffset = startRowOffset self.rowCount = rowCount self.uncompressedBytes = uncompressedBytes self.compressedBytes = compressedBytes + self.fileLink = fileLink + self.linkExpiryTime = linkExpiryTime + self.httpHeaders = httpHeaders def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -3077,6 +3452,27 @@ def read(self, iprot): self.compressedBytes = iprot.readI64() else: iprot.skip(ftype) + elif fid == 6: + if ftype == TType.STRING: + self.fileLink = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 7: + if ftype == TType.I64: + self.linkExpiryTime = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 8: + if ftype == TType.MAP: + self.httpHeaders = {} + (_ktype114, _vtype115, _size113) = iprot.readMapBegin() + for _i117 in range(_size113): + _key118 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val119 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.httpHeaders[_key118] = _val119 + iprot.readMapEnd() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -3107,20 +3503,26 @@ def write(self, oprot): oprot.writeFieldBegin('compressedBytes', TType.I64, 5) oprot.writeI64(self.compressedBytes) oprot.writeFieldEnd() + if self.fileLink is not None: + oprot.writeFieldBegin('fileLink', TType.STRING, 6) + oprot.writeString(self.fileLink.encode('utf-8') if sys.version_info[0] == 2 else self.fileLink) + oprot.writeFieldEnd() + if self.linkExpiryTime is not None: + oprot.writeFieldBegin('linkExpiryTime', TType.I64, 7) + oprot.writeI64(self.linkExpiryTime) + oprot.writeFieldEnd() + if self.httpHeaders is not None: + oprot.writeFieldBegin('httpHeaders', TType.MAP, 8) + oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.httpHeaders)) + for kiter120, viter121 in self.httpHeaders.items(): + oprot.writeString(kiter120.encode('utf-8') if sys.version_info[0] == 2 else kiter120) + oprot.writeString(viter121.encode('utf-8') if sys.version_info[0] == 2 else viter121) + oprot.writeMapEnd() + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() def validate(self): - if self.filePath is None: - raise TProtocolException(message='Required field filePath is unset!') - if self.startRowOffset is None: - raise TProtocolException(message='Required field startRowOffset is unset!') - if self.rowCount is None: - raise TProtocolException(message='Required field rowCount is unset!') - if self.uncompressedBytes is None: - raise TProtocolException(message='Required field uncompressedBytes is unset!') - if self.compressedBytes is None: - raise TProtocolException(message='Required field compressedBytes is unset!') return def __repr__(self): @@ -3145,11 +3547,12 @@ class TRowSet(object): - columnCount - arrowBatches - resultLinks + - cloudFetchResults """ - def __init__(self, startRowOffset=None, rows=None, columns=None, binaryColumns=None, columnCount=None, arrowBatches=None, resultLinks=None,): + def __init__(self, startRowOffset=None, rows=None, columns=None, binaryColumns=None, columnCount=None, arrowBatches=None, resultLinks=None, cloudFetchResults=None,): self.startRowOffset = startRowOffset self.rows = rows self.columns = columns @@ -3157,6 +3560,7 @@ def __init__(self, startRowOffset=None, rows=None, columns=None, binaryColumns=N self.columnCount = columnCount self.arrowBatches = arrowBatches self.resultLinks = resultLinks + self.cloudFetchResults = cloudFetchResults def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -3175,22 +3579,22 @@ def read(self, iprot): elif fid == 2: if ftype == TType.LIST: self.rows = [] - (_etype107, _size104) = iprot.readListBegin() - for _i108 in range(_size104): - _elem109 = TRow() - _elem109.read(iprot) - self.rows.append(_elem109) + (_etype125, _size122) = iprot.readListBegin() + for _i126 in range(_size122): + _elem127 = TRow() + _elem127.read(iprot) + self.rows.append(_elem127) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 3: if ftype == TType.LIST: self.columns = [] - (_etype113, _size110) = iprot.readListBegin() - for _i114 in range(_size110): - _elem115 = TColumn() - _elem115.read(iprot) - self.columns.append(_elem115) + (_etype131, _size128) = iprot.readListBegin() + for _i132 in range(_size128): + _elem133 = TColumn() + _elem133.read(iprot) + self.columns.append(_elem133) iprot.readListEnd() else: iprot.skip(ftype) @@ -3207,22 +3611,33 @@ def read(self, iprot): elif fid == 1281: if ftype == TType.LIST: self.arrowBatches = [] - (_etype119, _size116) = iprot.readListBegin() - for _i120 in range(_size116): - _elem121 = TSparkArrowBatch() - _elem121.read(iprot) - self.arrowBatches.append(_elem121) + (_etype137, _size134) = iprot.readListBegin() + for _i138 in range(_size134): + _elem139 = TSparkArrowBatch() + _elem139.read(iprot) + self.arrowBatches.append(_elem139) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 1282: if ftype == TType.LIST: self.resultLinks = [] - (_etype125, _size122) = iprot.readListBegin() - for _i126 in range(_size122): - _elem127 = TSparkArrowResultLink() - _elem127.read(iprot) - self.resultLinks.append(_elem127) + (_etype143, _size140) = iprot.readListBegin() + for _i144 in range(_size140): + _elem145 = TSparkArrowResultLink() + _elem145.read(iprot) + self.resultLinks.append(_elem145) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 3329: + if ftype == TType.LIST: + self.cloudFetchResults = [] + (_etype149, _size146) = iprot.readListBegin() + for _i150 in range(_size146): + _elem151 = TDBSqlCloudResultFile() + _elem151.read(iprot) + self.cloudFetchResults.append(_elem151) iprot.readListEnd() else: iprot.skip(ftype) @@ -3243,15 +3658,15 @@ def write(self, oprot): if self.rows is not None: oprot.writeFieldBegin('rows', TType.LIST, 2) oprot.writeListBegin(TType.STRUCT, len(self.rows)) - for iter128 in self.rows: - iter128.write(oprot) + for iter152 in self.rows: + iter152.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.columns is not None: oprot.writeFieldBegin('columns', TType.LIST, 3) oprot.writeListBegin(TType.STRUCT, len(self.columns)) - for iter129 in self.columns: - iter129.write(oprot) + for iter153 in self.columns: + iter153.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.binaryColumns is not None: @@ -3265,15 +3680,22 @@ def write(self, oprot): if self.arrowBatches is not None: oprot.writeFieldBegin('arrowBatches', TType.LIST, 1281) oprot.writeListBegin(TType.STRUCT, len(self.arrowBatches)) - for iter130 in self.arrowBatches: - iter130.write(oprot) + for iter154 in self.arrowBatches: + iter154.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.resultLinks is not None: oprot.writeFieldBegin('resultLinks', TType.LIST, 1282) oprot.writeListBegin(TType.STRUCT, len(self.resultLinks)) - for iter131 in self.resultLinks: - iter131.write(oprot) + for iter155 in self.resultLinks: + iter155.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.cloudFetchResults is not None: + oprot.writeFieldBegin('cloudFetchResults', TType.LIST, 3329) + oprot.writeListBegin(TType.STRUCT, len(self.cloudFetchResults)) + for iter156 in self.cloudFetchResults: + iter156.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -3337,11 +3759,11 @@ def read(self, iprot): elif fid == 3: if ftype == TType.MAP: self.properties = {} - (_ktype133, _vtype134, _size132) = iprot.readMapBegin() - for _i136 in range(_size132): - _key137 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _val138 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.properties[_key137] = _val138 + (_ktype158, _vtype159, _size157) = iprot.readMapBegin() + for _i161 in range(_size157): + _key162 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val163 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.properties[_key162] = _val163 iprot.readMapEnd() else: iprot.skip(ftype) @@ -3371,9 +3793,9 @@ def write(self, oprot): if self.properties is not None: oprot.writeFieldBegin('properties', TType.MAP, 3) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.properties)) - for kiter139, viter140 in self.properties.items(): - oprot.writeString(kiter139.encode('utf-8') if sys.version_info[0] == 2 else kiter139) - oprot.writeString(viter140.encode('utf-8') if sys.version_info[0] == 2 else viter140) + for kiter164, viter165 in self.properties.items(): + oprot.writeString(kiter164.encode('utf-8') if sys.version_info[0] == 2 else kiter164) + oprot.writeString(viter165.encode('utf-8') if sys.version_info[0] == 2 else viter165) oprot.writeMapEnd() oprot.writeFieldEnd() if self.viewSchema is not None: @@ -3725,22 +4147,22 @@ def read(self, iprot): if fid == 1: if ftype == TType.MAP: self.confs = {} - (_ktype142, _vtype143, _size141) = iprot.readMapBegin() - for _i145 in range(_size141): - _key146 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _val147 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.confs[_key146] = _val147 + (_ktype167, _vtype168, _size166) = iprot.readMapBegin() + for _i170 in range(_size166): + _key171 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val172 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.confs[_key171] = _val172 iprot.readMapEnd() else: iprot.skip(ftype) elif fid == 2: if ftype == TType.LIST: self.tempViews = [] - (_etype151, _size148) = iprot.readListBegin() - for _i152 in range(_size148): - _elem153 = TDBSqlTempView() - _elem153.read(iprot) - self.tempViews.append(_elem153) + (_etype176, _size173) = iprot.readListBegin() + for _i177 in range(_size173): + _elem178 = TDBSqlTempView() + _elem178.read(iprot) + self.tempViews.append(_elem178) iprot.readListEnd() else: iprot.skip(ftype) @@ -3763,23 +4185,23 @@ def read(self, iprot): elif fid == 6: if ftype == TType.LIST: self.expressionsInfos = [] - (_etype157, _size154) = iprot.readListBegin() - for _i158 in range(_size154): - _elem159 = TExpressionInfo() - _elem159.read(iprot) - self.expressionsInfos.append(_elem159) + (_etype182, _size179) = iprot.readListBegin() + for _i183 in range(_size179): + _elem184 = TExpressionInfo() + _elem184.read(iprot) + self.expressionsInfos.append(_elem184) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 7: if ftype == TType.MAP: self.internalConfs = {} - (_ktype161, _vtype162, _size160) = iprot.readMapBegin() - for _i164 in range(_size160): - _key165 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _val166 = TDBSqlConfValue() - _val166.read(iprot) - self.internalConfs[_key165] = _val166 + (_ktype186, _vtype187, _size185) = iprot.readMapBegin() + for _i189 in range(_size185): + _key190 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val191 = TDBSqlConfValue() + _val191.read(iprot) + self.internalConfs[_key190] = _val191 iprot.readMapEnd() else: iprot.skip(ftype) @@ -3796,16 +4218,16 @@ def write(self, oprot): if self.confs is not None: oprot.writeFieldBegin('confs', TType.MAP, 1) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.confs)) - for kiter167, viter168 in self.confs.items(): - oprot.writeString(kiter167.encode('utf-8') if sys.version_info[0] == 2 else kiter167) - oprot.writeString(viter168.encode('utf-8') if sys.version_info[0] == 2 else viter168) + for kiter192, viter193 in self.confs.items(): + oprot.writeString(kiter192.encode('utf-8') if sys.version_info[0] == 2 else kiter192) + oprot.writeString(viter193.encode('utf-8') if sys.version_info[0] == 2 else viter193) oprot.writeMapEnd() oprot.writeFieldEnd() if self.tempViews is not None: oprot.writeFieldBegin('tempViews', TType.LIST, 2) oprot.writeListBegin(TType.STRUCT, len(self.tempViews)) - for iter169 in self.tempViews: - iter169.write(oprot) + for iter194 in self.tempViews: + iter194.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.currentDatabase is not None: @@ -3823,16 +4245,16 @@ def write(self, oprot): if self.expressionsInfos is not None: oprot.writeFieldBegin('expressionsInfos', TType.LIST, 6) oprot.writeListBegin(TType.STRUCT, len(self.expressionsInfos)) - for iter170 in self.expressionsInfos: - iter170.write(oprot) + for iter195 in self.expressionsInfos: + iter195.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.internalConfs is not None: oprot.writeFieldBegin('internalConfs', TType.MAP, 7) oprot.writeMapBegin(TType.STRING, TType.STRUCT, len(self.internalConfs)) - for kiter171, viter172 in self.internalConfs.items(): - oprot.writeString(kiter171.encode('utf-8') if sys.version_info[0] == 2 else kiter171) - viter172.write(oprot) + for kiter196, viter197 in self.internalConfs.items(): + oprot.writeString(kiter196.encode('utf-8') if sys.version_info[0] == 2 else kiter196) + viter197.write(oprot) oprot.writeMapEnd() oprot.writeFieldEnd() oprot.writeFieldStop() @@ -3862,18 +4284,20 @@ class TStatus(object): - errorCode - errorMessage - displayMessage + - errorDetailsJson - responseValidation """ - def __init__(self, statusCode=None, infoMessages=None, sqlState=None, errorCode=None, errorMessage=None, displayMessage=None, responseValidation=None,): + def __init__(self, statusCode=None, infoMessages=None, sqlState=None, errorCode=None, errorMessage=None, displayMessage=None, errorDetailsJson=None, responseValidation=None,): self.statusCode = statusCode self.infoMessages = infoMessages self.sqlState = sqlState self.errorCode = errorCode self.errorMessage = errorMessage self.displayMessage = displayMessage + self.errorDetailsJson = errorDetailsJson self.responseValidation = responseValidation def read(self, iprot): @@ -3893,10 +4317,10 @@ def read(self, iprot): elif fid == 2: if ftype == TType.LIST: self.infoMessages = [] - (_etype176, _size173) = iprot.readListBegin() - for _i177 in range(_size173): - _elem178 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.infoMessages.append(_elem178) + (_etype201, _size198) = iprot.readListBegin() + for _i202 in range(_size198): + _elem203 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.infoMessages.append(_elem203) iprot.readListEnd() else: iprot.skip(ftype) @@ -3920,6 +4344,11 @@ def read(self, iprot): self.displayMessage = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() else: iprot.skip(ftype) + elif fid == 1281: + if ftype == TType.STRING: + self.errorDetailsJson = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) elif fid == 3329: if ftype == TType.STRING: self.responseValidation = iprot.readBinary() @@ -3942,8 +4371,8 @@ def write(self, oprot): if self.infoMessages is not None: oprot.writeFieldBegin('infoMessages', TType.LIST, 2) oprot.writeListBegin(TType.STRING, len(self.infoMessages)) - for iter179 in self.infoMessages: - oprot.writeString(iter179.encode('utf-8') if sys.version_info[0] == 2 else iter179) + for iter204 in self.infoMessages: + oprot.writeString(iter204.encode('utf-8') if sys.version_info[0] == 2 else iter204) oprot.writeListEnd() oprot.writeFieldEnd() if self.sqlState is not None: @@ -3962,6 +4391,10 @@ def write(self, oprot): oprot.writeFieldBegin('displayMessage', TType.STRING, 6) oprot.writeString(self.displayMessage.encode('utf-8') if sys.version_info[0] == 2 else self.displayMessage) oprot.writeFieldEnd() + if self.errorDetailsJson is not None: + oprot.writeFieldBegin('errorDetailsJson', TType.STRING, 1281) + oprot.writeString(self.errorDetailsJson.encode('utf-8') if sys.version_info[0] == 2 else self.errorDetailsJson) + oprot.writeFieldEnd() if self.responseValidation is not None: oprot.writeFieldBegin('responseValidation', TType.STRING, 3329) oprot.writeBinary(self.responseValidation) @@ -4361,21 +4794,21 @@ def read(self, iprot): elif fid == 4: if ftype == TType.MAP: self.configuration = {} - (_ktype181, _vtype182, _size180) = iprot.readMapBegin() - for _i184 in range(_size180): - _key185 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _val186 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.configuration[_key185] = _val186 + (_ktype206, _vtype207, _size205) = iprot.readMapBegin() + for _i209 in range(_size205): + _key210 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val211 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.configuration[_key210] = _val211 iprot.readMapEnd() else: iprot.skip(ftype) elif fid == 1281: if ftype == TType.LIST: self.getInfos = [] - (_etype190, _size187) = iprot.readListBegin() - for _i191 in range(_size187): - _elem192 = iprot.readI32() - self.getInfos.append(_elem192) + (_etype215, _size212) = iprot.readListBegin() + for _i216 in range(_size212): + _elem217 = iprot.readI32() + self.getInfos.append(_elem217) iprot.readListEnd() else: iprot.skip(ftype) @@ -4387,11 +4820,11 @@ def read(self, iprot): elif fid == 1283: if ftype == TType.MAP: self.connectionProperties = {} - (_ktype194, _vtype195, _size193) = iprot.readMapBegin() - for _i197 in range(_size193): - _key198 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _val199 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.connectionProperties[_key198] = _val199 + (_ktype219, _vtype220, _size218) = iprot.readMapBegin() + for _i222 in range(_size218): + _key223 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val224 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.connectionProperties[_key223] = _val224 iprot.readMapEnd() else: iprot.skip(ftype) @@ -4437,16 +4870,16 @@ def write(self, oprot): if self.configuration is not None: oprot.writeFieldBegin('configuration', TType.MAP, 4) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.configuration)) - for kiter200, viter201 in self.configuration.items(): - oprot.writeString(kiter200.encode('utf-8') if sys.version_info[0] == 2 else kiter200) - oprot.writeString(viter201.encode('utf-8') if sys.version_info[0] == 2 else viter201) + for kiter225, viter226 in self.configuration.items(): + oprot.writeString(kiter225.encode('utf-8') if sys.version_info[0] == 2 else kiter225) + oprot.writeString(viter226.encode('utf-8') if sys.version_info[0] == 2 else viter226) oprot.writeMapEnd() oprot.writeFieldEnd() if self.getInfos is not None: oprot.writeFieldBegin('getInfos', TType.LIST, 1281) oprot.writeListBegin(TType.I32, len(self.getInfos)) - for iter202 in self.getInfos: - oprot.writeI32(iter202) + for iter227 in self.getInfos: + oprot.writeI32(iter227) oprot.writeListEnd() oprot.writeFieldEnd() if self.client_protocol_i64 is not None: @@ -4456,9 +4889,9 @@ def write(self, oprot): if self.connectionProperties is not None: oprot.writeFieldBegin('connectionProperties', TType.MAP, 1283) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.connectionProperties)) - for kiter203, viter204 in self.connectionProperties.items(): - oprot.writeString(kiter203.encode('utf-8') if sys.version_info[0] == 2 else kiter203) - oprot.writeString(viter204.encode('utf-8') if sys.version_info[0] == 2 else viter204) + for kiter228, viter229 in self.connectionProperties.items(): + oprot.writeString(kiter228.encode('utf-8') if sys.version_info[0] == 2 else kiter228) + oprot.writeString(viter229.encode('utf-8') if sys.version_info[0] == 2 else viter229) oprot.writeMapEnd() oprot.writeFieldEnd() if self.initialNamespace is not None: @@ -4543,11 +4976,11 @@ def read(self, iprot): elif fid == 4: if ftype == TType.MAP: self.configuration = {} - (_ktype206, _vtype207, _size205) = iprot.readMapBegin() - for _i209 in range(_size205): - _key210 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _val211 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.configuration[_key210] = _val211 + (_ktype231, _vtype232, _size230) = iprot.readMapBegin() + for _i234 in range(_size230): + _key235 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val236 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.configuration[_key235] = _val236 iprot.readMapEnd() else: iprot.skip(ftype) @@ -4565,11 +4998,11 @@ def read(self, iprot): elif fid == 1281: if ftype == TType.LIST: self.getInfos = [] - (_etype215, _size212) = iprot.readListBegin() - for _i216 in range(_size212): - _elem217 = TGetInfoValue() - _elem217.read(iprot) - self.getInfos.append(_elem217) + (_etype240, _size237) = iprot.readListBegin() + for _i241 in range(_size237): + _elem242 = TGetInfoValue() + _elem242.read(iprot) + self.getInfos.append(_elem242) iprot.readListEnd() else: iprot.skip(ftype) @@ -4598,16 +5031,16 @@ def write(self, oprot): if self.configuration is not None: oprot.writeFieldBegin('configuration', TType.MAP, 4) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.configuration)) - for kiter218, viter219 in self.configuration.items(): - oprot.writeString(kiter218.encode('utf-8') if sys.version_info[0] == 2 else kiter218) - oprot.writeString(viter219.encode('utf-8') if sys.version_info[0] == 2 else viter219) + for kiter243, viter244 in self.configuration.items(): + oprot.writeString(kiter243.encode('utf-8') if sys.version_info[0] == 2 else kiter243) + oprot.writeString(viter244.encode('utf-8') if sys.version_info[0] == 2 else viter244) oprot.writeMapEnd() oprot.writeFieldEnd() if self.getInfos is not None: oprot.writeFieldBegin('getInfos', TType.LIST, 1281) oprot.writeListBegin(TType.STRUCT, len(self.getInfos)) - for iter220 in self.getInfos: - iter220.write(oprot) + for iter245 in self.getInfos: + iter245.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.initialNamespace is not None: @@ -5202,15 +5635,17 @@ class TSparkArrowTypes(object): - decimalAsArrow - complexTypesAsArrow - intervalTypesAsArrow + - nullTypeAsArrow """ - def __init__(self, timestampAsArrow=None, decimalAsArrow=None, complexTypesAsArrow=None, intervalTypesAsArrow=None,): + def __init__(self, timestampAsArrow=None, decimalAsArrow=None, complexTypesAsArrow=None, intervalTypesAsArrow=None, nullTypeAsArrow=None,): self.timestampAsArrow = timestampAsArrow self.decimalAsArrow = decimalAsArrow self.complexTypesAsArrow = complexTypesAsArrow self.intervalTypesAsArrow = intervalTypesAsArrow + self.nullTypeAsArrow = nullTypeAsArrow def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -5241,6 +5676,11 @@ def read(self, iprot): self.intervalTypesAsArrow = iprot.readBool() else: iprot.skip(ftype) + elif fid == 5: + if ftype == TType.BOOL: + self.nullTypeAsArrow = iprot.readBool() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -5267,6 +5707,10 @@ def write(self, oprot): oprot.writeFieldBegin('intervalTypesAsArrow', TType.BOOL, 4) oprot.writeBool(self.intervalTypesAsArrow) oprot.writeFieldEnd() + if self.nullTypeAsArrow is not None: + oprot.writeFieldBegin('nullTypeAsArrow', TType.BOOL, 5) + oprot.writeBool(self.nullTypeAsArrow) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -5300,6 +5744,9 @@ class TExecuteStatementReq(object): - maxBytesPerFile - useArrowNativeTypes - resultRowLimit + - parameters + - maxBytesPerBatch + - statementConf - operationId - sessionConf - rejectHighCostQueries @@ -5308,11 +5755,24 @@ class TExecuteStatementReq(object): - requestValidation - resultPersistenceMode - trimArrowBatchesToLimit + - fetchDisposition + - enforceResultPersistenceMode + - statementList + - persistResultManifest + - resultRetentionSeconds + - resultByteLimit + - resultDataFormat + - originatingClientIdentity + - preferSingleFileResult + - preferDriverOnlyUpload + - enforceEmbeddedSchemaCorrectness + - idempotencyToken + - throwErrorOnByteLimitTruncation """ - def __init__(self, sessionHandle=None, statement=None, confOverlay=None, runAsync=False, getDirectResults=None, queryTimeout=0, canReadArrowResult=None, canDownloadResult=None, canDecompressLZ4Result=None, maxBytesPerFile=None, useArrowNativeTypes=None, resultRowLimit=None, operationId=None, sessionConf=None, rejectHighCostQueries=None, estimatedCost=None, executionVersion=None, requestValidation=None, resultPersistenceMode=None, trimArrowBatchesToLimit=None,): + def __init__(self, sessionHandle=None, statement=None, confOverlay=None, runAsync=False, getDirectResults=None, queryTimeout=0, canReadArrowResult=None, canDownloadResult=None, canDecompressLZ4Result=None, maxBytesPerFile=None, useArrowNativeTypes=None, resultRowLimit=None, parameters=None, maxBytesPerBatch=None, statementConf=None, operationId=None, sessionConf=None, rejectHighCostQueries=None, estimatedCost=None, executionVersion=None, requestValidation=None, resultPersistenceMode=None, trimArrowBatchesToLimit=None, fetchDisposition=None, enforceResultPersistenceMode=None, statementList=None, persistResultManifest=None, resultRetentionSeconds=None, resultByteLimit=None, resultDataFormat=None, originatingClientIdentity=None, preferSingleFileResult=None, preferDriverOnlyUpload=None, enforceEmbeddedSchemaCorrectness=False, idempotencyToken=None, throwErrorOnByteLimitTruncation=None,): self.sessionHandle = sessionHandle self.statement = statement self.confOverlay = confOverlay @@ -5325,6 +5785,9 @@ def __init__(self, sessionHandle=None, statement=None, confOverlay=None, runAsyn self.maxBytesPerFile = maxBytesPerFile self.useArrowNativeTypes = useArrowNativeTypes self.resultRowLimit = resultRowLimit + self.parameters = parameters + self.maxBytesPerBatch = maxBytesPerBatch + self.statementConf = statementConf self.operationId = operationId self.sessionConf = sessionConf self.rejectHighCostQueries = rejectHighCostQueries @@ -5333,6 +5796,19 @@ def __init__(self, sessionHandle=None, statement=None, confOverlay=None, runAsyn self.requestValidation = requestValidation self.resultPersistenceMode = resultPersistenceMode self.trimArrowBatchesToLimit = trimArrowBatchesToLimit + self.fetchDisposition = fetchDisposition + self.enforceResultPersistenceMode = enforceResultPersistenceMode + self.statementList = statementList + self.persistResultManifest = persistResultManifest + self.resultRetentionSeconds = resultRetentionSeconds + self.resultByteLimit = resultByteLimit + self.resultDataFormat = resultDataFormat + self.originatingClientIdentity = originatingClientIdentity + self.preferSingleFileResult = preferSingleFileResult + self.preferDriverOnlyUpload = preferDriverOnlyUpload + self.enforceEmbeddedSchemaCorrectness = enforceEmbeddedSchemaCorrectness + self.idempotencyToken = idempotencyToken + self.throwErrorOnByteLimitTruncation = throwErrorOnByteLimitTruncation def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -5357,11 +5833,11 @@ def read(self, iprot): elif fid == 3: if ftype == TType.MAP: self.confOverlay = {} - (_ktype222, _vtype223, _size221) = iprot.readMapBegin() - for _i225 in range(_size221): - _key226 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _val227 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.confOverlay[_key226] = _val227 + (_ktype247, _vtype248, _size246) = iprot.readMapBegin() + for _i250 in range(_size246): + _key251 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _val252 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.confOverlay[_key251] = _val252 iprot.readMapEnd() else: iprot.skip(ftype) @@ -5412,6 +5888,28 @@ def read(self, iprot): self.resultRowLimit = iprot.readI64() else: iprot.skip(ftype) + elif fid == 1288: + if ftype == TType.LIST: + self.parameters = [] + (_etype256, _size253) = iprot.readListBegin() + for _i257 in range(_size253): + _elem258 = TSparkParameter() + _elem258.read(iprot) + self.parameters.append(_elem258) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 1289: + if ftype == TType.I64: + self.maxBytesPerBatch = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 1296: + if ftype == TType.STRUCT: + self.statementConf = TStatementConf() + self.statementConf.read(iprot) + else: + iprot.skip(ftype) elif fid == 3329: if ftype == TType.STRUCT: self.operationId = THandleIdentifier() @@ -5454,6 +5952,78 @@ def read(self, iprot): self.trimArrowBatchesToLimit = iprot.readBool() else: iprot.skip(ftype) + elif fid == 3337: + if ftype == TType.I32: + self.fetchDisposition = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3344: + if ftype == TType.BOOL: + self.enforceResultPersistenceMode = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3345: + if ftype == TType.LIST: + self.statementList = [] + (_etype262, _size259) = iprot.readListBegin() + for _i263 in range(_size259): + _elem264 = TDBSqlStatement() + _elem264.read(iprot) + self.statementList.append(_elem264) + iprot.readListEnd() + else: + iprot.skip(ftype) + elif fid == 3346: + if ftype == TType.BOOL: + self.persistResultManifest = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3347: + if ftype == TType.I64: + self.resultRetentionSeconds = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3348: + if ftype == TType.I64: + self.resultByteLimit = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3349: + if ftype == TType.STRUCT: + self.resultDataFormat = TDBSqlResultFormat() + self.resultDataFormat.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3350: + if ftype == TType.STRING: + self.originatingClientIdentity = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 3351: + if ftype == TType.BOOL: + self.preferSingleFileResult = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3352: + if ftype == TType.BOOL: + self.preferDriverOnlyUpload = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3353: + if ftype == TType.BOOL: + self.enforceEmbeddedSchemaCorrectness = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3360: + if ftype == TType.STRING: + self.idempotencyToken = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 3361: + if ftype == TType.BOOL: + self.throwErrorOnByteLimitTruncation = iprot.readBool() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -5475,9 +6045,9 @@ def write(self, oprot): if self.confOverlay is not None: oprot.writeFieldBegin('confOverlay', TType.MAP, 3) oprot.writeMapBegin(TType.STRING, TType.STRING, len(self.confOverlay)) - for kiter228, viter229 in self.confOverlay.items(): - oprot.writeString(kiter228.encode('utf-8') if sys.version_info[0] == 2 else kiter228) - oprot.writeString(viter229.encode('utf-8') if sys.version_info[0] == 2 else viter229) + for kiter265, viter266 in self.confOverlay.items(): + oprot.writeString(kiter265.encode('utf-8') if sys.version_info[0] == 2 else kiter265) + oprot.writeString(viter266.encode('utf-8') if sys.version_info[0] == 2 else viter266) oprot.writeMapEnd() oprot.writeFieldEnd() if self.runAsync is not None: @@ -5516,6 +6086,21 @@ def write(self, oprot): oprot.writeFieldBegin('resultRowLimit', TType.I64, 1287) oprot.writeI64(self.resultRowLimit) oprot.writeFieldEnd() + if self.parameters is not None: + oprot.writeFieldBegin('parameters', TType.LIST, 1288) + oprot.writeListBegin(TType.STRUCT, len(self.parameters)) + for iter267 in self.parameters: + iter267.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.maxBytesPerBatch is not None: + oprot.writeFieldBegin('maxBytesPerBatch', TType.I64, 1289) + oprot.writeI64(self.maxBytesPerBatch) + oprot.writeFieldEnd() + if self.statementConf is not None: + oprot.writeFieldBegin('statementConf', TType.STRUCT, 1296) + self.statementConf.write(oprot) + oprot.writeFieldEnd() if self.operationId is not None: oprot.writeFieldBegin('operationId', TType.STRUCT, 3329) self.operationId.write(oprot) @@ -5548,6 +6133,61 @@ def write(self, oprot): oprot.writeFieldBegin('trimArrowBatchesToLimit', TType.BOOL, 3336) oprot.writeBool(self.trimArrowBatchesToLimit) oprot.writeFieldEnd() + if self.fetchDisposition is not None: + oprot.writeFieldBegin('fetchDisposition', TType.I32, 3337) + oprot.writeI32(self.fetchDisposition) + oprot.writeFieldEnd() + if self.enforceResultPersistenceMode is not None: + oprot.writeFieldBegin('enforceResultPersistenceMode', TType.BOOL, 3344) + oprot.writeBool(self.enforceResultPersistenceMode) + oprot.writeFieldEnd() + if self.statementList is not None: + oprot.writeFieldBegin('statementList', TType.LIST, 3345) + oprot.writeListBegin(TType.STRUCT, len(self.statementList)) + for iter268 in self.statementList: + iter268.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() + if self.persistResultManifest is not None: + oprot.writeFieldBegin('persistResultManifest', TType.BOOL, 3346) + oprot.writeBool(self.persistResultManifest) + oprot.writeFieldEnd() + if self.resultRetentionSeconds is not None: + oprot.writeFieldBegin('resultRetentionSeconds', TType.I64, 3347) + oprot.writeI64(self.resultRetentionSeconds) + oprot.writeFieldEnd() + if self.resultByteLimit is not None: + oprot.writeFieldBegin('resultByteLimit', TType.I64, 3348) + oprot.writeI64(self.resultByteLimit) + oprot.writeFieldEnd() + if self.resultDataFormat is not None: + oprot.writeFieldBegin('resultDataFormat', TType.STRUCT, 3349) + self.resultDataFormat.write(oprot) + oprot.writeFieldEnd() + if self.originatingClientIdentity is not None: + oprot.writeFieldBegin('originatingClientIdentity', TType.STRING, 3350) + oprot.writeString(self.originatingClientIdentity.encode('utf-8') if sys.version_info[0] == 2 else self.originatingClientIdentity) + oprot.writeFieldEnd() + if self.preferSingleFileResult is not None: + oprot.writeFieldBegin('preferSingleFileResult', TType.BOOL, 3351) + oprot.writeBool(self.preferSingleFileResult) + oprot.writeFieldEnd() + if self.preferDriverOnlyUpload is not None: + oprot.writeFieldBegin('preferDriverOnlyUpload', TType.BOOL, 3352) + oprot.writeBool(self.preferDriverOnlyUpload) + oprot.writeFieldEnd() + if self.enforceEmbeddedSchemaCorrectness is not None: + oprot.writeFieldBegin('enforceEmbeddedSchemaCorrectness', TType.BOOL, 3353) + oprot.writeBool(self.enforceEmbeddedSchemaCorrectness) + oprot.writeFieldEnd() + if self.idempotencyToken is not None: + oprot.writeFieldBegin('idempotencyToken', TType.STRING, 3360) + oprot.writeString(self.idempotencyToken.encode('utf-8') if sys.version_info[0] == 2 else self.idempotencyToken) + oprot.writeFieldEnd() + if self.throwErrorOnByteLimitTruncation is not None: + oprot.writeFieldBegin('throwErrorOnByteLimitTruncation', TType.BOOL, 3361) + oprot.writeBool(self.throwErrorOnByteLimitTruncation) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -5570,6 +6210,324 @@ def __ne__(self, other): return not (self == other) +class TDBSqlStatement(object): + """ + Attributes: + - statement + + """ + + + def __init__(self, statement=None,): + self.statement = statement + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.statement = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TDBSqlStatement') + if self.statement is not None: + oprot.writeFieldBegin('statement', TType.STRING, 1) + oprot.writeString(self.statement.encode('utf-8') if sys.version_info[0] == 2 else self.statement) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TSparkParameterValue(object): + """ + Attributes: + - stringValue + - doubleValue + - booleanValue + + """ + + + def __init__(self, stringValue=None, doubleValue=None, booleanValue=None,): + self.stringValue = stringValue + self.doubleValue = doubleValue + self.booleanValue = booleanValue + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.STRING: + self.stringValue = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.DOUBLE: + self.doubleValue = iprot.readDouble() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.BOOL: + self.booleanValue = iprot.readBool() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TSparkParameterValue') + if self.stringValue is not None: + oprot.writeFieldBegin('stringValue', TType.STRING, 1) + oprot.writeString(self.stringValue.encode('utf-8') if sys.version_info[0] == 2 else self.stringValue) + oprot.writeFieldEnd() + if self.doubleValue is not None: + oprot.writeFieldBegin('doubleValue', TType.DOUBLE, 2) + oprot.writeDouble(self.doubleValue) + oprot.writeFieldEnd() + if self.booleanValue is not None: + oprot.writeFieldBegin('booleanValue', TType.BOOL, 3) + oprot.writeBool(self.booleanValue) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TSparkParameter(object): + """ + Attributes: + - ordinal + - name + - type + - value + + """ + + + def __init__(self, ordinal=None, name=None, type=None, value=None,): + self.ordinal = ordinal + self.name = name + self.type = type + self.value = value + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.I32: + self.ordinal = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRING: + self.name = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.STRING: + self.type = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.STRUCT: + self.value = TSparkParameterValue() + self.value.read(iprot) + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TSparkParameter') + if self.ordinal is not None: + oprot.writeFieldBegin('ordinal', TType.I32, 1) + oprot.writeI32(self.ordinal) + oprot.writeFieldEnd() + if self.name is not None: + oprot.writeFieldBegin('name', TType.STRING, 2) + oprot.writeString(self.name.encode('utf-8') if sys.version_info[0] == 2 else self.name) + oprot.writeFieldEnd() + if self.type is not None: + oprot.writeFieldBegin('type', TType.STRING, 3) + oprot.writeString(self.type.encode('utf-8') if sys.version_info[0] == 2 else self.type) + oprot.writeFieldEnd() + if self.value is not None: + oprot.writeFieldBegin('value', TType.STRUCT, 4) + self.value.write(oprot) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + +class TStatementConf(object): + """ + Attributes: + - sessionless + - initialNamespace + - client_protocol + - client_protocol_i64 + + """ + + + def __init__(self, sessionless=None, initialNamespace=None, client_protocol=None, client_protocol_i64=None,): + self.sessionless = sessionless + self.initialNamespace = initialNamespace + self.client_protocol = client_protocol + self.client_protocol_i64 = client_protocol_i64 + + def read(self, iprot): + if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: + iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) + return + iprot.readStructBegin() + while True: + (fname, ftype, fid) = iprot.readFieldBegin() + if ftype == TType.STOP: + break + if fid == 1: + if ftype == TType.BOOL: + self.sessionless = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 2: + if ftype == TType.STRUCT: + self.initialNamespace = TNamespace() + self.initialNamespace.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3: + if ftype == TType.I32: + self.client_protocol = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 4: + if ftype == TType.I64: + self.client_protocol_i64 = iprot.readI64() + else: + iprot.skip(ftype) + else: + iprot.skip(ftype) + iprot.readFieldEnd() + iprot.readStructEnd() + + def write(self, oprot): + if oprot._fast_encode is not None and self.thrift_spec is not None: + oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) + return + oprot.writeStructBegin('TStatementConf') + if self.sessionless is not None: + oprot.writeFieldBegin('sessionless', TType.BOOL, 1) + oprot.writeBool(self.sessionless) + oprot.writeFieldEnd() + if self.initialNamespace is not None: + oprot.writeFieldBegin('initialNamespace', TType.STRUCT, 2) + self.initialNamespace.write(oprot) + oprot.writeFieldEnd() + if self.client_protocol is not None: + oprot.writeFieldBegin('client_protocol', TType.I32, 3) + oprot.writeI32(self.client_protocol) + oprot.writeFieldEnd() + if self.client_protocol_i64 is not None: + oprot.writeFieldBegin('client_protocol_i64', TType.I64, 4) + oprot.writeI64(self.client_protocol_i64) + oprot.writeFieldEnd() + oprot.writeFieldStop() + oprot.writeStructEnd() + + def validate(self): + return + + def __repr__(self): + L = ['%s=%r' % (key, value) + for key, value in self.__dict__.items()] + return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) + + def __eq__(self, other): + return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ + + def __ne__(self, other): + return not (self == other) + + class TExecuteStatementResp(object): """ Attributes: @@ -5582,11 +6540,14 @@ class TExecuteStatementResp(object): - sessionConf - currentClusterLoad - idempotencyType + - remoteResultCacheEnabled + - isServerless + - operationHandles """ - def __init__(self, status=None, operationHandle=None, directResults=None, executionRejected=None, maxClusterCapacity=None, queryCost=None, sessionConf=None, currentClusterLoad=None, idempotencyType=None,): + def __init__(self, status=None, operationHandle=None, directResults=None, executionRejected=None, maxClusterCapacity=None, queryCost=None, sessionConf=None, currentClusterLoad=None, idempotencyType=None, remoteResultCacheEnabled=None, isServerless=None, operationHandles=None,): self.status = status self.operationHandle = operationHandle self.directResults = directResults @@ -5596,6 +6557,9 @@ def __init__(self, status=None, operationHandle=None, directResults=None, execut self.sessionConf = sessionConf self.currentClusterLoad = currentClusterLoad self.idempotencyType = idempotencyType + self.remoteResultCacheEnabled = remoteResultCacheEnabled + self.isServerless = isServerless + self.operationHandles = operationHandles def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -5655,6 +6619,27 @@ def read(self, iprot): self.idempotencyType = iprot.readI32() else: iprot.skip(ftype) + elif fid == 3335: + if ftype == TType.BOOL: + self.remoteResultCacheEnabled = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3336: + if ftype == TType.BOOL: + self.isServerless = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3337: + if ftype == TType.LIST: + self.operationHandles = [] + (_etype272, _size269) = iprot.readListBegin() + for _i273 in range(_size269): + _elem274 = TOperationHandle() + _elem274.read(iprot) + self.operationHandles.append(_elem274) + iprot.readListEnd() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -5701,6 +6686,21 @@ def write(self, oprot): oprot.writeFieldBegin('idempotencyType', TType.I32, 3334) oprot.writeI32(self.idempotencyType) oprot.writeFieldEnd() + if self.remoteResultCacheEnabled is not None: + oprot.writeFieldBegin('remoteResultCacheEnabled', TType.BOOL, 3335) + oprot.writeBool(self.remoteResultCacheEnabled) + oprot.writeFieldEnd() + if self.isServerless is not None: + oprot.writeFieldBegin('isServerless', TType.BOOL, 3336) + oprot.writeBool(self.isServerless) + oprot.writeFieldEnd() + if self.operationHandles is not None: + oprot.writeFieldBegin('operationHandles', TType.LIST, 3337) + oprot.writeListBegin(TType.STRUCT, len(self.operationHandles)) + for iter275 in self.operationHandles: + iter275.write(oprot) + oprot.writeListEnd() + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -6376,10 +7376,10 @@ def read(self, iprot): elif fid == 5: if ftype == TType.LIST: self.tableTypes = [] - (_etype233, _size230) = iprot.readListBegin() - for _i234 in range(_size230): - _elem235 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.tableTypes.append(_elem235) + (_etype279, _size276) = iprot.readListBegin() + for _i280 in range(_size276): + _elem281 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.tableTypes.append(_elem281) iprot.readListEnd() else: iprot.skip(ftype) @@ -6435,8 +7435,8 @@ def write(self, oprot): if self.tableTypes is not None: oprot.writeFieldBegin('tableTypes', TType.LIST, 5) oprot.writeListBegin(TType.STRING, len(self.tableTypes)) - for iter236 in self.tableTypes: - oprot.writeString(iter236.encode('utf-8') if sys.version_info[0] == 2 else iter236) + for iter282 in self.tableTypes: + oprot.writeString(iter282.encode('utf-8') if sys.version_info[0] == 2 else iter282) oprot.writeListEnd() oprot.writeFieldEnd() if self.getDirectResults is not None: @@ -7783,6 +8783,7 @@ class TGetOperationStatusResp(object): - numModifiedRows - displayMessage - diagnosticInfo + - errorDetailsJson - responseValidation - idempotencyType - statementTimeout @@ -7791,7 +8792,7 @@ class TGetOperationStatusResp(object): """ - def __init__(self, status=None, operationState=None, sqlState=None, errorCode=None, errorMessage=None, taskStatus=None, operationStarted=None, operationCompleted=None, hasResultSet=None, progressUpdateResponse=None, numModifiedRows=None, displayMessage=None, diagnosticInfo=None, responseValidation=None, idempotencyType=None, statementTimeout=None, statementTimeoutLevel=None,): + def __init__(self, status=None, operationState=None, sqlState=None, errorCode=None, errorMessage=None, taskStatus=None, operationStarted=None, operationCompleted=None, hasResultSet=None, progressUpdateResponse=None, numModifiedRows=None, displayMessage=None, diagnosticInfo=None, errorDetailsJson=None, responseValidation=None, idempotencyType=None, statementTimeout=None, statementTimeoutLevel=None,): self.status = status self.operationState = operationState self.sqlState = sqlState @@ -7805,6 +8806,7 @@ def __init__(self, status=None, operationState=None, sqlState=None, errorCode=No self.numModifiedRows = numModifiedRows self.displayMessage = displayMessage self.diagnosticInfo = diagnosticInfo + self.errorDetailsJson = errorDetailsJson self.responseValidation = responseValidation self.idempotencyType = idempotencyType self.statementTimeout = statementTimeout @@ -7886,6 +8888,11 @@ def read(self, iprot): self.diagnosticInfo = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() else: iprot.skip(ftype) + elif fid == 1283: + if ftype == TType.STRING: + self.errorDetailsJson = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) elif fid == 3329: if ftype == TType.STRING: self.responseValidation = iprot.readBinary() @@ -7968,6 +8975,10 @@ def write(self, oprot): oprot.writeFieldBegin('diagnosticInfo', TType.STRING, 1282) oprot.writeString(self.diagnosticInfo.encode('utf-8') if sys.version_info[0] == 2 else self.diagnosticInfo) oprot.writeFieldEnd() + if self.errorDetailsJson is not None: + oprot.writeFieldBegin('errorDetailsJson', TType.STRING, 1283) + oprot.writeString(self.errorDetailsJson.encode('utf-8') if sys.version_info[0] == 2 else self.errorDetailsJson) + oprot.writeFieldEnd() if self.responseValidation is not None: oprot.writeFieldBegin('responseValidation', TType.STRING, 3329) oprot.writeBinary(self.responseValidation) @@ -8150,12 +9161,14 @@ class TCloseOperationReq(object): """ Attributes: - operationHandle + - closeReason """ - def __init__(self, operationHandle=None,): + def __init__(self, operationHandle=None, closeReason= 0,): self.operationHandle = operationHandle + self.closeReason = closeReason def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -8172,6 +9185,11 @@ def read(self, iprot): self.operationHandle.read(iprot) else: iprot.skip(ftype) + elif fid == 3329: + if ftype == TType.I32: + self.closeReason = iprot.readI32() + else: + iprot.skip(ftype) else: iprot.skip(ftype) iprot.readFieldEnd() @@ -8186,6 +9204,10 @@ def write(self, oprot): oprot.writeFieldBegin('operationHandle', TType.STRUCT, 1) self.operationHandle.write(oprot) oprot.writeFieldEnd() + if self.closeReason is not None: + oprot.writeFieldBegin('closeReason', TType.I32, 3329) + oprot.writeI32(self.closeReason) + oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -8353,11 +9375,19 @@ class TGetResultSetMetadataResp(object): - resultFiles - manifestFile - manifestFileFormat + - cacheLookupLatency + - remoteCacheMissReason + - fetchDisposition + - remoteResultCacheEnabled + - isServerless + - resultDataFormat + - truncatedByThriftLimit + - resultByteLimit """ - def __init__(self, status=None, schema=None, resultFormat=None, lz4Compressed=None, arrowSchema=None, cacheLookupResult=None, uncompressedBytes=None, compressedBytes=None, isStagingOperation=None, reasonForNoCloudFetch=None, resultFiles=None, manifestFile=None, manifestFileFormat=None,): + def __init__(self, status=None, schema=None, resultFormat=None, lz4Compressed=None, arrowSchema=None, cacheLookupResult=None, uncompressedBytes=None, compressedBytes=None, isStagingOperation=None, reasonForNoCloudFetch=None, resultFiles=None, manifestFile=None, manifestFileFormat=None, cacheLookupLatency=None, remoteCacheMissReason=None, fetchDisposition=None, remoteResultCacheEnabled=None, isServerless=None, resultDataFormat=None, truncatedByThriftLimit=None, resultByteLimit=None,): self.status = status self.schema = schema self.resultFormat = resultFormat @@ -8371,6 +9401,14 @@ def __init__(self, status=None, schema=None, resultFormat=None, lz4Compressed=No self.resultFiles = resultFiles self.manifestFile = manifestFile self.manifestFileFormat = manifestFileFormat + self.cacheLookupLatency = cacheLookupLatency + self.remoteCacheMissReason = remoteCacheMissReason + self.fetchDisposition = fetchDisposition + self.remoteResultCacheEnabled = remoteResultCacheEnabled + self.isServerless = isServerless + self.resultDataFormat = resultDataFormat + self.truncatedByThriftLimit = truncatedByThriftLimit + self.resultByteLimit = resultByteLimit def read(self, iprot): if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: @@ -8436,11 +9474,11 @@ def read(self, iprot): elif fid == 3330: if ftype == TType.LIST: self.resultFiles = [] - (_etype240, _size237) = iprot.readListBegin() - for _i241 in range(_size237): - _elem242 = TDBSqlCloudResultFile() - _elem242.read(iprot) - self.resultFiles.append(_elem242) + (_etype286, _size283) = iprot.readListBegin() + for _i287 in range(_size283): + _elem288 = TDBSqlCloudResultFile() + _elem288.read(iprot) + self.resultFiles.append(_elem288) iprot.readListEnd() else: iprot.skip(ftype) @@ -8450,8 +9488,49 @@ def read(self, iprot): else: iprot.skip(ftype) elif fid == 3332: + if ftype == TType.I32: + self.manifestFileFormat = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3333: + if ftype == TType.I64: + self.cacheLookupLatency = iprot.readI64() + else: + iprot.skip(ftype) + elif fid == 3334: if ftype == TType.STRING: - self.manifestFileFormat = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.remoteCacheMissReason = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + else: + iprot.skip(ftype) + elif fid == 3335: + if ftype == TType.I32: + self.fetchDisposition = iprot.readI32() + else: + iprot.skip(ftype) + elif fid == 3336: + if ftype == TType.BOOL: + self.remoteResultCacheEnabled = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3337: + if ftype == TType.BOOL: + self.isServerless = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3344: + if ftype == TType.STRUCT: + self.resultDataFormat = TDBSqlResultFormat() + self.resultDataFormat.read(iprot) + else: + iprot.skip(ftype) + elif fid == 3345: + if ftype == TType.BOOL: + self.truncatedByThriftLimit = iprot.readBool() + else: + iprot.skip(ftype) + elif fid == 3346: + if ftype == TType.I64: + self.resultByteLimit = iprot.readI64() else: iprot.skip(ftype) else: @@ -8507,8 +9586,8 @@ def write(self, oprot): if self.resultFiles is not None: oprot.writeFieldBegin('resultFiles', TType.LIST, 3330) oprot.writeListBegin(TType.STRUCT, len(self.resultFiles)) - for iter243 in self.resultFiles: - iter243.write(oprot) + for iter289 in self.resultFiles: + iter289.write(oprot) oprot.writeListEnd() oprot.writeFieldEnd() if self.manifestFile is not None: @@ -8516,8 +9595,40 @@ def write(self, oprot): oprot.writeString(self.manifestFile.encode('utf-8') if sys.version_info[0] == 2 else self.manifestFile) oprot.writeFieldEnd() if self.manifestFileFormat is not None: - oprot.writeFieldBegin('manifestFileFormat', TType.STRING, 3332) - oprot.writeString(self.manifestFileFormat.encode('utf-8') if sys.version_info[0] == 2 else self.manifestFileFormat) + oprot.writeFieldBegin('manifestFileFormat', TType.I32, 3332) + oprot.writeI32(self.manifestFileFormat) + oprot.writeFieldEnd() + if self.cacheLookupLatency is not None: + oprot.writeFieldBegin('cacheLookupLatency', TType.I64, 3333) + oprot.writeI64(self.cacheLookupLatency) + oprot.writeFieldEnd() + if self.remoteCacheMissReason is not None: + oprot.writeFieldBegin('remoteCacheMissReason', TType.STRING, 3334) + oprot.writeString(self.remoteCacheMissReason.encode('utf-8') if sys.version_info[0] == 2 else self.remoteCacheMissReason) + oprot.writeFieldEnd() + if self.fetchDisposition is not None: + oprot.writeFieldBegin('fetchDisposition', TType.I32, 3335) + oprot.writeI32(self.fetchDisposition) + oprot.writeFieldEnd() + if self.remoteResultCacheEnabled is not None: + oprot.writeFieldBegin('remoteResultCacheEnabled', TType.BOOL, 3336) + oprot.writeBool(self.remoteResultCacheEnabled) + oprot.writeFieldEnd() + if self.isServerless is not None: + oprot.writeFieldBegin('isServerless', TType.BOOL, 3337) + oprot.writeBool(self.isServerless) + oprot.writeFieldEnd() + if self.resultDataFormat is not None: + oprot.writeFieldBegin('resultDataFormat', TType.STRUCT, 3344) + self.resultDataFormat.write(oprot) + oprot.writeFieldEnd() + if self.truncatedByThriftLimit is not None: + oprot.writeFieldBegin('truncatedByThriftLimit', TType.BOOL, 3345) + oprot.writeBool(self.truncatedByThriftLimit) + oprot.writeFieldEnd() + if self.resultByteLimit is not None: + oprot.writeFieldBegin('resultByteLimit', TType.I64, 3346) + oprot.writeI64(self.resultByteLimit) oprot.writeFieldEnd() oprot.writeFieldStop() oprot.writeStructEnd() @@ -9267,25 +10378,25 @@ def read(self, iprot): if fid == 1: if ftype == TType.LIST: self.headerNames = [] - (_etype247, _size244) = iprot.readListBegin() - for _i248 in range(_size244): - _elem249 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - self.headerNames.append(_elem249) + (_etype293, _size290) = iprot.readListBegin() + for _i294 in range(_size290): + _elem295 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + self.headerNames.append(_elem295) iprot.readListEnd() else: iprot.skip(ftype) elif fid == 2: if ftype == TType.LIST: self.rows = [] - (_etype253, _size250) = iprot.readListBegin() - for _i254 in range(_size250): - _elem255 = [] - (_etype259, _size256) = iprot.readListBegin() - for _i260 in range(_size256): - _elem261 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() - _elem255.append(_elem261) + (_etype299, _size296) = iprot.readListBegin() + for _i300 in range(_size296): + _elem301 = [] + (_etype305, _size302) = iprot.readListBegin() + for _i306 in range(_size302): + _elem307 = iprot.readString().decode('utf-8', errors='replace') if sys.version_info[0] == 2 else iprot.readString() + _elem301.append(_elem307) iprot.readListEnd() - self.rows.append(_elem255) + self.rows.append(_elem301) iprot.readListEnd() else: iprot.skip(ftype) @@ -9322,17 +10433,17 @@ def write(self, oprot): if self.headerNames is not None: oprot.writeFieldBegin('headerNames', TType.LIST, 1) oprot.writeListBegin(TType.STRING, len(self.headerNames)) - for iter262 in self.headerNames: - oprot.writeString(iter262.encode('utf-8') if sys.version_info[0] == 2 else iter262) + for iter308 in self.headerNames: + oprot.writeString(iter308.encode('utf-8') if sys.version_info[0] == 2 else iter308) oprot.writeListEnd() oprot.writeFieldEnd() if self.rows is not None: oprot.writeFieldBegin('rows', TType.LIST, 2) oprot.writeListBegin(TType.LIST, len(self.rows)) - for iter263 in self.rows: - oprot.writeListBegin(TType.STRING, len(iter263)) - for iter264 in iter263: - oprot.writeString(iter264.encode('utf-8') if sys.version_info[0] == 2 else iter264) + for iter309 in self.rows: + oprot.writeListBegin(TType.STRING, len(iter309)) + for iter310 in iter309: + oprot.writeString(iter310.encode('utf-8') if sys.version_info[0] == 2 else iter310) oprot.writeListEnd() oprot.writeListEnd() oprot.writeFieldEnd() @@ -9380,532 +10491,6 @@ def __eq__(self, other): def __ne__(self, other): return not (self == other) - - -class TDBSqlClusterMetrics(object): - """ - Attributes: - - clusterCapacity - - numRunningTasks - - numPendingTasks - - rejectionThreshold - - tasksCompletedPerMinute - - """ - - - def __init__(self, clusterCapacity=None, numRunningTasks=None, numPendingTasks=None, rejectionThreshold=None, tasksCompletedPerMinute=None,): - self.clusterCapacity = clusterCapacity - self.numRunningTasks = numRunningTasks - self.numPendingTasks = numPendingTasks - self.rejectionThreshold = rejectionThreshold - self.tasksCompletedPerMinute = tasksCompletedPerMinute - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.I32: - self.clusterCapacity = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.I32: - self.numRunningTasks = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.I32: - self.numPendingTasks = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.DOUBLE: - self.rejectionThreshold = iprot.readDouble() - else: - iprot.skip(ftype) - elif fid == 5: - if ftype == TType.DOUBLE: - self.tasksCompletedPerMinute = iprot.readDouble() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('TDBSqlClusterMetrics') - if self.clusterCapacity is not None: - oprot.writeFieldBegin('clusterCapacity', TType.I32, 1) - oprot.writeI32(self.clusterCapacity) - oprot.writeFieldEnd() - if self.numRunningTasks is not None: - oprot.writeFieldBegin('numRunningTasks', TType.I32, 2) - oprot.writeI32(self.numRunningTasks) - oprot.writeFieldEnd() - if self.numPendingTasks is not None: - oprot.writeFieldBegin('numPendingTasks', TType.I32, 3) - oprot.writeI32(self.numPendingTasks) - oprot.writeFieldEnd() - if self.rejectionThreshold is not None: - oprot.writeFieldBegin('rejectionThreshold', TType.DOUBLE, 4) - oprot.writeDouble(self.rejectionThreshold) - oprot.writeFieldEnd() - if self.tasksCompletedPerMinute is not None: - oprot.writeFieldBegin('tasksCompletedPerMinute', TType.DOUBLE, 5) - oprot.writeDouble(self.tasksCompletedPerMinute) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class TDBSqlQueryLaneMetrics(object): - """ - Attributes: - - fastLaneReservation - - numFastLaneRunningTasks - - numFastLanePendingTasks - - slowLaneReservation - - numSlowLaneRunningTasks - - numSlowLanePendingTasks - - """ - - - def __init__(self, fastLaneReservation=None, numFastLaneRunningTasks=None, numFastLanePendingTasks=None, slowLaneReservation=None, numSlowLaneRunningTasks=None, numSlowLanePendingTasks=None,): - self.fastLaneReservation = fastLaneReservation - self.numFastLaneRunningTasks = numFastLaneRunningTasks - self.numFastLanePendingTasks = numFastLanePendingTasks - self.slowLaneReservation = slowLaneReservation - self.numSlowLaneRunningTasks = numSlowLaneRunningTasks - self.numSlowLanePendingTasks = numSlowLanePendingTasks - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.I32: - self.fastLaneReservation = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.I32: - self.numFastLaneRunningTasks = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.I32: - self.numFastLanePendingTasks = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.I32: - self.slowLaneReservation = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 5: - if ftype == TType.I32: - self.numSlowLaneRunningTasks = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 6: - if ftype == TType.I32: - self.numSlowLanePendingTasks = iprot.readI32() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('TDBSqlQueryLaneMetrics') - if self.fastLaneReservation is not None: - oprot.writeFieldBegin('fastLaneReservation', TType.I32, 1) - oprot.writeI32(self.fastLaneReservation) - oprot.writeFieldEnd() - if self.numFastLaneRunningTasks is not None: - oprot.writeFieldBegin('numFastLaneRunningTasks', TType.I32, 2) - oprot.writeI32(self.numFastLaneRunningTasks) - oprot.writeFieldEnd() - if self.numFastLanePendingTasks is not None: - oprot.writeFieldBegin('numFastLanePendingTasks', TType.I32, 3) - oprot.writeI32(self.numFastLanePendingTasks) - oprot.writeFieldEnd() - if self.slowLaneReservation is not None: - oprot.writeFieldBegin('slowLaneReservation', TType.I32, 4) - oprot.writeI32(self.slowLaneReservation) - oprot.writeFieldEnd() - if self.numSlowLaneRunningTasks is not None: - oprot.writeFieldBegin('numSlowLaneRunningTasks', TType.I32, 5) - oprot.writeI32(self.numSlowLaneRunningTasks) - oprot.writeFieldEnd() - if self.numSlowLanePendingTasks is not None: - oprot.writeFieldBegin('numSlowLanePendingTasks', TType.I32, 6) - oprot.writeI32(self.numSlowLanePendingTasks) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class TDBSqlQueryMetrics(object): - """ - Attributes: - - status - - operationHandle - - idempotencyType - - sessionHandle - - operationStarted - - queryCost - - numRunningTasks - - numPendingTasks - - numCompletedTasks - - """ - - - def __init__(self, status=None, operationHandle=None, idempotencyType=None, sessionHandle=None, operationStarted=None, queryCost=None, numRunningTasks=None, numPendingTasks=None, numCompletedTasks=None,): - self.status = status - self.operationHandle = operationHandle - self.idempotencyType = idempotencyType - self.sessionHandle = sessionHandle - self.operationStarted = operationStarted - self.queryCost = queryCost - self.numRunningTasks = numRunningTasks - self.numPendingTasks = numPendingTasks - self.numCompletedTasks = numCompletedTasks - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRUCT: - self.status = TStatus() - self.status.read(iprot) - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.STRUCT: - self.operationHandle = TOperationHandle() - self.operationHandle.read(iprot) - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.I32: - self.idempotencyType = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.STRUCT: - self.sessionHandle = TSessionHandle() - self.sessionHandle.read(iprot) - else: - iprot.skip(ftype) - elif fid == 5: - if ftype == TType.I64: - self.operationStarted = iprot.readI64() - else: - iprot.skip(ftype) - elif fid == 6: - if ftype == TType.DOUBLE: - self.queryCost = iprot.readDouble() - else: - iprot.skip(ftype) - elif fid == 7: - if ftype == TType.I32: - self.numRunningTasks = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 8: - if ftype == TType.I32: - self.numPendingTasks = iprot.readI32() - else: - iprot.skip(ftype) - elif fid == 9: - if ftype == TType.I32: - self.numCompletedTasks = iprot.readI32() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('TDBSqlQueryMetrics') - if self.status is not None: - oprot.writeFieldBegin('status', TType.STRUCT, 1) - self.status.write(oprot) - oprot.writeFieldEnd() - if self.operationHandle is not None: - oprot.writeFieldBegin('operationHandle', TType.STRUCT, 2) - self.operationHandle.write(oprot) - oprot.writeFieldEnd() - if self.idempotencyType is not None: - oprot.writeFieldBegin('idempotencyType', TType.I32, 3) - oprot.writeI32(self.idempotencyType) - oprot.writeFieldEnd() - if self.sessionHandle is not None: - oprot.writeFieldBegin('sessionHandle', TType.STRUCT, 4) - self.sessionHandle.write(oprot) - oprot.writeFieldEnd() - if self.operationStarted is not None: - oprot.writeFieldBegin('operationStarted', TType.I64, 5) - oprot.writeI64(self.operationStarted) - oprot.writeFieldEnd() - if self.queryCost is not None: - oprot.writeFieldBegin('queryCost', TType.DOUBLE, 6) - oprot.writeDouble(self.queryCost) - oprot.writeFieldEnd() - if self.numRunningTasks is not None: - oprot.writeFieldBegin('numRunningTasks', TType.I32, 7) - oprot.writeI32(self.numRunningTasks) - oprot.writeFieldEnd() - if self.numPendingTasks is not None: - oprot.writeFieldBegin('numPendingTasks', TType.I32, 8) - oprot.writeI32(self.numPendingTasks) - oprot.writeFieldEnd() - if self.numCompletedTasks is not None: - oprot.writeFieldBegin('numCompletedTasks', TType.I32, 9) - oprot.writeI32(self.numCompletedTasks) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - if self.status is None: - raise TProtocolException(message='Required field status is unset!') - if self.operationHandle is None: - raise TProtocolException(message='Required field operationHandle is unset!') - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class TDBSqlGetLoadInformationReq(object): - """ - Attributes: - - includeQueryMetrics - - """ - - - def __init__(self, includeQueryMetrics=False,): - self.includeQueryMetrics = includeQueryMetrics - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.BOOL: - self.includeQueryMetrics = iprot.readBool() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('TDBSqlGetLoadInformationReq') - if self.includeQueryMetrics is not None: - oprot.writeFieldBegin('includeQueryMetrics', TType.BOOL, 1) - oprot.writeBool(self.includeQueryMetrics) - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) - - -class TDBSqlGetLoadInformationResp(object): - """ - Attributes: - - status - - clusterMetrics - - queryLaneMetrics - - queryMetrics - - """ - - - def __init__(self, status=None, clusterMetrics=None, queryLaneMetrics=None, queryMetrics=None,): - self.status = status - self.clusterMetrics = clusterMetrics - self.queryLaneMetrics = queryLaneMetrics - self.queryMetrics = queryMetrics - - def read(self, iprot): - if iprot._fast_decode is not None and isinstance(iprot.trans, TTransport.CReadableTransport) and self.thrift_spec is not None: - iprot._fast_decode(self, iprot, [self.__class__, self.thrift_spec]) - return - iprot.readStructBegin() - while True: - (fname, ftype, fid) = iprot.readFieldBegin() - if ftype == TType.STOP: - break - if fid == 1: - if ftype == TType.STRUCT: - self.status = TStatus() - self.status.read(iprot) - else: - iprot.skip(ftype) - elif fid == 2: - if ftype == TType.STRUCT: - self.clusterMetrics = TDBSqlClusterMetrics() - self.clusterMetrics.read(iprot) - else: - iprot.skip(ftype) - elif fid == 3: - if ftype == TType.STRUCT: - self.queryLaneMetrics = TDBSqlQueryLaneMetrics() - self.queryLaneMetrics.read(iprot) - else: - iprot.skip(ftype) - elif fid == 4: - if ftype == TType.LIST: - self.queryMetrics = [] - (_etype268, _size265) = iprot.readListBegin() - for _i269 in range(_size265): - _elem270 = TDBSqlQueryMetrics() - _elem270.read(iprot) - self.queryMetrics.append(_elem270) - iprot.readListEnd() - else: - iprot.skip(ftype) - else: - iprot.skip(ftype) - iprot.readFieldEnd() - iprot.readStructEnd() - - def write(self, oprot): - if oprot._fast_encode is not None and self.thrift_spec is not None: - oprot.trans.write(oprot._fast_encode(self, [self.__class__, self.thrift_spec])) - return - oprot.writeStructBegin('TDBSqlGetLoadInformationResp') - if self.status is not None: - oprot.writeFieldBegin('status', TType.STRUCT, 1) - self.status.write(oprot) - oprot.writeFieldEnd() - if self.clusterMetrics is not None: - oprot.writeFieldBegin('clusterMetrics', TType.STRUCT, 2) - self.clusterMetrics.write(oprot) - oprot.writeFieldEnd() - if self.queryLaneMetrics is not None: - oprot.writeFieldBegin('queryLaneMetrics', TType.STRUCT, 3) - self.queryLaneMetrics.write(oprot) - oprot.writeFieldEnd() - if self.queryMetrics is not None: - oprot.writeFieldBegin('queryMetrics', TType.LIST, 4) - oprot.writeListBegin(TType.STRUCT, len(self.queryMetrics)) - for iter271 in self.queryMetrics: - iter271.write(oprot) - oprot.writeListEnd() - oprot.writeFieldEnd() - oprot.writeFieldStop() - oprot.writeStructEnd() - - def validate(self): - if self.status is None: - raise TProtocolException(message='Required field status is unset!') - return - - def __repr__(self): - L = ['%s=%r' % (key, value) - for key, value in self.__dict__.items()] - return '%s(%s)' % (self.__class__.__name__, ', '.join(L)) - - def __eq__(self, other): - return isinstance(other, self.__class__) and self.__dict__ == other.__dict__ - - def __ne__(self, other): - return not (self == other) all_structs.append(TTypeQualifierValue) TTypeQualifierValue.thrift_spec = ( None, # 0 @@ -9917,8 +10502,8 @@ def __ne__(self, other): None, # 0 (1, TType.MAP, 'qualifiers', (TType.STRING, 'UTF8', TType.STRUCT, [TTypeQualifierValue, None], False), None, ), # 1 ) -all_structs.append(TPrimitiveTypeEntry) -TPrimitiveTypeEntry.thrift_spec = ( +all_structs.append(TTAllowedParameterValueEntry) +TTAllowedParameterValueEntry.thrift_spec = ( None, # 0 (1, TType.I32, 'type', None, None, ), # 1 (2, TType.STRUCT, 'typeQualifiers', [TTypeQualifiers, None], None, ), # 2 @@ -9952,7 +10537,7 @@ def __ne__(self, other): all_structs.append(TTypeEntry) TTypeEntry.thrift_spec = ( None, # 0 - (1, TType.STRUCT, 'primitiveEntry', [TPrimitiveTypeEntry, None], None, ), # 1 + (1, TType.STRUCT, 'primitiveEntry', [TTAllowedParameterValueEntry, None], None, ), # 1 (2, TType.STRUCT, 'arrayEntry', [TArrayTypeEntry, None], None, ), # 2 (3, TType.STRUCT, 'mapEntry', [TMapTypeEntry, None], None, ), # 3 (4, TType.STRUCT, 'structEntry', [TStructTypeEntry, None], None, ), # 4 @@ -10088,6 +10673,29 @@ def __ne__(self, other): (7, TType.STRUCT, 'stringVal', [TStringColumn, None], None, ), # 7 (8, TType.STRUCT, 'binaryVal', [TBinaryColumn, None], None, ), # 8 ) +all_structs.append(TDBSqlJsonArrayFormat) +TDBSqlJsonArrayFormat.thrift_spec = ( + None, # 0 + (1, TType.I32, 'compressionCodec', None, None, ), # 1 +) +all_structs.append(TDBSqlCsvFormat) +TDBSqlCsvFormat.thrift_spec = ( + None, # 0 + (1, TType.I32, 'compressionCodec', None, None, ), # 1 +) +all_structs.append(TDBSqlArrowFormat) +TDBSqlArrowFormat.thrift_spec = ( + None, # 0 + (1, TType.I32, 'arrowLayout', None, None, ), # 1 + (2, TType.I32, 'compressionCodec', None, None, ), # 2 +) +all_structs.append(TDBSqlResultFormat) +TDBSqlResultFormat.thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'arrowFormat', [TDBSqlArrowFormat, None], None, ), # 1 + (2, TType.STRUCT, 'csvFormat', [TDBSqlCsvFormat, None], None, ), # 2 + (3, TType.STRUCT, 'jsonArrayFormat', [TDBSqlJsonArrayFormat, None], None, ), # 3 +) all_structs.append(TSparkArrowBatch) TSparkArrowBatch.thrift_spec = ( None, # 0 @@ -10102,6 +10710,7 @@ def __ne__(self, other): (3, TType.I64, 'startRowOffset', None, None, ), # 3 (4, TType.I64, 'rowCount', None, None, ), # 4 (5, TType.I64, 'bytesNum', None, None, ), # 5 + (6, TType.MAP, 'httpHeaders', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 6 ) all_structs.append(TDBSqlCloudResultFile) TDBSqlCloudResultFile.thrift_spec = ( @@ -10111,6 +10720,9 @@ def __ne__(self, other): (3, TType.I64, 'rowCount', None, None, ), # 3 (4, TType.I64, 'uncompressedBytes', None, None, ), # 4 (5, TType.I64, 'compressedBytes', None, None, ), # 5 + (6, TType.STRING, 'fileLink', 'UTF8', None, ), # 6 + (7, TType.I64, 'linkExpiryTime', None, None, ), # 7 + (8, TType.MAP, 'httpHeaders', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 8 ) all_structs.append(TRowSet) TRowSet.thrift_spec = ( @@ -11397,510 +12009,2557 @@ def __ne__(self, other): None, # 1280 (1281, TType.LIST, 'arrowBatches', (TType.STRUCT, [TSparkArrowBatch, None], False), None, ), # 1281 (1282, TType.LIST, 'resultLinks', (TType.STRUCT, [TSparkArrowResultLink, None], False), None, ), # 1282 -) -all_structs.append(TDBSqlTempView) -TDBSqlTempView.thrift_spec = ( - None, # 0 - (1, TType.STRING, 'name', 'UTF8', None, ), # 1 - (2, TType.STRING, 'sqlStatement', 'UTF8', None, ), # 2 - (3, TType.MAP, 'properties', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 3 - (4, TType.STRING, 'viewSchema', 'UTF8', None, ), # 4 -) -all_structs.append(TDBSqlSessionCapabilities) -TDBSqlSessionCapabilities.thrift_spec = ( - None, # 0 - (1, TType.BOOL, 'supportsMultipleCatalogs', None, None, ), # 1 -) -all_structs.append(TExpressionInfo) -TExpressionInfo.thrift_spec = ( - None, # 0 - (1, TType.STRING, 'className', 'UTF8', None, ), # 1 - (2, TType.STRING, 'usage', 'UTF8', None, ), # 2 - (3, TType.STRING, 'name', 'UTF8', None, ), # 3 - (4, TType.STRING, 'extended', 'UTF8', None, ), # 4 - (5, TType.STRING, 'db', 'UTF8', None, ), # 5 - (6, TType.STRING, 'arguments', 'UTF8', None, ), # 6 - (7, TType.STRING, 'examples', 'UTF8', None, ), # 7 - (8, TType.STRING, 'note', 'UTF8', None, ), # 8 - (9, TType.STRING, 'group', 'UTF8', None, ), # 9 - (10, TType.STRING, 'since', 'UTF8', None, ), # 10 - (11, TType.STRING, 'deprecated', 'UTF8', None, ), # 11 - (12, TType.STRING, 'source', 'UTF8', None, ), # 12 -) -all_structs.append(TDBSqlConfValue) -TDBSqlConfValue.thrift_spec = ( - None, # 0 - (1, TType.STRING, 'value', 'UTF8', None, ), # 1 -) -all_structs.append(TDBSqlSessionConf) -TDBSqlSessionConf.thrift_spec = ( - None, # 0 - (1, TType.MAP, 'confs', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 1 - (2, TType.LIST, 'tempViews', (TType.STRUCT, [TDBSqlTempView, None], False), None, ), # 2 - (3, TType.STRING, 'currentDatabase', 'UTF8', None, ), # 3 - (4, TType.STRING, 'currentCatalog', 'UTF8', None, ), # 4 - (5, TType.STRUCT, 'sessionCapabilities', [TDBSqlSessionCapabilities, None], None, ), # 5 - (6, TType.LIST, 'expressionsInfos', (TType.STRUCT, [TExpressionInfo, None], False), None, ), # 6 - (7, TType.MAP, 'internalConfs', (TType.STRING, 'UTF8', TType.STRUCT, [TDBSqlConfValue, None], False), None, ), # 7 -) -all_structs.append(TStatus) -TStatus.thrift_spec = ( - None, # 0 - (1, TType.I32, 'statusCode', None, None, ), # 1 - (2, TType.LIST, 'infoMessages', (TType.STRING, 'UTF8', False), None, ), # 2 - (3, TType.STRING, 'sqlState', 'UTF8', None, ), # 3 - (4, TType.I32, 'errorCode', None, None, ), # 4 - (5, TType.STRING, 'errorMessage', 'UTF8', None, ), # 5 - (6, TType.STRING, 'displayMessage', 'UTF8', None, ), # 6 - None, # 7 - None, # 8 - None, # 9 - None, # 10 - None, # 11 - None, # 12 - None, # 13 - None, # 14 - None, # 15 - None, # 16 - None, # 17 - None, # 18 - None, # 19 - None, # 20 - None, # 21 - None, # 22 - None, # 23 - None, # 24 - None, # 25 - None, # 26 - None, # 27 - None, # 28 - None, # 29 - None, # 30 - None, # 31 - None, # 32 - None, # 33 - None, # 34 - None, # 35 - None, # 36 - None, # 37 - None, # 38 - None, # 39 - None, # 40 - None, # 41 - None, # 42 - None, # 43 - None, # 44 - None, # 45 - None, # 46 - None, # 47 - None, # 48 - None, # 49 - None, # 50 - None, # 51 - None, # 52 - None, # 53 - None, # 54 - None, # 55 - None, # 56 - None, # 57 - None, # 58 - None, # 59 - None, # 60 - None, # 61 - None, # 62 - None, # 63 - None, # 64 - None, # 65 - None, # 66 - None, # 67 - None, # 68 - None, # 69 - None, # 70 - None, # 71 - None, # 72 - None, # 73 - None, # 74 - None, # 75 - None, # 76 - None, # 77 - None, # 78 - None, # 79 - None, # 80 - None, # 81 - None, # 82 - None, # 83 - None, # 84 - None, # 85 - None, # 86 - None, # 87 - None, # 88 - None, # 89 - None, # 90 - None, # 91 - None, # 92 - None, # 93 - None, # 94 - None, # 95 - None, # 96 - None, # 97 - None, # 98 - None, # 99 - None, # 100 - None, # 101 - None, # 102 - None, # 103 - None, # 104 - None, # 105 - None, # 106 - None, # 107 - None, # 108 - None, # 109 - None, # 110 - None, # 111 - None, # 112 - None, # 113 - None, # 114 - None, # 115 - None, # 116 - None, # 117 - None, # 118 - None, # 119 - None, # 120 - None, # 121 - None, # 122 - None, # 123 - None, # 124 - None, # 125 - None, # 126 - None, # 127 - None, # 128 - None, # 129 - None, # 130 - None, # 131 - None, # 132 - None, # 133 - None, # 134 - None, # 135 - None, # 136 - None, # 137 - None, # 138 - None, # 139 - None, # 140 - None, # 141 - None, # 142 - None, # 143 - None, # 144 - None, # 145 - None, # 146 - None, # 147 - None, # 148 - None, # 149 - None, # 150 - None, # 151 - None, # 152 - None, # 153 - None, # 154 - None, # 155 - None, # 156 - None, # 157 - None, # 158 - None, # 159 - None, # 160 - None, # 161 - None, # 162 - None, # 163 - None, # 164 - None, # 165 - None, # 166 - None, # 167 - None, # 168 - None, # 169 - None, # 170 - None, # 171 - None, # 172 - None, # 173 - None, # 174 - None, # 175 - None, # 176 - None, # 177 - None, # 178 - None, # 179 - None, # 180 - None, # 181 - None, # 182 - None, # 183 - None, # 184 - None, # 185 - None, # 186 - None, # 187 - None, # 188 - None, # 189 - None, # 190 - None, # 191 - None, # 192 - None, # 193 - None, # 194 - None, # 195 - None, # 196 - None, # 197 - None, # 198 - None, # 199 - None, # 200 - None, # 201 - None, # 202 - None, # 203 - None, # 204 - None, # 205 - None, # 206 - None, # 207 - None, # 208 - None, # 209 - None, # 210 - None, # 211 - None, # 212 - None, # 213 - None, # 214 - None, # 215 - None, # 216 - None, # 217 - None, # 218 - None, # 219 - None, # 220 - None, # 221 - None, # 222 - None, # 223 - None, # 224 - None, # 225 - None, # 226 - None, # 227 - None, # 228 - None, # 229 - None, # 230 - None, # 231 - None, # 232 - None, # 233 - None, # 234 - None, # 235 - None, # 236 - None, # 237 - None, # 238 - None, # 239 - None, # 240 - None, # 241 - None, # 242 - None, # 243 - None, # 244 - None, # 245 - None, # 246 - None, # 247 - None, # 248 - None, # 249 - None, # 250 - None, # 251 - None, # 252 - None, # 253 - None, # 254 - None, # 255 - None, # 256 - None, # 257 - None, # 258 - None, # 259 - None, # 260 - None, # 261 - None, # 262 - None, # 263 - None, # 264 - None, # 265 - None, # 266 - None, # 267 - None, # 268 - None, # 269 - None, # 270 - None, # 271 - None, # 272 - None, # 273 - None, # 274 - None, # 275 - None, # 276 - None, # 277 - None, # 278 - None, # 279 - None, # 280 - None, # 281 - None, # 282 - None, # 283 - None, # 284 - None, # 285 - None, # 286 - None, # 287 - None, # 288 - None, # 289 - None, # 290 - None, # 291 - None, # 292 - None, # 293 - None, # 294 - None, # 295 - None, # 296 - None, # 297 - None, # 298 - None, # 299 - None, # 300 - None, # 301 - None, # 302 - None, # 303 - None, # 304 - None, # 305 - None, # 306 - None, # 307 - None, # 308 - None, # 309 - None, # 310 - None, # 311 - None, # 312 - None, # 313 - None, # 314 - None, # 315 - None, # 316 - None, # 317 - None, # 318 - None, # 319 - None, # 320 - None, # 321 - None, # 322 - None, # 323 - None, # 324 - None, # 325 - None, # 326 - None, # 327 - None, # 328 - None, # 329 - None, # 330 - None, # 331 - None, # 332 - None, # 333 - None, # 334 - None, # 335 - None, # 336 - None, # 337 - None, # 338 - None, # 339 - None, # 340 - None, # 341 - None, # 342 - None, # 343 - None, # 344 - None, # 345 - None, # 346 - None, # 347 - None, # 348 - None, # 349 - None, # 350 - None, # 351 - None, # 352 - None, # 353 - None, # 354 - None, # 355 - None, # 356 - None, # 357 - None, # 358 - None, # 359 - None, # 360 - None, # 361 - None, # 362 - None, # 363 - None, # 364 - None, # 365 - None, # 366 - None, # 367 - None, # 368 - None, # 369 - None, # 370 - None, # 371 - None, # 372 - None, # 373 - None, # 374 - None, # 375 - None, # 376 - None, # 377 - None, # 378 - None, # 379 - None, # 380 - None, # 381 - None, # 382 - None, # 383 - None, # 384 - None, # 385 - None, # 386 - None, # 387 - None, # 388 - None, # 389 - None, # 390 - None, # 391 - None, # 392 - None, # 393 - None, # 394 - None, # 395 - None, # 396 - None, # 397 - None, # 398 - None, # 399 - None, # 400 - None, # 401 - None, # 402 - None, # 403 - None, # 404 - None, # 405 - None, # 406 - None, # 407 - None, # 408 - None, # 409 - None, # 410 - None, # 411 - None, # 412 - None, # 413 - None, # 414 - None, # 415 - None, # 416 - None, # 417 - None, # 418 - None, # 419 - None, # 420 - None, # 421 - None, # 422 - None, # 423 - None, # 424 - None, # 425 - None, # 426 - None, # 427 - None, # 428 - None, # 429 - None, # 430 - None, # 431 - None, # 432 - None, # 433 - None, # 434 - None, # 435 - None, # 436 - None, # 437 - None, # 438 - None, # 439 - None, # 440 - None, # 441 - None, # 442 - None, # 443 - None, # 444 - None, # 445 - None, # 446 - None, # 447 - None, # 448 - None, # 449 - None, # 450 - None, # 451 - None, # 452 - None, # 453 - None, # 454 - None, # 455 + None, # 1283 + None, # 1284 + None, # 1285 + None, # 1286 + None, # 1287 + None, # 1288 + None, # 1289 + None, # 1290 + None, # 1291 + None, # 1292 + None, # 1293 + None, # 1294 + None, # 1295 + None, # 1296 + None, # 1297 + None, # 1298 + None, # 1299 + None, # 1300 + None, # 1301 + None, # 1302 + None, # 1303 + None, # 1304 + None, # 1305 + None, # 1306 + None, # 1307 + None, # 1308 + None, # 1309 + None, # 1310 + None, # 1311 + None, # 1312 + None, # 1313 + None, # 1314 + None, # 1315 + None, # 1316 + None, # 1317 + None, # 1318 + None, # 1319 + None, # 1320 + None, # 1321 + None, # 1322 + None, # 1323 + None, # 1324 + None, # 1325 + None, # 1326 + None, # 1327 + None, # 1328 + None, # 1329 + None, # 1330 + None, # 1331 + None, # 1332 + None, # 1333 + None, # 1334 + None, # 1335 + None, # 1336 + None, # 1337 + None, # 1338 + None, # 1339 + None, # 1340 + None, # 1341 + None, # 1342 + None, # 1343 + None, # 1344 + None, # 1345 + None, # 1346 + None, # 1347 + None, # 1348 + None, # 1349 + None, # 1350 + None, # 1351 + None, # 1352 + None, # 1353 + None, # 1354 + None, # 1355 + None, # 1356 + None, # 1357 + None, # 1358 + None, # 1359 + None, # 1360 + None, # 1361 + None, # 1362 + None, # 1363 + None, # 1364 + None, # 1365 + None, # 1366 + None, # 1367 + None, # 1368 + None, # 1369 + None, # 1370 + None, # 1371 + None, # 1372 + None, # 1373 + None, # 1374 + None, # 1375 + None, # 1376 + None, # 1377 + None, # 1378 + None, # 1379 + None, # 1380 + None, # 1381 + None, # 1382 + None, # 1383 + None, # 1384 + None, # 1385 + None, # 1386 + None, # 1387 + None, # 1388 + None, # 1389 + None, # 1390 + None, # 1391 + None, # 1392 + None, # 1393 + None, # 1394 + None, # 1395 + None, # 1396 + None, # 1397 + None, # 1398 + None, # 1399 + None, # 1400 + None, # 1401 + None, # 1402 + None, # 1403 + None, # 1404 + None, # 1405 + None, # 1406 + None, # 1407 + None, # 1408 + None, # 1409 + None, # 1410 + None, # 1411 + None, # 1412 + None, # 1413 + None, # 1414 + None, # 1415 + None, # 1416 + None, # 1417 + None, # 1418 + None, # 1419 + None, # 1420 + None, # 1421 + None, # 1422 + None, # 1423 + None, # 1424 + None, # 1425 + None, # 1426 + None, # 1427 + None, # 1428 + None, # 1429 + None, # 1430 + None, # 1431 + None, # 1432 + None, # 1433 + None, # 1434 + None, # 1435 + None, # 1436 + None, # 1437 + None, # 1438 + None, # 1439 + None, # 1440 + None, # 1441 + None, # 1442 + None, # 1443 + None, # 1444 + None, # 1445 + None, # 1446 + None, # 1447 + None, # 1448 + None, # 1449 + None, # 1450 + None, # 1451 + None, # 1452 + None, # 1453 + None, # 1454 + None, # 1455 + None, # 1456 + None, # 1457 + None, # 1458 + None, # 1459 + None, # 1460 + None, # 1461 + None, # 1462 + None, # 1463 + None, # 1464 + None, # 1465 + None, # 1466 + None, # 1467 + None, # 1468 + None, # 1469 + None, # 1470 + None, # 1471 + None, # 1472 + None, # 1473 + None, # 1474 + None, # 1475 + None, # 1476 + None, # 1477 + None, # 1478 + None, # 1479 + None, # 1480 + None, # 1481 + None, # 1482 + None, # 1483 + None, # 1484 + None, # 1485 + None, # 1486 + None, # 1487 + None, # 1488 + None, # 1489 + None, # 1490 + None, # 1491 + None, # 1492 + None, # 1493 + None, # 1494 + None, # 1495 + None, # 1496 + None, # 1497 + None, # 1498 + None, # 1499 + None, # 1500 + None, # 1501 + None, # 1502 + None, # 1503 + None, # 1504 + None, # 1505 + None, # 1506 + None, # 1507 + None, # 1508 + None, # 1509 + None, # 1510 + None, # 1511 + None, # 1512 + None, # 1513 + None, # 1514 + None, # 1515 + None, # 1516 + None, # 1517 + None, # 1518 + None, # 1519 + None, # 1520 + None, # 1521 + None, # 1522 + None, # 1523 + None, # 1524 + None, # 1525 + None, # 1526 + None, # 1527 + None, # 1528 + None, # 1529 + None, # 1530 + None, # 1531 + None, # 1532 + None, # 1533 + None, # 1534 + None, # 1535 + None, # 1536 + None, # 1537 + None, # 1538 + None, # 1539 + None, # 1540 + None, # 1541 + None, # 1542 + None, # 1543 + None, # 1544 + None, # 1545 + None, # 1546 + None, # 1547 + None, # 1548 + None, # 1549 + None, # 1550 + None, # 1551 + None, # 1552 + None, # 1553 + None, # 1554 + None, # 1555 + None, # 1556 + None, # 1557 + None, # 1558 + None, # 1559 + None, # 1560 + None, # 1561 + None, # 1562 + None, # 1563 + None, # 1564 + None, # 1565 + None, # 1566 + None, # 1567 + None, # 1568 + None, # 1569 + None, # 1570 + None, # 1571 + None, # 1572 + None, # 1573 + None, # 1574 + None, # 1575 + None, # 1576 + None, # 1577 + None, # 1578 + None, # 1579 + None, # 1580 + None, # 1581 + None, # 1582 + None, # 1583 + None, # 1584 + None, # 1585 + None, # 1586 + None, # 1587 + None, # 1588 + None, # 1589 + None, # 1590 + None, # 1591 + None, # 1592 + None, # 1593 + None, # 1594 + None, # 1595 + None, # 1596 + None, # 1597 + None, # 1598 + None, # 1599 + None, # 1600 + None, # 1601 + None, # 1602 + None, # 1603 + None, # 1604 + None, # 1605 + None, # 1606 + None, # 1607 + None, # 1608 + None, # 1609 + None, # 1610 + None, # 1611 + None, # 1612 + None, # 1613 + None, # 1614 + None, # 1615 + None, # 1616 + None, # 1617 + None, # 1618 + None, # 1619 + None, # 1620 + None, # 1621 + None, # 1622 + None, # 1623 + None, # 1624 + None, # 1625 + None, # 1626 + None, # 1627 + None, # 1628 + None, # 1629 + None, # 1630 + None, # 1631 + None, # 1632 + None, # 1633 + None, # 1634 + None, # 1635 + None, # 1636 + None, # 1637 + None, # 1638 + None, # 1639 + None, # 1640 + None, # 1641 + None, # 1642 + None, # 1643 + None, # 1644 + None, # 1645 + None, # 1646 + None, # 1647 + None, # 1648 + None, # 1649 + None, # 1650 + None, # 1651 + None, # 1652 + None, # 1653 + None, # 1654 + None, # 1655 + None, # 1656 + None, # 1657 + None, # 1658 + None, # 1659 + None, # 1660 + None, # 1661 + None, # 1662 + None, # 1663 + None, # 1664 + None, # 1665 + None, # 1666 + None, # 1667 + None, # 1668 + None, # 1669 + None, # 1670 + None, # 1671 + None, # 1672 + None, # 1673 + None, # 1674 + None, # 1675 + None, # 1676 + None, # 1677 + None, # 1678 + None, # 1679 + None, # 1680 + None, # 1681 + None, # 1682 + None, # 1683 + None, # 1684 + None, # 1685 + None, # 1686 + None, # 1687 + None, # 1688 + None, # 1689 + None, # 1690 + None, # 1691 + None, # 1692 + None, # 1693 + None, # 1694 + None, # 1695 + None, # 1696 + None, # 1697 + None, # 1698 + None, # 1699 + None, # 1700 + None, # 1701 + None, # 1702 + None, # 1703 + None, # 1704 + None, # 1705 + None, # 1706 + None, # 1707 + None, # 1708 + None, # 1709 + None, # 1710 + None, # 1711 + None, # 1712 + None, # 1713 + None, # 1714 + None, # 1715 + None, # 1716 + None, # 1717 + None, # 1718 + None, # 1719 + None, # 1720 + None, # 1721 + None, # 1722 + None, # 1723 + None, # 1724 + None, # 1725 + None, # 1726 + None, # 1727 + None, # 1728 + None, # 1729 + None, # 1730 + None, # 1731 + None, # 1732 + None, # 1733 + None, # 1734 + None, # 1735 + None, # 1736 + None, # 1737 + None, # 1738 + None, # 1739 + None, # 1740 + None, # 1741 + None, # 1742 + None, # 1743 + None, # 1744 + None, # 1745 + None, # 1746 + None, # 1747 + None, # 1748 + None, # 1749 + None, # 1750 + None, # 1751 + None, # 1752 + None, # 1753 + None, # 1754 + None, # 1755 + None, # 1756 + None, # 1757 + None, # 1758 + None, # 1759 + None, # 1760 + None, # 1761 + None, # 1762 + None, # 1763 + None, # 1764 + None, # 1765 + None, # 1766 + None, # 1767 + None, # 1768 + None, # 1769 + None, # 1770 + None, # 1771 + None, # 1772 + None, # 1773 + None, # 1774 + None, # 1775 + None, # 1776 + None, # 1777 + None, # 1778 + None, # 1779 + None, # 1780 + None, # 1781 + None, # 1782 + None, # 1783 + None, # 1784 + None, # 1785 + None, # 1786 + None, # 1787 + None, # 1788 + None, # 1789 + None, # 1790 + None, # 1791 + None, # 1792 + None, # 1793 + None, # 1794 + None, # 1795 + None, # 1796 + None, # 1797 + None, # 1798 + None, # 1799 + None, # 1800 + None, # 1801 + None, # 1802 + None, # 1803 + None, # 1804 + None, # 1805 + None, # 1806 + None, # 1807 + None, # 1808 + None, # 1809 + None, # 1810 + None, # 1811 + None, # 1812 + None, # 1813 + None, # 1814 + None, # 1815 + None, # 1816 + None, # 1817 + None, # 1818 + None, # 1819 + None, # 1820 + None, # 1821 + None, # 1822 + None, # 1823 + None, # 1824 + None, # 1825 + None, # 1826 + None, # 1827 + None, # 1828 + None, # 1829 + None, # 1830 + None, # 1831 + None, # 1832 + None, # 1833 + None, # 1834 + None, # 1835 + None, # 1836 + None, # 1837 + None, # 1838 + None, # 1839 + None, # 1840 + None, # 1841 + None, # 1842 + None, # 1843 + None, # 1844 + None, # 1845 + None, # 1846 + None, # 1847 + None, # 1848 + None, # 1849 + None, # 1850 + None, # 1851 + None, # 1852 + None, # 1853 + None, # 1854 + None, # 1855 + None, # 1856 + None, # 1857 + None, # 1858 + None, # 1859 + None, # 1860 + None, # 1861 + None, # 1862 + None, # 1863 + None, # 1864 + None, # 1865 + None, # 1866 + None, # 1867 + None, # 1868 + None, # 1869 + None, # 1870 + None, # 1871 + None, # 1872 + None, # 1873 + None, # 1874 + None, # 1875 + None, # 1876 + None, # 1877 + None, # 1878 + None, # 1879 + None, # 1880 + None, # 1881 + None, # 1882 + None, # 1883 + None, # 1884 + None, # 1885 + None, # 1886 + None, # 1887 + None, # 1888 + None, # 1889 + None, # 1890 + None, # 1891 + None, # 1892 + None, # 1893 + None, # 1894 + None, # 1895 + None, # 1896 + None, # 1897 + None, # 1898 + None, # 1899 + None, # 1900 + None, # 1901 + None, # 1902 + None, # 1903 + None, # 1904 + None, # 1905 + None, # 1906 + None, # 1907 + None, # 1908 + None, # 1909 + None, # 1910 + None, # 1911 + None, # 1912 + None, # 1913 + None, # 1914 + None, # 1915 + None, # 1916 + None, # 1917 + None, # 1918 + None, # 1919 + None, # 1920 + None, # 1921 + None, # 1922 + None, # 1923 + None, # 1924 + None, # 1925 + None, # 1926 + None, # 1927 + None, # 1928 + None, # 1929 + None, # 1930 + None, # 1931 + None, # 1932 + None, # 1933 + None, # 1934 + None, # 1935 + None, # 1936 + None, # 1937 + None, # 1938 + None, # 1939 + None, # 1940 + None, # 1941 + None, # 1942 + None, # 1943 + None, # 1944 + None, # 1945 + None, # 1946 + None, # 1947 + None, # 1948 + None, # 1949 + None, # 1950 + None, # 1951 + None, # 1952 + None, # 1953 + None, # 1954 + None, # 1955 + None, # 1956 + None, # 1957 + None, # 1958 + None, # 1959 + None, # 1960 + None, # 1961 + None, # 1962 + None, # 1963 + None, # 1964 + None, # 1965 + None, # 1966 + None, # 1967 + None, # 1968 + None, # 1969 + None, # 1970 + None, # 1971 + None, # 1972 + None, # 1973 + None, # 1974 + None, # 1975 + None, # 1976 + None, # 1977 + None, # 1978 + None, # 1979 + None, # 1980 + None, # 1981 + None, # 1982 + None, # 1983 + None, # 1984 + None, # 1985 + None, # 1986 + None, # 1987 + None, # 1988 + None, # 1989 + None, # 1990 + None, # 1991 + None, # 1992 + None, # 1993 + None, # 1994 + None, # 1995 + None, # 1996 + None, # 1997 + None, # 1998 + None, # 1999 + None, # 2000 + None, # 2001 + None, # 2002 + None, # 2003 + None, # 2004 + None, # 2005 + None, # 2006 + None, # 2007 + None, # 2008 + None, # 2009 + None, # 2010 + None, # 2011 + None, # 2012 + None, # 2013 + None, # 2014 + None, # 2015 + None, # 2016 + None, # 2017 + None, # 2018 + None, # 2019 + None, # 2020 + None, # 2021 + None, # 2022 + None, # 2023 + None, # 2024 + None, # 2025 + None, # 2026 + None, # 2027 + None, # 2028 + None, # 2029 + None, # 2030 + None, # 2031 + None, # 2032 + None, # 2033 + None, # 2034 + None, # 2035 + None, # 2036 + None, # 2037 + None, # 2038 + None, # 2039 + None, # 2040 + None, # 2041 + None, # 2042 + None, # 2043 + None, # 2044 + None, # 2045 + None, # 2046 + None, # 2047 + None, # 2048 + None, # 2049 + None, # 2050 + None, # 2051 + None, # 2052 + None, # 2053 + None, # 2054 + None, # 2055 + None, # 2056 + None, # 2057 + None, # 2058 + None, # 2059 + None, # 2060 + None, # 2061 + None, # 2062 + None, # 2063 + None, # 2064 + None, # 2065 + None, # 2066 + None, # 2067 + None, # 2068 + None, # 2069 + None, # 2070 + None, # 2071 + None, # 2072 + None, # 2073 + None, # 2074 + None, # 2075 + None, # 2076 + None, # 2077 + None, # 2078 + None, # 2079 + None, # 2080 + None, # 2081 + None, # 2082 + None, # 2083 + None, # 2084 + None, # 2085 + None, # 2086 + None, # 2087 + None, # 2088 + None, # 2089 + None, # 2090 + None, # 2091 + None, # 2092 + None, # 2093 + None, # 2094 + None, # 2095 + None, # 2096 + None, # 2097 + None, # 2098 + None, # 2099 + None, # 2100 + None, # 2101 + None, # 2102 + None, # 2103 + None, # 2104 + None, # 2105 + None, # 2106 + None, # 2107 + None, # 2108 + None, # 2109 + None, # 2110 + None, # 2111 + None, # 2112 + None, # 2113 + None, # 2114 + None, # 2115 + None, # 2116 + None, # 2117 + None, # 2118 + None, # 2119 + None, # 2120 + None, # 2121 + None, # 2122 + None, # 2123 + None, # 2124 + None, # 2125 + None, # 2126 + None, # 2127 + None, # 2128 + None, # 2129 + None, # 2130 + None, # 2131 + None, # 2132 + None, # 2133 + None, # 2134 + None, # 2135 + None, # 2136 + None, # 2137 + None, # 2138 + None, # 2139 + None, # 2140 + None, # 2141 + None, # 2142 + None, # 2143 + None, # 2144 + None, # 2145 + None, # 2146 + None, # 2147 + None, # 2148 + None, # 2149 + None, # 2150 + None, # 2151 + None, # 2152 + None, # 2153 + None, # 2154 + None, # 2155 + None, # 2156 + None, # 2157 + None, # 2158 + None, # 2159 + None, # 2160 + None, # 2161 + None, # 2162 + None, # 2163 + None, # 2164 + None, # 2165 + None, # 2166 + None, # 2167 + None, # 2168 + None, # 2169 + None, # 2170 + None, # 2171 + None, # 2172 + None, # 2173 + None, # 2174 + None, # 2175 + None, # 2176 + None, # 2177 + None, # 2178 + None, # 2179 + None, # 2180 + None, # 2181 + None, # 2182 + None, # 2183 + None, # 2184 + None, # 2185 + None, # 2186 + None, # 2187 + None, # 2188 + None, # 2189 + None, # 2190 + None, # 2191 + None, # 2192 + None, # 2193 + None, # 2194 + None, # 2195 + None, # 2196 + None, # 2197 + None, # 2198 + None, # 2199 + None, # 2200 + None, # 2201 + None, # 2202 + None, # 2203 + None, # 2204 + None, # 2205 + None, # 2206 + None, # 2207 + None, # 2208 + None, # 2209 + None, # 2210 + None, # 2211 + None, # 2212 + None, # 2213 + None, # 2214 + None, # 2215 + None, # 2216 + None, # 2217 + None, # 2218 + None, # 2219 + None, # 2220 + None, # 2221 + None, # 2222 + None, # 2223 + None, # 2224 + None, # 2225 + None, # 2226 + None, # 2227 + None, # 2228 + None, # 2229 + None, # 2230 + None, # 2231 + None, # 2232 + None, # 2233 + None, # 2234 + None, # 2235 + None, # 2236 + None, # 2237 + None, # 2238 + None, # 2239 + None, # 2240 + None, # 2241 + None, # 2242 + None, # 2243 + None, # 2244 + None, # 2245 + None, # 2246 + None, # 2247 + None, # 2248 + None, # 2249 + None, # 2250 + None, # 2251 + None, # 2252 + None, # 2253 + None, # 2254 + None, # 2255 + None, # 2256 + None, # 2257 + None, # 2258 + None, # 2259 + None, # 2260 + None, # 2261 + None, # 2262 + None, # 2263 + None, # 2264 + None, # 2265 + None, # 2266 + None, # 2267 + None, # 2268 + None, # 2269 + None, # 2270 + None, # 2271 + None, # 2272 + None, # 2273 + None, # 2274 + None, # 2275 + None, # 2276 + None, # 2277 + None, # 2278 + None, # 2279 + None, # 2280 + None, # 2281 + None, # 2282 + None, # 2283 + None, # 2284 + None, # 2285 + None, # 2286 + None, # 2287 + None, # 2288 + None, # 2289 + None, # 2290 + None, # 2291 + None, # 2292 + None, # 2293 + None, # 2294 + None, # 2295 + None, # 2296 + None, # 2297 + None, # 2298 + None, # 2299 + None, # 2300 + None, # 2301 + None, # 2302 + None, # 2303 + None, # 2304 + None, # 2305 + None, # 2306 + None, # 2307 + None, # 2308 + None, # 2309 + None, # 2310 + None, # 2311 + None, # 2312 + None, # 2313 + None, # 2314 + None, # 2315 + None, # 2316 + None, # 2317 + None, # 2318 + None, # 2319 + None, # 2320 + None, # 2321 + None, # 2322 + None, # 2323 + None, # 2324 + None, # 2325 + None, # 2326 + None, # 2327 + None, # 2328 + None, # 2329 + None, # 2330 + None, # 2331 + None, # 2332 + None, # 2333 + None, # 2334 + None, # 2335 + None, # 2336 + None, # 2337 + None, # 2338 + None, # 2339 + None, # 2340 + None, # 2341 + None, # 2342 + None, # 2343 + None, # 2344 + None, # 2345 + None, # 2346 + None, # 2347 + None, # 2348 + None, # 2349 + None, # 2350 + None, # 2351 + None, # 2352 + None, # 2353 + None, # 2354 + None, # 2355 + None, # 2356 + None, # 2357 + None, # 2358 + None, # 2359 + None, # 2360 + None, # 2361 + None, # 2362 + None, # 2363 + None, # 2364 + None, # 2365 + None, # 2366 + None, # 2367 + None, # 2368 + None, # 2369 + None, # 2370 + None, # 2371 + None, # 2372 + None, # 2373 + None, # 2374 + None, # 2375 + None, # 2376 + None, # 2377 + None, # 2378 + None, # 2379 + None, # 2380 + None, # 2381 + None, # 2382 + None, # 2383 + None, # 2384 + None, # 2385 + None, # 2386 + None, # 2387 + None, # 2388 + None, # 2389 + None, # 2390 + None, # 2391 + None, # 2392 + None, # 2393 + None, # 2394 + None, # 2395 + None, # 2396 + None, # 2397 + None, # 2398 + None, # 2399 + None, # 2400 + None, # 2401 + None, # 2402 + None, # 2403 + None, # 2404 + None, # 2405 + None, # 2406 + None, # 2407 + None, # 2408 + None, # 2409 + None, # 2410 + None, # 2411 + None, # 2412 + None, # 2413 + None, # 2414 + None, # 2415 + None, # 2416 + None, # 2417 + None, # 2418 + None, # 2419 + None, # 2420 + None, # 2421 + None, # 2422 + None, # 2423 + None, # 2424 + None, # 2425 + None, # 2426 + None, # 2427 + None, # 2428 + None, # 2429 + None, # 2430 + None, # 2431 + None, # 2432 + None, # 2433 + None, # 2434 + None, # 2435 + None, # 2436 + None, # 2437 + None, # 2438 + None, # 2439 + None, # 2440 + None, # 2441 + None, # 2442 + None, # 2443 + None, # 2444 + None, # 2445 + None, # 2446 + None, # 2447 + None, # 2448 + None, # 2449 + None, # 2450 + None, # 2451 + None, # 2452 + None, # 2453 + None, # 2454 + None, # 2455 + None, # 2456 + None, # 2457 + None, # 2458 + None, # 2459 + None, # 2460 + None, # 2461 + None, # 2462 + None, # 2463 + None, # 2464 + None, # 2465 + None, # 2466 + None, # 2467 + None, # 2468 + None, # 2469 + None, # 2470 + None, # 2471 + None, # 2472 + None, # 2473 + None, # 2474 + None, # 2475 + None, # 2476 + None, # 2477 + None, # 2478 + None, # 2479 + None, # 2480 + None, # 2481 + None, # 2482 + None, # 2483 + None, # 2484 + None, # 2485 + None, # 2486 + None, # 2487 + None, # 2488 + None, # 2489 + None, # 2490 + None, # 2491 + None, # 2492 + None, # 2493 + None, # 2494 + None, # 2495 + None, # 2496 + None, # 2497 + None, # 2498 + None, # 2499 + None, # 2500 + None, # 2501 + None, # 2502 + None, # 2503 + None, # 2504 + None, # 2505 + None, # 2506 + None, # 2507 + None, # 2508 + None, # 2509 + None, # 2510 + None, # 2511 + None, # 2512 + None, # 2513 + None, # 2514 + None, # 2515 + None, # 2516 + None, # 2517 + None, # 2518 + None, # 2519 + None, # 2520 + None, # 2521 + None, # 2522 + None, # 2523 + None, # 2524 + None, # 2525 + None, # 2526 + None, # 2527 + None, # 2528 + None, # 2529 + None, # 2530 + None, # 2531 + None, # 2532 + None, # 2533 + None, # 2534 + None, # 2535 + None, # 2536 + None, # 2537 + None, # 2538 + None, # 2539 + None, # 2540 + None, # 2541 + None, # 2542 + None, # 2543 + None, # 2544 + None, # 2545 + None, # 2546 + None, # 2547 + None, # 2548 + None, # 2549 + None, # 2550 + None, # 2551 + None, # 2552 + None, # 2553 + None, # 2554 + None, # 2555 + None, # 2556 + None, # 2557 + None, # 2558 + None, # 2559 + None, # 2560 + None, # 2561 + None, # 2562 + None, # 2563 + None, # 2564 + None, # 2565 + None, # 2566 + None, # 2567 + None, # 2568 + None, # 2569 + None, # 2570 + None, # 2571 + None, # 2572 + None, # 2573 + None, # 2574 + None, # 2575 + None, # 2576 + None, # 2577 + None, # 2578 + None, # 2579 + None, # 2580 + None, # 2581 + None, # 2582 + None, # 2583 + None, # 2584 + None, # 2585 + None, # 2586 + None, # 2587 + None, # 2588 + None, # 2589 + None, # 2590 + None, # 2591 + None, # 2592 + None, # 2593 + None, # 2594 + None, # 2595 + None, # 2596 + None, # 2597 + None, # 2598 + None, # 2599 + None, # 2600 + None, # 2601 + None, # 2602 + None, # 2603 + None, # 2604 + None, # 2605 + None, # 2606 + None, # 2607 + None, # 2608 + None, # 2609 + None, # 2610 + None, # 2611 + None, # 2612 + None, # 2613 + None, # 2614 + None, # 2615 + None, # 2616 + None, # 2617 + None, # 2618 + None, # 2619 + None, # 2620 + None, # 2621 + None, # 2622 + None, # 2623 + None, # 2624 + None, # 2625 + None, # 2626 + None, # 2627 + None, # 2628 + None, # 2629 + None, # 2630 + None, # 2631 + None, # 2632 + None, # 2633 + None, # 2634 + None, # 2635 + None, # 2636 + None, # 2637 + None, # 2638 + None, # 2639 + None, # 2640 + None, # 2641 + None, # 2642 + None, # 2643 + None, # 2644 + None, # 2645 + None, # 2646 + None, # 2647 + None, # 2648 + None, # 2649 + None, # 2650 + None, # 2651 + None, # 2652 + None, # 2653 + None, # 2654 + None, # 2655 + None, # 2656 + None, # 2657 + None, # 2658 + None, # 2659 + None, # 2660 + None, # 2661 + None, # 2662 + None, # 2663 + None, # 2664 + None, # 2665 + None, # 2666 + None, # 2667 + None, # 2668 + None, # 2669 + None, # 2670 + None, # 2671 + None, # 2672 + None, # 2673 + None, # 2674 + None, # 2675 + None, # 2676 + None, # 2677 + None, # 2678 + None, # 2679 + None, # 2680 + None, # 2681 + None, # 2682 + None, # 2683 + None, # 2684 + None, # 2685 + None, # 2686 + None, # 2687 + None, # 2688 + None, # 2689 + None, # 2690 + None, # 2691 + None, # 2692 + None, # 2693 + None, # 2694 + None, # 2695 + None, # 2696 + None, # 2697 + None, # 2698 + None, # 2699 + None, # 2700 + None, # 2701 + None, # 2702 + None, # 2703 + None, # 2704 + None, # 2705 + None, # 2706 + None, # 2707 + None, # 2708 + None, # 2709 + None, # 2710 + None, # 2711 + None, # 2712 + None, # 2713 + None, # 2714 + None, # 2715 + None, # 2716 + None, # 2717 + None, # 2718 + None, # 2719 + None, # 2720 + None, # 2721 + None, # 2722 + None, # 2723 + None, # 2724 + None, # 2725 + None, # 2726 + None, # 2727 + None, # 2728 + None, # 2729 + None, # 2730 + None, # 2731 + None, # 2732 + None, # 2733 + None, # 2734 + None, # 2735 + None, # 2736 + None, # 2737 + None, # 2738 + None, # 2739 + None, # 2740 + None, # 2741 + None, # 2742 + None, # 2743 + None, # 2744 + None, # 2745 + None, # 2746 + None, # 2747 + None, # 2748 + None, # 2749 + None, # 2750 + None, # 2751 + None, # 2752 + None, # 2753 + None, # 2754 + None, # 2755 + None, # 2756 + None, # 2757 + None, # 2758 + None, # 2759 + None, # 2760 + None, # 2761 + None, # 2762 + None, # 2763 + None, # 2764 + None, # 2765 + None, # 2766 + None, # 2767 + None, # 2768 + None, # 2769 + None, # 2770 + None, # 2771 + None, # 2772 + None, # 2773 + None, # 2774 + None, # 2775 + None, # 2776 + None, # 2777 + None, # 2778 + None, # 2779 + None, # 2780 + None, # 2781 + None, # 2782 + None, # 2783 + None, # 2784 + None, # 2785 + None, # 2786 + None, # 2787 + None, # 2788 + None, # 2789 + None, # 2790 + None, # 2791 + None, # 2792 + None, # 2793 + None, # 2794 + None, # 2795 + None, # 2796 + None, # 2797 + None, # 2798 + None, # 2799 + None, # 2800 + None, # 2801 + None, # 2802 + None, # 2803 + None, # 2804 + None, # 2805 + None, # 2806 + None, # 2807 + None, # 2808 + None, # 2809 + None, # 2810 + None, # 2811 + None, # 2812 + None, # 2813 + None, # 2814 + None, # 2815 + None, # 2816 + None, # 2817 + None, # 2818 + None, # 2819 + None, # 2820 + None, # 2821 + None, # 2822 + None, # 2823 + None, # 2824 + None, # 2825 + None, # 2826 + None, # 2827 + None, # 2828 + None, # 2829 + None, # 2830 + None, # 2831 + None, # 2832 + None, # 2833 + None, # 2834 + None, # 2835 + None, # 2836 + None, # 2837 + None, # 2838 + None, # 2839 + None, # 2840 + None, # 2841 + None, # 2842 + None, # 2843 + None, # 2844 + None, # 2845 + None, # 2846 + None, # 2847 + None, # 2848 + None, # 2849 + None, # 2850 + None, # 2851 + None, # 2852 + None, # 2853 + None, # 2854 + None, # 2855 + None, # 2856 + None, # 2857 + None, # 2858 + None, # 2859 + None, # 2860 + None, # 2861 + None, # 2862 + None, # 2863 + None, # 2864 + None, # 2865 + None, # 2866 + None, # 2867 + None, # 2868 + None, # 2869 + None, # 2870 + None, # 2871 + None, # 2872 + None, # 2873 + None, # 2874 + None, # 2875 + None, # 2876 + None, # 2877 + None, # 2878 + None, # 2879 + None, # 2880 + None, # 2881 + None, # 2882 + None, # 2883 + None, # 2884 + None, # 2885 + None, # 2886 + None, # 2887 + None, # 2888 + None, # 2889 + None, # 2890 + None, # 2891 + None, # 2892 + None, # 2893 + None, # 2894 + None, # 2895 + None, # 2896 + None, # 2897 + None, # 2898 + None, # 2899 + None, # 2900 + None, # 2901 + None, # 2902 + None, # 2903 + None, # 2904 + None, # 2905 + None, # 2906 + None, # 2907 + None, # 2908 + None, # 2909 + None, # 2910 + None, # 2911 + None, # 2912 + None, # 2913 + None, # 2914 + None, # 2915 + None, # 2916 + None, # 2917 + None, # 2918 + None, # 2919 + None, # 2920 + None, # 2921 + None, # 2922 + None, # 2923 + None, # 2924 + None, # 2925 + None, # 2926 + None, # 2927 + None, # 2928 + None, # 2929 + None, # 2930 + None, # 2931 + None, # 2932 + None, # 2933 + None, # 2934 + None, # 2935 + None, # 2936 + None, # 2937 + None, # 2938 + None, # 2939 + None, # 2940 + None, # 2941 + None, # 2942 + None, # 2943 + None, # 2944 + None, # 2945 + None, # 2946 + None, # 2947 + None, # 2948 + None, # 2949 + None, # 2950 + None, # 2951 + None, # 2952 + None, # 2953 + None, # 2954 + None, # 2955 + None, # 2956 + None, # 2957 + None, # 2958 + None, # 2959 + None, # 2960 + None, # 2961 + None, # 2962 + None, # 2963 + None, # 2964 + None, # 2965 + None, # 2966 + None, # 2967 + None, # 2968 + None, # 2969 + None, # 2970 + None, # 2971 + None, # 2972 + None, # 2973 + None, # 2974 + None, # 2975 + None, # 2976 + None, # 2977 + None, # 2978 + None, # 2979 + None, # 2980 + None, # 2981 + None, # 2982 + None, # 2983 + None, # 2984 + None, # 2985 + None, # 2986 + None, # 2987 + None, # 2988 + None, # 2989 + None, # 2990 + None, # 2991 + None, # 2992 + None, # 2993 + None, # 2994 + None, # 2995 + None, # 2996 + None, # 2997 + None, # 2998 + None, # 2999 + None, # 3000 + None, # 3001 + None, # 3002 + None, # 3003 + None, # 3004 + None, # 3005 + None, # 3006 + None, # 3007 + None, # 3008 + None, # 3009 + None, # 3010 + None, # 3011 + None, # 3012 + None, # 3013 + None, # 3014 + None, # 3015 + None, # 3016 + None, # 3017 + None, # 3018 + None, # 3019 + None, # 3020 + None, # 3021 + None, # 3022 + None, # 3023 + None, # 3024 + None, # 3025 + None, # 3026 + None, # 3027 + None, # 3028 + None, # 3029 + None, # 3030 + None, # 3031 + None, # 3032 + None, # 3033 + None, # 3034 + None, # 3035 + None, # 3036 + None, # 3037 + None, # 3038 + None, # 3039 + None, # 3040 + None, # 3041 + None, # 3042 + None, # 3043 + None, # 3044 + None, # 3045 + None, # 3046 + None, # 3047 + None, # 3048 + None, # 3049 + None, # 3050 + None, # 3051 + None, # 3052 + None, # 3053 + None, # 3054 + None, # 3055 + None, # 3056 + None, # 3057 + None, # 3058 + None, # 3059 + None, # 3060 + None, # 3061 + None, # 3062 + None, # 3063 + None, # 3064 + None, # 3065 + None, # 3066 + None, # 3067 + None, # 3068 + None, # 3069 + None, # 3070 + None, # 3071 + None, # 3072 + None, # 3073 + None, # 3074 + None, # 3075 + None, # 3076 + None, # 3077 + None, # 3078 + None, # 3079 + None, # 3080 + None, # 3081 + None, # 3082 + None, # 3083 + None, # 3084 + None, # 3085 + None, # 3086 + None, # 3087 + None, # 3088 + None, # 3089 + None, # 3090 + None, # 3091 + None, # 3092 + None, # 3093 + None, # 3094 + None, # 3095 + None, # 3096 + None, # 3097 + None, # 3098 + None, # 3099 + None, # 3100 + None, # 3101 + None, # 3102 + None, # 3103 + None, # 3104 + None, # 3105 + None, # 3106 + None, # 3107 + None, # 3108 + None, # 3109 + None, # 3110 + None, # 3111 + None, # 3112 + None, # 3113 + None, # 3114 + None, # 3115 + None, # 3116 + None, # 3117 + None, # 3118 + None, # 3119 + None, # 3120 + None, # 3121 + None, # 3122 + None, # 3123 + None, # 3124 + None, # 3125 + None, # 3126 + None, # 3127 + None, # 3128 + None, # 3129 + None, # 3130 + None, # 3131 + None, # 3132 + None, # 3133 + None, # 3134 + None, # 3135 + None, # 3136 + None, # 3137 + None, # 3138 + None, # 3139 + None, # 3140 + None, # 3141 + None, # 3142 + None, # 3143 + None, # 3144 + None, # 3145 + None, # 3146 + None, # 3147 + None, # 3148 + None, # 3149 + None, # 3150 + None, # 3151 + None, # 3152 + None, # 3153 + None, # 3154 + None, # 3155 + None, # 3156 + None, # 3157 + None, # 3158 + None, # 3159 + None, # 3160 + None, # 3161 + None, # 3162 + None, # 3163 + None, # 3164 + None, # 3165 + None, # 3166 + None, # 3167 + None, # 3168 + None, # 3169 + None, # 3170 + None, # 3171 + None, # 3172 + None, # 3173 + None, # 3174 + None, # 3175 + None, # 3176 + None, # 3177 + None, # 3178 + None, # 3179 + None, # 3180 + None, # 3181 + None, # 3182 + None, # 3183 + None, # 3184 + None, # 3185 + None, # 3186 + None, # 3187 + None, # 3188 + None, # 3189 + None, # 3190 + None, # 3191 + None, # 3192 + None, # 3193 + None, # 3194 + None, # 3195 + None, # 3196 + None, # 3197 + None, # 3198 + None, # 3199 + None, # 3200 + None, # 3201 + None, # 3202 + None, # 3203 + None, # 3204 + None, # 3205 + None, # 3206 + None, # 3207 + None, # 3208 + None, # 3209 + None, # 3210 + None, # 3211 + None, # 3212 + None, # 3213 + None, # 3214 + None, # 3215 + None, # 3216 + None, # 3217 + None, # 3218 + None, # 3219 + None, # 3220 + None, # 3221 + None, # 3222 + None, # 3223 + None, # 3224 + None, # 3225 + None, # 3226 + None, # 3227 + None, # 3228 + None, # 3229 + None, # 3230 + None, # 3231 + None, # 3232 + None, # 3233 + None, # 3234 + None, # 3235 + None, # 3236 + None, # 3237 + None, # 3238 + None, # 3239 + None, # 3240 + None, # 3241 + None, # 3242 + None, # 3243 + None, # 3244 + None, # 3245 + None, # 3246 + None, # 3247 + None, # 3248 + None, # 3249 + None, # 3250 + None, # 3251 + None, # 3252 + None, # 3253 + None, # 3254 + None, # 3255 + None, # 3256 + None, # 3257 + None, # 3258 + None, # 3259 + None, # 3260 + None, # 3261 + None, # 3262 + None, # 3263 + None, # 3264 + None, # 3265 + None, # 3266 + None, # 3267 + None, # 3268 + None, # 3269 + None, # 3270 + None, # 3271 + None, # 3272 + None, # 3273 + None, # 3274 + None, # 3275 + None, # 3276 + None, # 3277 + None, # 3278 + None, # 3279 + None, # 3280 + None, # 3281 + None, # 3282 + None, # 3283 + None, # 3284 + None, # 3285 + None, # 3286 + None, # 3287 + None, # 3288 + None, # 3289 + None, # 3290 + None, # 3291 + None, # 3292 + None, # 3293 + None, # 3294 + None, # 3295 + None, # 3296 + None, # 3297 + None, # 3298 + None, # 3299 + None, # 3300 + None, # 3301 + None, # 3302 + None, # 3303 + None, # 3304 + None, # 3305 + None, # 3306 + None, # 3307 + None, # 3308 + None, # 3309 + None, # 3310 + None, # 3311 + None, # 3312 + None, # 3313 + None, # 3314 + None, # 3315 + None, # 3316 + None, # 3317 + None, # 3318 + None, # 3319 + None, # 3320 + None, # 3321 + None, # 3322 + None, # 3323 + None, # 3324 + None, # 3325 + None, # 3326 + None, # 3327 + None, # 3328 + (3329, TType.LIST, 'cloudFetchResults', (TType.STRUCT, [TDBSqlCloudResultFile, None], False), None, ), # 3329 +) +all_structs.append(TDBSqlTempView) +TDBSqlTempView.thrift_spec = ( + None, # 0 + (1, TType.STRING, 'name', 'UTF8', None, ), # 1 + (2, TType.STRING, 'sqlStatement', 'UTF8', None, ), # 2 + (3, TType.MAP, 'properties', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 3 + (4, TType.STRING, 'viewSchema', 'UTF8', None, ), # 4 +) +all_structs.append(TDBSqlSessionCapabilities) +TDBSqlSessionCapabilities.thrift_spec = ( + None, # 0 + (1, TType.BOOL, 'supportsMultipleCatalogs', None, None, ), # 1 +) +all_structs.append(TExpressionInfo) +TExpressionInfo.thrift_spec = ( + None, # 0 + (1, TType.STRING, 'className', 'UTF8', None, ), # 1 + (2, TType.STRING, 'usage', 'UTF8', None, ), # 2 + (3, TType.STRING, 'name', 'UTF8', None, ), # 3 + (4, TType.STRING, 'extended', 'UTF8', None, ), # 4 + (5, TType.STRING, 'db', 'UTF8', None, ), # 5 + (6, TType.STRING, 'arguments', 'UTF8', None, ), # 6 + (7, TType.STRING, 'examples', 'UTF8', None, ), # 7 + (8, TType.STRING, 'note', 'UTF8', None, ), # 8 + (9, TType.STRING, 'group', 'UTF8', None, ), # 9 + (10, TType.STRING, 'since', 'UTF8', None, ), # 10 + (11, TType.STRING, 'deprecated', 'UTF8', None, ), # 11 + (12, TType.STRING, 'source', 'UTF8', None, ), # 12 +) +all_structs.append(TDBSqlConfValue) +TDBSqlConfValue.thrift_spec = ( + None, # 0 + (1, TType.STRING, 'value', 'UTF8', None, ), # 1 +) +all_structs.append(TDBSqlSessionConf) +TDBSqlSessionConf.thrift_spec = ( + None, # 0 + (1, TType.MAP, 'confs', (TType.STRING, 'UTF8', TType.STRING, 'UTF8', False), None, ), # 1 + (2, TType.LIST, 'tempViews', (TType.STRUCT, [TDBSqlTempView, None], False), None, ), # 2 + (3, TType.STRING, 'currentDatabase', 'UTF8', None, ), # 3 + (4, TType.STRING, 'currentCatalog', 'UTF8', None, ), # 4 + (5, TType.STRUCT, 'sessionCapabilities', [TDBSqlSessionCapabilities, None], None, ), # 5 + (6, TType.LIST, 'expressionsInfos', (TType.STRUCT, [TExpressionInfo, None], False), None, ), # 6 + (7, TType.MAP, 'internalConfs', (TType.STRING, 'UTF8', TType.STRUCT, [TDBSqlConfValue, None], False), None, ), # 7 +) +all_structs.append(TStatus) +TStatus.thrift_spec = ( + None, # 0 + (1, TType.I32, 'statusCode', None, None, ), # 1 + (2, TType.LIST, 'infoMessages', (TType.STRING, 'UTF8', False), None, ), # 2 + (3, TType.STRING, 'sqlState', 'UTF8', None, ), # 3 + (4, TType.I32, 'errorCode', None, None, ), # 4 + (5, TType.STRING, 'errorMessage', 'UTF8', None, ), # 5 + (6, TType.STRING, 'displayMessage', 'UTF8', None, ), # 6 + None, # 7 + None, # 8 + None, # 9 + None, # 10 + None, # 11 + None, # 12 + None, # 13 + None, # 14 + None, # 15 + None, # 16 + None, # 17 + None, # 18 + None, # 19 + None, # 20 + None, # 21 + None, # 22 + None, # 23 + None, # 24 + None, # 25 + None, # 26 + None, # 27 + None, # 28 + None, # 29 + None, # 30 + None, # 31 + None, # 32 + None, # 33 + None, # 34 + None, # 35 + None, # 36 + None, # 37 + None, # 38 + None, # 39 + None, # 40 + None, # 41 + None, # 42 + None, # 43 + None, # 44 + None, # 45 + None, # 46 + None, # 47 + None, # 48 + None, # 49 + None, # 50 + None, # 51 + None, # 52 + None, # 53 + None, # 54 + None, # 55 + None, # 56 + None, # 57 + None, # 58 + None, # 59 + None, # 60 + None, # 61 + None, # 62 + None, # 63 + None, # 64 + None, # 65 + None, # 66 + None, # 67 + None, # 68 + None, # 69 + None, # 70 + None, # 71 + None, # 72 + None, # 73 + None, # 74 + None, # 75 + None, # 76 + None, # 77 + None, # 78 + None, # 79 + None, # 80 + None, # 81 + None, # 82 + None, # 83 + None, # 84 + None, # 85 + None, # 86 + None, # 87 + None, # 88 + None, # 89 + None, # 90 + None, # 91 + None, # 92 + None, # 93 + None, # 94 + None, # 95 + None, # 96 + None, # 97 + None, # 98 + None, # 99 + None, # 100 + None, # 101 + None, # 102 + None, # 103 + None, # 104 + None, # 105 + None, # 106 + None, # 107 + None, # 108 + None, # 109 + None, # 110 + None, # 111 + None, # 112 + None, # 113 + None, # 114 + None, # 115 + None, # 116 + None, # 117 + None, # 118 + None, # 119 + None, # 120 + None, # 121 + None, # 122 + None, # 123 + None, # 124 + None, # 125 + None, # 126 + None, # 127 + None, # 128 + None, # 129 + None, # 130 + None, # 131 + None, # 132 + None, # 133 + None, # 134 + None, # 135 + None, # 136 + None, # 137 + None, # 138 + None, # 139 + None, # 140 + None, # 141 + None, # 142 + None, # 143 + None, # 144 + None, # 145 + None, # 146 + None, # 147 + None, # 148 + None, # 149 + None, # 150 + None, # 151 + None, # 152 + None, # 153 + None, # 154 + None, # 155 + None, # 156 + None, # 157 + None, # 158 + None, # 159 + None, # 160 + None, # 161 + None, # 162 + None, # 163 + None, # 164 + None, # 165 + None, # 166 + None, # 167 + None, # 168 + None, # 169 + None, # 170 + None, # 171 + None, # 172 + None, # 173 + None, # 174 + None, # 175 + None, # 176 + None, # 177 + None, # 178 + None, # 179 + None, # 180 + None, # 181 + None, # 182 + None, # 183 + None, # 184 + None, # 185 + None, # 186 + None, # 187 + None, # 188 + None, # 189 + None, # 190 + None, # 191 + None, # 192 + None, # 193 + None, # 194 + None, # 195 + None, # 196 + None, # 197 + None, # 198 + None, # 199 + None, # 200 + None, # 201 + None, # 202 + None, # 203 + None, # 204 + None, # 205 + None, # 206 + None, # 207 + None, # 208 + None, # 209 + None, # 210 + None, # 211 + None, # 212 + None, # 213 + None, # 214 + None, # 215 + None, # 216 + None, # 217 + None, # 218 + None, # 219 + None, # 220 + None, # 221 + None, # 222 + None, # 223 + None, # 224 + None, # 225 + None, # 226 + None, # 227 + None, # 228 + None, # 229 + None, # 230 + None, # 231 + None, # 232 + None, # 233 + None, # 234 + None, # 235 + None, # 236 + None, # 237 + None, # 238 + None, # 239 + None, # 240 + None, # 241 + None, # 242 + None, # 243 + None, # 244 + None, # 245 + None, # 246 + None, # 247 + None, # 248 + None, # 249 + None, # 250 + None, # 251 + None, # 252 + None, # 253 + None, # 254 + None, # 255 + None, # 256 + None, # 257 + None, # 258 + None, # 259 + None, # 260 + None, # 261 + None, # 262 + None, # 263 + None, # 264 + None, # 265 + None, # 266 + None, # 267 + None, # 268 + None, # 269 + None, # 270 + None, # 271 + None, # 272 + None, # 273 + None, # 274 + None, # 275 + None, # 276 + None, # 277 + None, # 278 + None, # 279 + None, # 280 + None, # 281 + None, # 282 + None, # 283 + None, # 284 + None, # 285 + None, # 286 + None, # 287 + None, # 288 + None, # 289 + None, # 290 + None, # 291 + None, # 292 + None, # 293 + None, # 294 + None, # 295 + None, # 296 + None, # 297 + None, # 298 + None, # 299 + None, # 300 + None, # 301 + None, # 302 + None, # 303 + None, # 304 + None, # 305 + None, # 306 + None, # 307 + None, # 308 + None, # 309 + None, # 310 + None, # 311 + None, # 312 + None, # 313 + None, # 314 + None, # 315 + None, # 316 + None, # 317 + None, # 318 + None, # 319 + None, # 320 + None, # 321 + None, # 322 + None, # 323 + None, # 324 + None, # 325 + None, # 326 + None, # 327 + None, # 328 + None, # 329 + None, # 330 + None, # 331 + None, # 332 + None, # 333 + None, # 334 + None, # 335 + None, # 336 + None, # 337 + None, # 338 + None, # 339 + None, # 340 + None, # 341 + None, # 342 + None, # 343 + None, # 344 + None, # 345 + None, # 346 + None, # 347 + None, # 348 + None, # 349 + None, # 350 + None, # 351 + None, # 352 + None, # 353 + None, # 354 + None, # 355 + None, # 356 + None, # 357 + None, # 358 + None, # 359 + None, # 360 + None, # 361 + None, # 362 + None, # 363 + None, # 364 + None, # 365 + None, # 366 + None, # 367 + None, # 368 + None, # 369 + None, # 370 + None, # 371 + None, # 372 + None, # 373 + None, # 374 + None, # 375 + None, # 376 + None, # 377 + None, # 378 + None, # 379 + None, # 380 + None, # 381 + None, # 382 + None, # 383 + None, # 384 + None, # 385 + None, # 386 + None, # 387 + None, # 388 + None, # 389 + None, # 390 + None, # 391 + None, # 392 + None, # 393 + None, # 394 + None, # 395 + None, # 396 + None, # 397 + None, # 398 + None, # 399 + None, # 400 + None, # 401 + None, # 402 + None, # 403 + None, # 404 + None, # 405 + None, # 406 + None, # 407 + None, # 408 + None, # 409 + None, # 410 + None, # 411 + None, # 412 + None, # 413 + None, # 414 + None, # 415 + None, # 416 + None, # 417 + None, # 418 + None, # 419 + None, # 420 + None, # 421 + None, # 422 + None, # 423 + None, # 424 + None, # 425 + None, # 426 + None, # 427 + None, # 428 + None, # 429 + None, # 430 + None, # 431 + None, # 432 + None, # 433 + None, # 434 + None, # 435 + None, # 436 + None, # 437 + None, # 438 + None, # 439 + None, # 440 + None, # 441 + None, # 442 + None, # 443 + None, # 444 + None, # 445 + None, # 446 + None, # 447 + None, # 448 + None, # 449 + None, # 450 + None, # 451 + None, # 452 + None, # 453 + None, # 454 + None, # 455 None, # 456 None, # 457 None, # 458 @@ -12726,7 +15385,7 @@ def __ne__(self, other): None, # 1278 None, # 1279 None, # 1280 - None, # 1281 + (1281, TType.STRING, 'errorDetailsJson', 'UTF8', None, ), # 1281 None, # 1282 None, # 1283 None, # 1284 @@ -29458,6 +32117,7 @@ def __ne__(self, other): (2, TType.BOOL, 'decimalAsArrow', None, None, ), # 2 (3, TType.BOOL, 'complexTypesAsArrow', None, None, ), # 3 (4, TType.BOOL, 'intervalTypesAsArrow', None, None, ), # 4 + (5, TType.BOOL, 'nullTypeAsArrow', None, None, ), # 5 ) all_structs.append(TExecuteStatementReq) TExecuteStatementReq.thrift_spec = ( @@ -30749,15 +33409,15 @@ def __ne__(self, other): (1285, TType.I64, 'maxBytesPerFile', None, None, ), # 1285 (1286, TType.STRUCT, 'useArrowNativeTypes', [TSparkArrowTypes, None], None, ), # 1286 (1287, TType.I64, 'resultRowLimit', None, None, ), # 1287 - None, # 1288 - None, # 1289 + (1288, TType.LIST, 'parameters', (TType.STRUCT, [TSparkParameter, None], False), None, ), # 1288 + (1289, TType.I64, 'maxBytesPerBatch', None, None, ), # 1289 None, # 1290 None, # 1291 None, # 1292 None, # 1293 None, # 1294 None, # 1295 - None, # 1296 + (1296, TType.STRUCT, 'statementConf', [TStatementConf, None], None, ), # 1296 None, # 1297 None, # 1298 None, # 1299 @@ -32798,6 +35458,59 @@ def __ne__(self, other): (3334, TType.STRING, 'requestValidation', 'BINARY', None, ), # 3334 (3335, TType.I32, 'resultPersistenceMode', None, None, ), # 3335 (3336, TType.BOOL, 'trimArrowBatchesToLimit', None, None, ), # 3336 + (3337, TType.I32, 'fetchDisposition', None, None, ), # 3337 + None, # 3338 + None, # 3339 + None, # 3340 + None, # 3341 + None, # 3342 + None, # 3343 + (3344, TType.BOOL, 'enforceResultPersistenceMode', None, None, ), # 3344 + (3345, TType.LIST, 'statementList', (TType.STRUCT, [TDBSqlStatement, None], False), None, ), # 3345 + (3346, TType.BOOL, 'persistResultManifest', None, None, ), # 3346 + (3347, TType.I64, 'resultRetentionSeconds', None, None, ), # 3347 + (3348, TType.I64, 'resultByteLimit', None, None, ), # 3348 + (3349, TType.STRUCT, 'resultDataFormat', [TDBSqlResultFormat, None], None, ), # 3349 + (3350, TType.STRING, 'originatingClientIdentity', 'UTF8', None, ), # 3350 + (3351, TType.BOOL, 'preferSingleFileResult', None, None, ), # 3351 + (3352, TType.BOOL, 'preferDriverOnlyUpload', None, None, ), # 3352 + (3353, TType.BOOL, 'enforceEmbeddedSchemaCorrectness', None, False, ), # 3353 + None, # 3354 + None, # 3355 + None, # 3356 + None, # 3357 + None, # 3358 + None, # 3359 + (3360, TType.STRING, 'idempotencyToken', 'UTF8', None, ), # 3360 + (3361, TType.BOOL, 'throwErrorOnByteLimitTruncation', None, None, ), # 3361 +) +all_structs.append(TDBSqlStatement) +TDBSqlStatement.thrift_spec = ( + None, # 0 + (1, TType.STRING, 'statement', 'UTF8', None, ), # 1 +) +all_structs.append(TSparkParameterValue) +TSparkParameterValue.thrift_spec = ( + None, # 0 + (1, TType.STRING, 'stringValue', 'UTF8', None, ), # 1 + (2, TType.DOUBLE, 'doubleValue', None, None, ), # 2 + (3, TType.BOOL, 'booleanValue', None, None, ), # 3 +) +all_structs.append(TSparkParameter) +TSparkParameter.thrift_spec = ( + None, # 0 + (1, TType.I32, 'ordinal', None, None, ), # 1 + (2, TType.STRING, 'name', 'UTF8', None, ), # 2 + (3, TType.STRING, 'type', 'UTF8', None, ), # 3 + (4, TType.STRUCT, 'value', [TSparkParameterValue, None], None, ), # 4 +) +all_structs.append(TStatementConf) +TStatementConf.thrift_spec = ( + None, # 0 + (1, TType.BOOL, 'sessionless', None, None, ), # 1 + (2, TType.STRUCT, 'initialNamespace', [TNamespace, None], None, ), # 2 + (3, TType.I32, 'client_protocol', None, None, ), # 3 + (4, TType.I64, 'client_protocol_i64', None, None, ), # 4 ) all_structs.append(TExecuteStatementResp) TExecuteStatementResp.thrift_spec = ( @@ -36136,6 +38849,9 @@ def __ne__(self, other): (3332, TType.STRUCT, 'sessionConf', [TDBSqlSessionConf, None], None, ), # 3332 (3333, TType.DOUBLE, 'currentClusterLoad', None, None, ), # 3333 (3334, TType.I32, 'idempotencyType', None, None, ), # 3334 + (3335, TType.BOOL, 'remoteResultCacheEnabled', None, None, ), # 3335 + (3336, TType.BOOL, 'isServerless', None, None, ), # 3336 + (3337, TType.LIST, 'operationHandles', (TType.STRUCT, [TOperationHandle, None], False), None, ), # 3337 ) all_structs.append(TGetTypeInfoReq) TGetTypeInfoReq.thrift_spec = ( @@ -76423,20 +79139,1311 @@ def __ne__(self, other): (3329, TType.STRUCT, 'operationId', [THandleIdentifier, None], None, ), # 3329 (3330, TType.STRUCT, 'sessionConf', [TDBSqlSessionConf, None], None, ), # 3330 ) -all_structs.append(TGetCrossReferenceResp) -TGetCrossReferenceResp.thrift_spec = ( +all_structs.append(TGetCrossReferenceResp) +TGetCrossReferenceResp.thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'status', [TStatus, None], None, ), # 1 + (2, TType.STRUCT, 'operationHandle', [TOperationHandle, None], None, ), # 2 + None, # 3 + None, # 4 + None, # 5 + None, # 6 + None, # 7 + None, # 8 + None, # 9 + None, # 10 + None, # 11 + None, # 12 + None, # 13 + None, # 14 + None, # 15 + None, # 16 + None, # 17 + None, # 18 + None, # 19 + None, # 20 + None, # 21 + None, # 22 + None, # 23 + None, # 24 + None, # 25 + None, # 26 + None, # 27 + None, # 28 + None, # 29 + None, # 30 + None, # 31 + None, # 32 + None, # 33 + None, # 34 + None, # 35 + None, # 36 + None, # 37 + None, # 38 + None, # 39 + None, # 40 + None, # 41 + None, # 42 + None, # 43 + None, # 44 + None, # 45 + None, # 46 + None, # 47 + None, # 48 + None, # 49 + None, # 50 + None, # 51 + None, # 52 + None, # 53 + None, # 54 + None, # 55 + None, # 56 + None, # 57 + None, # 58 + None, # 59 + None, # 60 + None, # 61 + None, # 62 + None, # 63 + None, # 64 + None, # 65 + None, # 66 + None, # 67 + None, # 68 + None, # 69 + None, # 70 + None, # 71 + None, # 72 + None, # 73 + None, # 74 + None, # 75 + None, # 76 + None, # 77 + None, # 78 + None, # 79 + None, # 80 + None, # 81 + None, # 82 + None, # 83 + None, # 84 + None, # 85 + None, # 86 + None, # 87 + None, # 88 + None, # 89 + None, # 90 + None, # 91 + None, # 92 + None, # 93 + None, # 94 + None, # 95 + None, # 96 + None, # 97 + None, # 98 + None, # 99 + None, # 100 + None, # 101 + None, # 102 + None, # 103 + None, # 104 + None, # 105 + None, # 106 + None, # 107 + None, # 108 + None, # 109 + None, # 110 + None, # 111 + None, # 112 + None, # 113 + None, # 114 + None, # 115 + None, # 116 + None, # 117 + None, # 118 + None, # 119 + None, # 120 + None, # 121 + None, # 122 + None, # 123 + None, # 124 + None, # 125 + None, # 126 + None, # 127 + None, # 128 + None, # 129 + None, # 130 + None, # 131 + None, # 132 + None, # 133 + None, # 134 + None, # 135 + None, # 136 + None, # 137 + None, # 138 + None, # 139 + None, # 140 + None, # 141 + None, # 142 + None, # 143 + None, # 144 + None, # 145 + None, # 146 + None, # 147 + None, # 148 + None, # 149 + None, # 150 + None, # 151 + None, # 152 + None, # 153 + None, # 154 + None, # 155 + None, # 156 + None, # 157 + None, # 158 + None, # 159 + None, # 160 + None, # 161 + None, # 162 + None, # 163 + None, # 164 + None, # 165 + None, # 166 + None, # 167 + None, # 168 + None, # 169 + None, # 170 + None, # 171 + None, # 172 + None, # 173 + None, # 174 + None, # 175 + None, # 176 + None, # 177 + None, # 178 + None, # 179 + None, # 180 + None, # 181 + None, # 182 + None, # 183 + None, # 184 + None, # 185 + None, # 186 + None, # 187 + None, # 188 + None, # 189 + None, # 190 + None, # 191 + None, # 192 + None, # 193 + None, # 194 + None, # 195 + None, # 196 + None, # 197 + None, # 198 + None, # 199 + None, # 200 + None, # 201 + None, # 202 + None, # 203 + None, # 204 + None, # 205 + None, # 206 + None, # 207 + None, # 208 + None, # 209 + None, # 210 + None, # 211 + None, # 212 + None, # 213 + None, # 214 + None, # 215 + None, # 216 + None, # 217 + None, # 218 + None, # 219 + None, # 220 + None, # 221 + None, # 222 + None, # 223 + None, # 224 + None, # 225 + None, # 226 + None, # 227 + None, # 228 + None, # 229 + None, # 230 + None, # 231 + None, # 232 + None, # 233 + None, # 234 + None, # 235 + None, # 236 + None, # 237 + None, # 238 + None, # 239 + None, # 240 + None, # 241 + None, # 242 + None, # 243 + None, # 244 + None, # 245 + None, # 246 + None, # 247 + None, # 248 + None, # 249 + None, # 250 + None, # 251 + None, # 252 + None, # 253 + None, # 254 + None, # 255 + None, # 256 + None, # 257 + None, # 258 + None, # 259 + None, # 260 + None, # 261 + None, # 262 + None, # 263 + None, # 264 + None, # 265 + None, # 266 + None, # 267 + None, # 268 + None, # 269 + None, # 270 + None, # 271 + None, # 272 + None, # 273 + None, # 274 + None, # 275 + None, # 276 + None, # 277 + None, # 278 + None, # 279 + None, # 280 + None, # 281 + None, # 282 + None, # 283 + None, # 284 + None, # 285 + None, # 286 + None, # 287 + None, # 288 + None, # 289 + None, # 290 + None, # 291 + None, # 292 + None, # 293 + None, # 294 + None, # 295 + None, # 296 + None, # 297 + None, # 298 + None, # 299 + None, # 300 + None, # 301 + None, # 302 + None, # 303 + None, # 304 + None, # 305 + None, # 306 + None, # 307 + None, # 308 + None, # 309 + None, # 310 + None, # 311 + None, # 312 + None, # 313 + None, # 314 + None, # 315 + None, # 316 + None, # 317 + None, # 318 + None, # 319 + None, # 320 + None, # 321 + None, # 322 + None, # 323 + None, # 324 + None, # 325 + None, # 326 + None, # 327 + None, # 328 + None, # 329 + None, # 330 + None, # 331 + None, # 332 + None, # 333 + None, # 334 + None, # 335 + None, # 336 + None, # 337 + None, # 338 + None, # 339 + None, # 340 + None, # 341 + None, # 342 + None, # 343 + None, # 344 + None, # 345 + None, # 346 + None, # 347 + None, # 348 + None, # 349 + None, # 350 + None, # 351 + None, # 352 + None, # 353 + None, # 354 + None, # 355 + None, # 356 + None, # 357 + None, # 358 + None, # 359 + None, # 360 + None, # 361 + None, # 362 + None, # 363 + None, # 364 + None, # 365 + None, # 366 + None, # 367 + None, # 368 + None, # 369 + None, # 370 + None, # 371 + None, # 372 + None, # 373 + None, # 374 + None, # 375 + None, # 376 + None, # 377 + None, # 378 + None, # 379 + None, # 380 + None, # 381 + None, # 382 + None, # 383 + None, # 384 + None, # 385 + None, # 386 + None, # 387 + None, # 388 + None, # 389 + None, # 390 + None, # 391 + None, # 392 + None, # 393 + None, # 394 + None, # 395 + None, # 396 + None, # 397 + None, # 398 + None, # 399 + None, # 400 + None, # 401 + None, # 402 + None, # 403 + None, # 404 + None, # 405 + None, # 406 + None, # 407 + None, # 408 + None, # 409 + None, # 410 + None, # 411 + None, # 412 + None, # 413 + None, # 414 + None, # 415 + None, # 416 + None, # 417 + None, # 418 + None, # 419 + None, # 420 + None, # 421 + None, # 422 + None, # 423 + None, # 424 + None, # 425 + None, # 426 + None, # 427 + None, # 428 + None, # 429 + None, # 430 + None, # 431 + None, # 432 + None, # 433 + None, # 434 + None, # 435 + None, # 436 + None, # 437 + None, # 438 + None, # 439 + None, # 440 + None, # 441 + None, # 442 + None, # 443 + None, # 444 + None, # 445 + None, # 446 + None, # 447 + None, # 448 + None, # 449 + None, # 450 + None, # 451 + None, # 452 + None, # 453 + None, # 454 + None, # 455 + None, # 456 + None, # 457 + None, # 458 + None, # 459 + None, # 460 + None, # 461 + None, # 462 + None, # 463 + None, # 464 + None, # 465 + None, # 466 + None, # 467 + None, # 468 + None, # 469 + None, # 470 + None, # 471 + None, # 472 + None, # 473 + None, # 474 + None, # 475 + None, # 476 + None, # 477 + None, # 478 + None, # 479 + None, # 480 + None, # 481 + None, # 482 + None, # 483 + None, # 484 + None, # 485 + None, # 486 + None, # 487 + None, # 488 + None, # 489 + None, # 490 + None, # 491 + None, # 492 + None, # 493 + None, # 494 + None, # 495 + None, # 496 + None, # 497 + None, # 498 + None, # 499 + None, # 500 + None, # 501 + None, # 502 + None, # 503 + None, # 504 + None, # 505 + None, # 506 + None, # 507 + None, # 508 + None, # 509 + None, # 510 + None, # 511 + None, # 512 + None, # 513 + None, # 514 + None, # 515 + None, # 516 + None, # 517 + None, # 518 + None, # 519 + None, # 520 + None, # 521 + None, # 522 + None, # 523 + None, # 524 + None, # 525 + None, # 526 + None, # 527 + None, # 528 + None, # 529 + None, # 530 + None, # 531 + None, # 532 + None, # 533 + None, # 534 + None, # 535 + None, # 536 + None, # 537 + None, # 538 + None, # 539 + None, # 540 + None, # 541 + None, # 542 + None, # 543 + None, # 544 + None, # 545 + None, # 546 + None, # 547 + None, # 548 + None, # 549 + None, # 550 + None, # 551 + None, # 552 + None, # 553 + None, # 554 + None, # 555 + None, # 556 + None, # 557 + None, # 558 + None, # 559 + None, # 560 + None, # 561 + None, # 562 + None, # 563 + None, # 564 + None, # 565 + None, # 566 + None, # 567 + None, # 568 + None, # 569 + None, # 570 + None, # 571 + None, # 572 + None, # 573 + None, # 574 + None, # 575 + None, # 576 + None, # 577 + None, # 578 + None, # 579 + None, # 580 + None, # 581 + None, # 582 + None, # 583 + None, # 584 + None, # 585 + None, # 586 + None, # 587 + None, # 588 + None, # 589 + None, # 590 + None, # 591 + None, # 592 + None, # 593 + None, # 594 + None, # 595 + None, # 596 + None, # 597 + None, # 598 + None, # 599 + None, # 600 + None, # 601 + None, # 602 + None, # 603 + None, # 604 + None, # 605 + None, # 606 + None, # 607 + None, # 608 + None, # 609 + None, # 610 + None, # 611 + None, # 612 + None, # 613 + None, # 614 + None, # 615 + None, # 616 + None, # 617 + None, # 618 + None, # 619 + None, # 620 + None, # 621 + None, # 622 + None, # 623 + None, # 624 + None, # 625 + None, # 626 + None, # 627 + None, # 628 + None, # 629 + None, # 630 + None, # 631 + None, # 632 + None, # 633 + None, # 634 + None, # 635 + None, # 636 + None, # 637 + None, # 638 + None, # 639 + None, # 640 + None, # 641 + None, # 642 + None, # 643 + None, # 644 + None, # 645 + None, # 646 + None, # 647 + None, # 648 + None, # 649 + None, # 650 + None, # 651 + None, # 652 + None, # 653 + None, # 654 + None, # 655 + None, # 656 + None, # 657 + None, # 658 + None, # 659 + None, # 660 + None, # 661 + None, # 662 + None, # 663 + None, # 664 + None, # 665 + None, # 666 + None, # 667 + None, # 668 + None, # 669 + None, # 670 + None, # 671 + None, # 672 + None, # 673 + None, # 674 + None, # 675 + None, # 676 + None, # 677 + None, # 678 + None, # 679 + None, # 680 + None, # 681 + None, # 682 + None, # 683 + None, # 684 + None, # 685 + None, # 686 + None, # 687 + None, # 688 + None, # 689 + None, # 690 + None, # 691 + None, # 692 + None, # 693 + None, # 694 + None, # 695 + None, # 696 + None, # 697 + None, # 698 + None, # 699 + None, # 700 + None, # 701 + None, # 702 + None, # 703 + None, # 704 + None, # 705 + None, # 706 + None, # 707 + None, # 708 + None, # 709 + None, # 710 + None, # 711 + None, # 712 + None, # 713 + None, # 714 + None, # 715 + None, # 716 + None, # 717 + None, # 718 + None, # 719 + None, # 720 + None, # 721 + None, # 722 + None, # 723 + None, # 724 + None, # 725 + None, # 726 + None, # 727 + None, # 728 + None, # 729 + None, # 730 + None, # 731 + None, # 732 + None, # 733 + None, # 734 + None, # 735 + None, # 736 + None, # 737 + None, # 738 + None, # 739 + None, # 740 + None, # 741 + None, # 742 + None, # 743 + None, # 744 + None, # 745 + None, # 746 + None, # 747 + None, # 748 + None, # 749 + None, # 750 + None, # 751 + None, # 752 + None, # 753 + None, # 754 + None, # 755 + None, # 756 + None, # 757 + None, # 758 + None, # 759 + None, # 760 + None, # 761 + None, # 762 + None, # 763 + None, # 764 + None, # 765 + None, # 766 + None, # 767 + None, # 768 + None, # 769 + None, # 770 + None, # 771 + None, # 772 + None, # 773 + None, # 774 + None, # 775 + None, # 776 + None, # 777 + None, # 778 + None, # 779 + None, # 780 + None, # 781 + None, # 782 + None, # 783 + None, # 784 + None, # 785 + None, # 786 + None, # 787 + None, # 788 + None, # 789 + None, # 790 + None, # 791 + None, # 792 + None, # 793 + None, # 794 + None, # 795 + None, # 796 + None, # 797 + None, # 798 + None, # 799 + None, # 800 + None, # 801 + None, # 802 + None, # 803 + None, # 804 + None, # 805 + None, # 806 + None, # 807 + None, # 808 + None, # 809 + None, # 810 + None, # 811 + None, # 812 + None, # 813 + None, # 814 + None, # 815 + None, # 816 + None, # 817 + None, # 818 + None, # 819 + None, # 820 + None, # 821 + None, # 822 + None, # 823 + None, # 824 + None, # 825 + None, # 826 + None, # 827 + None, # 828 + None, # 829 + None, # 830 + None, # 831 + None, # 832 + None, # 833 + None, # 834 + None, # 835 + None, # 836 + None, # 837 + None, # 838 + None, # 839 + None, # 840 + None, # 841 + None, # 842 + None, # 843 + None, # 844 + None, # 845 + None, # 846 + None, # 847 + None, # 848 + None, # 849 + None, # 850 + None, # 851 + None, # 852 + None, # 853 + None, # 854 + None, # 855 + None, # 856 + None, # 857 + None, # 858 + None, # 859 + None, # 860 + None, # 861 + None, # 862 + None, # 863 + None, # 864 + None, # 865 + None, # 866 + None, # 867 + None, # 868 + None, # 869 + None, # 870 + None, # 871 + None, # 872 + None, # 873 + None, # 874 + None, # 875 + None, # 876 + None, # 877 + None, # 878 + None, # 879 + None, # 880 + None, # 881 + None, # 882 + None, # 883 + None, # 884 + None, # 885 + None, # 886 + None, # 887 + None, # 888 + None, # 889 + None, # 890 + None, # 891 + None, # 892 + None, # 893 + None, # 894 + None, # 895 + None, # 896 + None, # 897 + None, # 898 + None, # 899 + None, # 900 + None, # 901 + None, # 902 + None, # 903 + None, # 904 + None, # 905 + None, # 906 + None, # 907 + None, # 908 + None, # 909 + None, # 910 + None, # 911 + None, # 912 + None, # 913 + None, # 914 + None, # 915 + None, # 916 + None, # 917 + None, # 918 + None, # 919 + None, # 920 + None, # 921 + None, # 922 + None, # 923 + None, # 924 + None, # 925 + None, # 926 + None, # 927 + None, # 928 + None, # 929 + None, # 930 + None, # 931 + None, # 932 + None, # 933 + None, # 934 + None, # 935 + None, # 936 + None, # 937 + None, # 938 + None, # 939 + None, # 940 + None, # 941 + None, # 942 + None, # 943 + None, # 944 + None, # 945 + None, # 946 + None, # 947 + None, # 948 + None, # 949 + None, # 950 + None, # 951 + None, # 952 + None, # 953 + None, # 954 + None, # 955 + None, # 956 + None, # 957 + None, # 958 + None, # 959 + None, # 960 + None, # 961 + None, # 962 + None, # 963 + None, # 964 + None, # 965 + None, # 966 + None, # 967 + None, # 968 + None, # 969 + None, # 970 + None, # 971 + None, # 972 + None, # 973 + None, # 974 + None, # 975 + None, # 976 + None, # 977 + None, # 978 + None, # 979 + None, # 980 + None, # 981 + None, # 982 + None, # 983 + None, # 984 + None, # 985 + None, # 986 + None, # 987 + None, # 988 + None, # 989 + None, # 990 + None, # 991 + None, # 992 + None, # 993 + None, # 994 + None, # 995 + None, # 996 + None, # 997 + None, # 998 + None, # 999 + None, # 1000 + None, # 1001 + None, # 1002 + None, # 1003 + None, # 1004 + None, # 1005 + None, # 1006 + None, # 1007 + None, # 1008 + None, # 1009 + None, # 1010 + None, # 1011 + None, # 1012 + None, # 1013 + None, # 1014 + None, # 1015 + None, # 1016 + None, # 1017 + None, # 1018 + None, # 1019 + None, # 1020 + None, # 1021 + None, # 1022 + None, # 1023 + None, # 1024 + None, # 1025 + None, # 1026 + None, # 1027 + None, # 1028 + None, # 1029 + None, # 1030 + None, # 1031 + None, # 1032 + None, # 1033 + None, # 1034 + None, # 1035 + None, # 1036 + None, # 1037 + None, # 1038 + None, # 1039 + None, # 1040 + None, # 1041 + None, # 1042 + None, # 1043 + None, # 1044 + None, # 1045 + None, # 1046 + None, # 1047 + None, # 1048 + None, # 1049 + None, # 1050 + None, # 1051 + None, # 1052 + None, # 1053 + None, # 1054 + None, # 1055 + None, # 1056 + None, # 1057 + None, # 1058 + None, # 1059 + None, # 1060 + None, # 1061 + None, # 1062 + None, # 1063 + None, # 1064 + None, # 1065 + None, # 1066 + None, # 1067 + None, # 1068 + None, # 1069 + None, # 1070 + None, # 1071 + None, # 1072 + None, # 1073 + None, # 1074 + None, # 1075 + None, # 1076 + None, # 1077 + None, # 1078 + None, # 1079 + None, # 1080 + None, # 1081 + None, # 1082 + None, # 1083 + None, # 1084 + None, # 1085 + None, # 1086 + None, # 1087 + None, # 1088 + None, # 1089 + None, # 1090 + None, # 1091 + None, # 1092 + None, # 1093 + None, # 1094 + None, # 1095 + None, # 1096 + None, # 1097 + None, # 1098 + None, # 1099 + None, # 1100 + None, # 1101 + None, # 1102 + None, # 1103 + None, # 1104 + None, # 1105 + None, # 1106 + None, # 1107 + None, # 1108 + None, # 1109 + None, # 1110 + None, # 1111 + None, # 1112 + None, # 1113 + None, # 1114 + None, # 1115 + None, # 1116 + None, # 1117 + None, # 1118 + None, # 1119 + None, # 1120 + None, # 1121 + None, # 1122 + None, # 1123 + None, # 1124 + None, # 1125 + None, # 1126 + None, # 1127 + None, # 1128 + None, # 1129 + None, # 1130 + None, # 1131 + None, # 1132 + None, # 1133 + None, # 1134 + None, # 1135 + None, # 1136 + None, # 1137 + None, # 1138 + None, # 1139 + None, # 1140 + None, # 1141 + None, # 1142 + None, # 1143 + None, # 1144 + None, # 1145 + None, # 1146 + None, # 1147 + None, # 1148 + None, # 1149 + None, # 1150 + None, # 1151 + None, # 1152 + None, # 1153 + None, # 1154 + None, # 1155 + None, # 1156 + None, # 1157 + None, # 1158 + None, # 1159 + None, # 1160 + None, # 1161 + None, # 1162 + None, # 1163 + None, # 1164 + None, # 1165 + None, # 1166 + None, # 1167 + None, # 1168 + None, # 1169 + None, # 1170 + None, # 1171 + None, # 1172 + None, # 1173 + None, # 1174 + None, # 1175 + None, # 1176 + None, # 1177 + None, # 1178 + None, # 1179 + None, # 1180 + None, # 1181 + None, # 1182 + None, # 1183 + None, # 1184 + None, # 1185 + None, # 1186 + None, # 1187 + None, # 1188 + None, # 1189 + None, # 1190 + None, # 1191 + None, # 1192 + None, # 1193 + None, # 1194 + None, # 1195 + None, # 1196 + None, # 1197 + None, # 1198 + None, # 1199 + None, # 1200 + None, # 1201 + None, # 1202 + None, # 1203 + None, # 1204 + None, # 1205 + None, # 1206 + None, # 1207 + None, # 1208 + None, # 1209 + None, # 1210 + None, # 1211 + None, # 1212 + None, # 1213 + None, # 1214 + None, # 1215 + None, # 1216 + None, # 1217 + None, # 1218 + None, # 1219 + None, # 1220 + None, # 1221 + None, # 1222 + None, # 1223 + None, # 1224 + None, # 1225 + None, # 1226 + None, # 1227 + None, # 1228 + None, # 1229 + None, # 1230 + None, # 1231 + None, # 1232 + None, # 1233 + None, # 1234 + None, # 1235 + None, # 1236 + None, # 1237 + None, # 1238 + None, # 1239 + None, # 1240 + None, # 1241 + None, # 1242 + None, # 1243 + None, # 1244 + None, # 1245 + None, # 1246 + None, # 1247 + None, # 1248 + None, # 1249 + None, # 1250 + None, # 1251 + None, # 1252 + None, # 1253 + None, # 1254 + None, # 1255 + None, # 1256 + None, # 1257 + None, # 1258 + None, # 1259 + None, # 1260 + None, # 1261 + None, # 1262 + None, # 1263 + None, # 1264 + None, # 1265 + None, # 1266 + None, # 1267 + None, # 1268 + None, # 1269 + None, # 1270 + None, # 1271 + None, # 1272 + None, # 1273 + None, # 1274 + None, # 1275 + None, # 1276 + None, # 1277 + None, # 1278 + None, # 1279 + None, # 1280 + (1281, TType.STRUCT, 'directResults', [TSparkDirectResults, None], None, ), # 1281 +) +all_structs.append(TGetOperationStatusReq) +TGetOperationStatusReq.thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'operationHandle', [TOperationHandle, None], None, ), # 1 + (2, TType.BOOL, 'getProgressUpdate', None, None, ), # 2 +) +all_structs.append(TGetOperationStatusResp) +TGetOperationStatusResp.thrift_spec = ( None, # 0 (1, TType.STRUCT, 'status', [TStatus, None], None, ), # 1 - (2, TType.STRUCT, 'operationHandle', [TOperationHandle, None], None, ), # 2 - None, # 3 - None, # 4 - None, # 5 - None, # 6 - None, # 7 - None, # 8 - None, # 9 - None, # 10 - None, # 11 + (2, TType.I32, 'operationState', None, None, ), # 2 + (3, TType.STRING, 'sqlState', 'UTF8', None, ), # 3 + (4, TType.I32, 'errorCode', None, None, ), # 4 + (5, TType.STRING, 'errorMessage', 'UTF8', None, ), # 5 + (6, TType.STRING, 'taskStatus', 'UTF8', None, ), # 6 + (7, TType.I64, 'operationStarted', None, None, ), # 7 + (8, TType.I64, 'operationCompleted', None, None, ), # 8 + (9, TType.BOOL, 'hasResultSet', None, None, ), # 9 + (10, TType.STRUCT, 'progressUpdateResponse', [TProgressUpdateResp, None], None, ), # 10 + (11, TType.I64, 'numModifiedRows', None, None, ), # 11 None, # 12 None, # 13 None, # 14 @@ -77706,28 +81713,2073 @@ def __ne__(self, other): None, # 1278 None, # 1279 None, # 1280 - (1281, TType.STRUCT, 'directResults', [TSparkDirectResults, None], None, ), # 1281 + (1281, TType.STRING, 'displayMessage', 'UTF8', None, ), # 1281 + (1282, TType.STRING, 'diagnosticInfo', 'UTF8', None, ), # 1282 + (1283, TType.STRING, 'errorDetailsJson', 'UTF8', None, ), # 1283 + None, # 1284 + None, # 1285 + None, # 1286 + None, # 1287 + None, # 1288 + None, # 1289 + None, # 1290 + None, # 1291 + None, # 1292 + None, # 1293 + None, # 1294 + None, # 1295 + None, # 1296 + None, # 1297 + None, # 1298 + None, # 1299 + None, # 1300 + None, # 1301 + None, # 1302 + None, # 1303 + None, # 1304 + None, # 1305 + None, # 1306 + None, # 1307 + None, # 1308 + None, # 1309 + None, # 1310 + None, # 1311 + None, # 1312 + None, # 1313 + None, # 1314 + None, # 1315 + None, # 1316 + None, # 1317 + None, # 1318 + None, # 1319 + None, # 1320 + None, # 1321 + None, # 1322 + None, # 1323 + None, # 1324 + None, # 1325 + None, # 1326 + None, # 1327 + None, # 1328 + None, # 1329 + None, # 1330 + None, # 1331 + None, # 1332 + None, # 1333 + None, # 1334 + None, # 1335 + None, # 1336 + None, # 1337 + None, # 1338 + None, # 1339 + None, # 1340 + None, # 1341 + None, # 1342 + None, # 1343 + None, # 1344 + None, # 1345 + None, # 1346 + None, # 1347 + None, # 1348 + None, # 1349 + None, # 1350 + None, # 1351 + None, # 1352 + None, # 1353 + None, # 1354 + None, # 1355 + None, # 1356 + None, # 1357 + None, # 1358 + None, # 1359 + None, # 1360 + None, # 1361 + None, # 1362 + None, # 1363 + None, # 1364 + None, # 1365 + None, # 1366 + None, # 1367 + None, # 1368 + None, # 1369 + None, # 1370 + None, # 1371 + None, # 1372 + None, # 1373 + None, # 1374 + None, # 1375 + None, # 1376 + None, # 1377 + None, # 1378 + None, # 1379 + None, # 1380 + None, # 1381 + None, # 1382 + None, # 1383 + None, # 1384 + None, # 1385 + None, # 1386 + None, # 1387 + None, # 1388 + None, # 1389 + None, # 1390 + None, # 1391 + None, # 1392 + None, # 1393 + None, # 1394 + None, # 1395 + None, # 1396 + None, # 1397 + None, # 1398 + None, # 1399 + None, # 1400 + None, # 1401 + None, # 1402 + None, # 1403 + None, # 1404 + None, # 1405 + None, # 1406 + None, # 1407 + None, # 1408 + None, # 1409 + None, # 1410 + None, # 1411 + None, # 1412 + None, # 1413 + None, # 1414 + None, # 1415 + None, # 1416 + None, # 1417 + None, # 1418 + None, # 1419 + None, # 1420 + None, # 1421 + None, # 1422 + None, # 1423 + None, # 1424 + None, # 1425 + None, # 1426 + None, # 1427 + None, # 1428 + None, # 1429 + None, # 1430 + None, # 1431 + None, # 1432 + None, # 1433 + None, # 1434 + None, # 1435 + None, # 1436 + None, # 1437 + None, # 1438 + None, # 1439 + None, # 1440 + None, # 1441 + None, # 1442 + None, # 1443 + None, # 1444 + None, # 1445 + None, # 1446 + None, # 1447 + None, # 1448 + None, # 1449 + None, # 1450 + None, # 1451 + None, # 1452 + None, # 1453 + None, # 1454 + None, # 1455 + None, # 1456 + None, # 1457 + None, # 1458 + None, # 1459 + None, # 1460 + None, # 1461 + None, # 1462 + None, # 1463 + None, # 1464 + None, # 1465 + None, # 1466 + None, # 1467 + None, # 1468 + None, # 1469 + None, # 1470 + None, # 1471 + None, # 1472 + None, # 1473 + None, # 1474 + None, # 1475 + None, # 1476 + None, # 1477 + None, # 1478 + None, # 1479 + None, # 1480 + None, # 1481 + None, # 1482 + None, # 1483 + None, # 1484 + None, # 1485 + None, # 1486 + None, # 1487 + None, # 1488 + None, # 1489 + None, # 1490 + None, # 1491 + None, # 1492 + None, # 1493 + None, # 1494 + None, # 1495 + None, # 1496 + None, # 1497 + None, # 1498 + None, # 1499 + None, # 1500 + None, # 1501 + None, # 1502 + None, # 1503 + None, # 1504 + None, # 1505 + None, # 1506 + None, # 1507 + None, # 1508 + None, # 1509 + None, # 1510 + None, # 1511 + None, # 1512 + None, # 1513 + None, # 1514 + None, # 1515 + None, # 1516 + None, # 1517 + None, # 1518 + None, # 1519 + None, # 1520 + None, # 1521 + None, # 1522 + None, # 1523 + None, # 1524 + None, # 1525 + None, # 1526 + None, # 1527 + None, # 1528 + None, # 1529 + None, # 1530 + None, # 1531 + None, # 1532 + None, # 1533 + None, # 1534 + None, # 1535 + None, # 1536 + None, # 1537 + None, # 1538 + None, # 1539 + None, # 1540 + None, # 1541 + None, # 1542 + None, # 1543 + None, # 1544 + None, # 1545 + None, # 1546 + None, # 1547 + None, # 1548 + None, # 1549 + None, # 1550 + None, # 1551 + None, # 1552 + None, # 1553 + None, # 1554 + None, # 1555 + None, # 1556 + None, # 1557 + None, # 1558 + None, # 1559 + None, # 1560 + None, # 1561 + None, # 1562 + None, # 1563 + None, # 1564 + None, # 1565 + None, # 1566 + None, # 1567 + None, # 1568 + None, # 1569 + None, # 1570 + None, # 1571 + None, # 1572 + None, # 1573 + None, # 1574 + None, # 1575 + None, # 1576 + None, # 1577 + None, # 1578 + None, # 1579 + None, # 1580 + None, # 1581 + None, # 1582 + None, # 1583 + None, # 1584 + None, # 1585 + None, # 1586 + None, # 1587 + None, # 1588 + None, # 1589 + None, # 1590 + None, # 1591 + None, # 1592 + None, # 1593 + None, # 1594 + None, # 1595 + None, # 1596 + None, # 1597 + None, # 1598 + None, # 1599 + None, # 1600 + None, # 1601 + None, # 1602 + None, # 1603 + None, # 1604 + None, # 1605 + None, # 1606 + None, # 1607 + None, # 1608 + None, # 1609 + None, # 1610 + None, # 1611 + None, # 1612 + None, # 1613 + None, # 1614 + None, # 1615 + None, # 1616 + None, # 1617 + None, # 1618 + None, # 1619 + None, # 1620 + None, # 1621 + None, # 1622 + None, # 1623 + None, # 1624 + None, # 1625 + None, # 1626 + None, # 1627 + None, # 1628 + None, # 1629 + None, # 1630 + None, # 1631 + None, # 1632 + None, # 1633 + None, # 1634 + None, # 1635 + None, # 1636 + None, # 1637 + None, # 1638 + None, # 1639 + None, # 1640 + None, # 1641 + None, # 1642 + None, # 1643 + None, # 1644 + None, # 1645 + None, # 1646 + None, # 1647 + None, # 1648 + None, # 1649 + None, # 1650 + None, # 1651 + None, # 1652 + None, # 1653 + None, # 1654 + None, # 1655 + None, # 1656 + None, # 1657 + None, # 1658 + None, # 1659 + None, # 1660 + None, # 1661 + None, # 1662 + None, # 1663 + None, # 1664 + None, # 1665 + None, # 1666 + None, # 1667 + None, # 1668 + None, # 1669 + None, # 1670 + None, # 1671 + None, # 1672 + None, # 1673 + None, # 1674 + None, # 1675 + None, # 1676 + None, # 1677 + None, # 1678 + None, # 1679 + None, # 1680 + None, # 1681 + None, # 1682 + None, # 1683 + None, # 1684 + None, # 1685 + None, # 1686 + None, # 1687 + None, # 1688 + None, # 1689 + None, # 1690 + None, # 1691 + None, # 1692 + None, # 1693 + None, # 1694 + None, # 1695 + None, # 1696 + None, # 1697 + None, # 1698 + None, # 1699 + None, # 1700 + None, # 1701 + None, # 1702 + None, # 1703 + None, # 1704 + None, # 1705 + None, # 1706 + None, # 1707 + None, # 1708 + None, # 1709 + None, # 1710 + None, # 1711 + None, # 1712 + None, # 1713 + None, # 1714 + None, # 1715 + None, # 1716 + None, # 1717 + None, # 1718 + None, # 1719 + None, # 1720 + None, # 1721 + None, # 1722 + None, # 1723 + None, # 1724 + None, # 1725 + None, # 1726 + None, # 1727 + None, # 1728 + None, # 1729 + None, # 1730 + None, # 1731 + None, # 1732 + None, # 1733 + None, # 1734 + None, # 1735 + None, # 1736 + None, # 1737 + None, # 1738 + None, # 1739 + None, # 1740 + None, # 1741 + None, # 1742 + None, # 1743 + None, # 1744 + None, # 1745 + None, # 1746 + None, # 1747 + None, # 1748 + None, # 1749 + None, # 1750 + None, # 1751 + None, # 1752 + None, # 1753 + None, # 1754 + None, # 1755 + None, # 1756 + None, # 1757 + None, # 1758 + None, # 1759 + None, # 1760 + None, # 1761 + None, # 1762 + None, # 1763 + None, # 1764 + None, # 1765 + None, # 1766 + None, # 1767 + None, # 1768 + None, # 1769 + None, # 1770 + None, # 1771 + None, # 1772 + None, # 1773 + None, # 1774 + None, # 1775 + None, # 1776 + None, # 1777 + None, # 1778 + None, # 1779 + None, # 1780 + None, # 1781 + None, # 1782 + None, # 1783 + None, # 1784 + None, # 1785 + None, # 1786 + None, # 1787 + None, # 1788 + None, # 1789 + None, # 1790 + None, # 1791 + None, # 1792 + None, # 1793 + None, # 1794 + None, # 1795 + None, # 1796 + None, # 1797 + None, # 1798 + None, # 1799 + None, # 1800 + None, # 1801 + None, # 1802 + None, # 1803 + None, # 1804 + None, # 1805 + None, # 1806 + None, # 1807 + None, # 1808 + None, # 1809 + None, # 1810 + None, # 1811 + None, # 1812 + None, # 1813 + None, # 1814 + None, # 1815 + None, # 1816 + None, # 1817 + None, # 1818 + None, # 1819 + None, # 1820 + None, # 1821 + None, # 1822 + None, # 1823 + None, # 1824 + None, # 1825 + None, # 1826 + None, # 1827 + None, # 1828 + None, # 1829 + None, # 1830 + None, # 1831 + None, # 1832 + None, # 1833 + None, # 1834 + None, # 1835 + None, # 1836 + None, # 1837 + None, # 1838 + None, # 1839 + None, # 1840 + None, # 1841 + None, # 1842 + None, # 1843 + None, # 1844 + None, # 1845 + None, # 1846 + None, # 1847 + None, # 1848 + None, # 1849 + None, # 1850 + None, # 1851 + None, # 1852 + None, # 1853 + None, # 1854 + None, # 1855 + None, # 1856 + None, # 1857 + None, # 1858 + None, # 1859 + None, # 1860 + None, # 1861 + None, # 1862 + None, # 1863 + None, # 1864 + None, # 1865 + None, # 1866 + None, # 1867 + None, # 1868 + None, # 1869 + None, # 1870 + None, # 1871 + None, # 1872 + None, # 1873 + None, # 1874 + None, # 1875 + None, # 1876 + None, # 1877 + None, # 1878 + None, # 1879 + None, # 1880 + None, # 1881 + None, # 1882 + None, # 1883 + None, # 1884 + None, # 1885 + None, # 1886 + None, # 1887 + None, # 1888 + None, # 1889 + None, # 1890 + None, # 1891 + None, # 1892 + None, # 1893 + None, # 1894 + None, # 1895 + None, # 1896 + None, # 1897 + None, # 1898 + None, # 1899 + None, # 1900 + None, # 1901 + None, # 1902 + None, # 1903 + None, # 1904 + None, # 1905 + None, # 1906 + None, # 1907 + None, # 1908 + None, # 1909 + None, # 1910 + None, # 1911 + None, # 1912 + None, # 1913 + None, # 1914 + None, # 1915 + None, # 1916 + None, # 1917 + None, # 1918 + None, # 1919 + None, # 1920 + None, # 1921 + None, # 1922 + None, # 1923 + None, # 1924 + None, # 1925 + None, # 1926 + None, # 1927 + None, # 1928 + None, # 1929 + None, # 1930 + None, # 1931 + None, # 1932 + None, # 1933 + None, # 1934 + None, # 1935 + None, # 1936 + None, # 1937 + None, # 1938 + None, # 1939 + None, # 1940 + None, # 1941 + None, # 1942 + None, # 1943 + None, # 1944 + None, # 1945 + None, # 1946 + None, # 1947 + None, # 1948 + None, # 1949 + None, # 1950 + None, # 1951 + None, # 1952 + None, # 1953 + None, # 1954 + None, # 1955 + None, # 1956 + None, # 1957 + None, # 1958 + None, # 1959 + None, # 1960 + None, # 1961 + None, # 1962 + None, # 1963 + None, # 1964 + None, # 1965 + None, # 1966 + None, # 1967 + None, # 1968 + None, # 1969 + None, # 1970 + None, # 1971 + None, # 1972 + None, # 1973 + None, # 1974 + None, # 1975 + None, # 1976 + None, # 1977 + None, # 1978 + None, # 1979 + None, # 1980 + None, # 1981 + None, # 1982 + None, # 1983 + None, # 1984 + None, # 1985 + None, # 1986 + None, # 1987 + None, # 1988 + None, # 1989 + None, # 1990 + None, # 1991 + None, # 1992 + None, # 1993 + None, # 1994 + None, # 1995 + None, # 1996 + None, # 1997 + None, # 1998 + None, # 1999 + None, # 2000 + None, # 2001 + None, # 2002 + None, # 2003 + None, # 2004 + None, # 2005 + None, # 2006 + None, # 2007 + None, # 2008 + None, # 2009 + None, # 2010 + None, # 2011 + None, # 2012 + None, # 2013 + None, # 2014 + None, # 2015 + None, # 2016 + None, # 2017 + None, # 2018 + None, # 2019 + None, # 2020 + None, # 2021 + None, # 2022 + None, # 2023 + None, # 2024 + None, # 2025 + None, # 2026 + None, # 2027 + None, # 2028 + None, # 2029 + None, # 2030 + None, # 2031 + None, # 2032 + None, # 2033 + None, # 2034 + None, # 2035 + None, # 2036 + None, # 2037 + None, # 2038 + None, # 2039 + None, # 2040 + None, # 2041 + None, # 2042 + None, # 2043 + None, # 2044 + None, # 2045 + None, # 2046 + None, # 2047 + None, # 2048 + None, # 2049 + None, # 2050 + None, # 2051 + None, # 2052 + None, # 2053 + None, # 2054 + None, # 2055 + None, # 2056 + None, # 2057 + None, # 2058 + None, # 2059 + None, # 2060 + None, # 2061 + None, # 2062 + None, # 2063 + None, # 2064 + None, # 2065 + None, # 2066 + None, # 2067 + None, # 2068 + None, # 2069 + None, # 2070 + None, # 2071 + None, # 2072 + None, # 2073 + None, # 2074 + None, # 2075 + None, # 2076 + None, # 2077 + None, # 2078 + None, # 2079 + None, # 2080 + None, # 2081 + None, # 2082 + None, # 2083 + None, # 2084 + None, # 2085 + None, # 2086 + None, # 2087 + None, # 2088 + None, # 2089 + None, # 2090 + None, # 2091 + None, # 2092 + None, # 2093 + None, # 2094 + None, # 2095 + None, # 2096 + None, # 2097 + None, # 2098 + None, # 2099 + None, # 2100 + None, # 2101 + None, # 2102 + None, # 2103 + None, # 2104 + None, # 2105 + None, # 2106 + None, # 2107 + None, # 2108 + None, # 2109 + None, # 2110 + None, # 2111 + None, # 2112 + None, # 2113 + None, # 2114 + None, # 2115 + None, # 2116 + None, # 2117 + None, # 2118 + None, # 2119 + None, # 2120 + None, # 2121 + None, # 2122 + None, # 2123 + None, # 2124 + None, # 2125 + None, # 2126 + None, # 2127 + None, # 2128 + None, # 2129 + None, # 2130 + None, # 2131 + None, # 2132 + None, # 2133 + None, # 2134 + None, # 2135 + None, # 2136 + None, # 2137 + None, # 2138 + None, # 2139 + None, # 2140 + None, # 2141 + None, # 2142 + None, # 2143 + None, # 2144 + None, # 2145 + None, # 2146 + None, # 2147 + None, # 2148 + None, # 2149 + None, # 2150 + None, # 2151 + None, # 2152 + None, # 2153 + None, # 2154 + None, # 2155 + None, # 2156 + None, # 2157 + None, # 2158 + None, # 2159 + None, # 2160 + None, # 2161 + None, # 2162 + None, # 2163 + None, # 2164 + None, # 2165 + None, # 2166 + None, # 2167 + None, # 2168 + None, # 2169 + None, # 2170 + None, # 2171 + None, # 2172 + None, # 2173 + None, # 2174 + None, # 2175 + None, # 2176 + None, # 2177 + None, # 2178 + None, # 2179 + None, # 2180 + None, # 2181 + None, # 2182 + None, # 2183 + None, # 2184 + None, # 2185 + None, # 2186 + None, # 2187 + None, # 2188 + None, # 2189 + None, # 2190 + None, # 2191 + None, # 2192 + None, # 2193 + None, # 2194 + None, # 2195 + None, # 2196 + None, # 2197 + None, # 2198 + None, # 2199 + None, # 2200 + None, # 2201 + None, # 2202 + None, # 2203 + None, # 2204 + None, # 2205 + None, # 2206 + None, # 2207 + None, # 2208 + None, # 2209 + None, # 2210 + None, # 2211 + None, # 2212 + None, # 2213 + None, # 2214 + None, # 2215 + None, # 2216 + None, # 2217 + None, # 2218 + None, # 2219 + None, # 2220 + None, # 2221 + None, # 2222 + None, # 2223 + None, # 2224 + None, # 2225 + None, # 2226 + None, # 2227 + None, # 2228 + None, # 2229 + None, # 2230 + None, # 2231 + None, # 2232 + None, # 2233 + None, # 2234 + None, # 2235 + None, # 2236 + None, # 2237 + None, # 2238 + None, # 2239 + None, # 2240 + None, # 2241 + None, # 2242 + None, # 2243 + None, # 2244 + None, # 2245 + None, # 2246 + None, # 2247 + None, # 2248 + None, # 2249 + None, # 2250 + None, # 2251 + None, # 2252 + None, # 2253 + None, # 2254 + None, # 2255 + None, # 2256 + None, # 2257 + None, # 2258 + None, # 2259 + None, # 2260 + None, # 2261 + None, # 2262 + None, # 2263 + None, # 2264 + None, # 2265 + None, # 2266 + None, # 2267 + None, # 2268 + None, # 2269 + None, # 2270 + None, # 2271 + None, # 2272 + None, # 2273 + None, # 2274 + None, # 2275 + None, # 2276 + None, # 2277 + None, # 2278 + None, # 2279 + None, # 2280 + None, # 2281 + None, # 2282 + None, # 2283 + None, # 2284 + None, # 2285 + None, # 2286 + None, # 2287 + None, # 2288 + None, # 2289 + None, # 2290 + None, # 2291 + None, # 2292 + None, # 2293 + None, # 2294 + None, # 2295 + None, # 2296 + None, # 2297 + None, # 2298 + None, # 2299 + None, # 2300 + None, # 2301 + None, # 2302 + None, # 2303 + None, # 2304 + None, # 2305 + None, # 2306 + None, # 2307 + None, # 2308 + None, # 2309 + None, # 2310 + None, # 2311 + None, # 2312 + None, # 2313 + None, # 2314 + None, # 2315 + None, # 2316 + None, # 2317 + None, # 2318 + None, # 2319 + None, # 2320 + None, # 2321 + None, # 2322 + None, # 2323 + None, # 2324 + None, # 2325 + None, # 2326 + None, # 2327 + None, # 2328 + None, # 2329 + None, # 2330 + None, # 2331 + None, # 2332 + None, # 2333 + None, # 2334 + None, # 2335 + None, # 2336 + None, # 2337 + None, # 2338 + None, # 2339 + None, # 2340 + None, # 2341 + None, # 2342 + None, # 2343 + None, # 2344 + None, # 2345 + None, # 2346 + None, # 2347 + None, # 2348 + None, # 2349 + None, # 2350 + None, # 2351 + None, # 2352 + None, # 2353 + None, # 2354 + None, # 2355 + None, # 2356 + None, # 2357 + None, # 2358 + None, # 2359 + None, # 2360 + None, # 2361 + None, # 2362 + None, # 2363 + None, # 2364 + None, # 2365 + None, # 2366 + None, # 2367 + None, # 2368 + None, # 2369 + None, # 2370 + None, # 2371 + None, # 2372 + None, # 2373 + None, # 2374 + None, # 2375 + None, # 2376 + None, # 2377 + None, # 2378 + None, # 2379 + None, # 2380 + None, # 2381 + None, # 2382 + None, # 2383 + None, # 2384 + None, # 2385 + None, # 2386 + None, # 2387 + None, # 2388 + None, # 2389 + None, # 2390 + None, # 2391 + None, # 2392 + None, # 2393 + None, # 2394 + None, # 2395 + None, # 2396 + None, # 2397 + None, # 2398 + None, # 2399 + None, # 2400 + None, # 2401 + None, # 2402 + None, # 2403 + None, # 2404 + None, # 2405 + None, # 2406 + None, # 2407 + None, # 2408 + None, # 2409 + None, # 2410 + None, # 2411 + None, # 2412 + None, # 2413 + None, # 2414 + None, # 2415 + None, # 2416 + None, # 2417 + None, # 2418 + None, # 2419 + None, # 2420 + None, # 2421 + None, # 2422 + None, # 2423 + None, # 2424 + None, # 2425 + None, # 2426 + None, # 2427 + None, # 2428 + None, # 2429 + None, # 2430 + None, # 2431 + None, # 2432 + None, # 2433 + None, # 2434 + None, # 2435 + None, # 2436 + None, # 2437 + None, # 2438 + None, # 2439 + None, # 2440 + None, # 2441 + None, # 2442 + None, # 2443 + None, # 2444 + None, # 2445 + None, # 2446 + None, # 2447 + None, # 2448 + None, # 2449 + None, # 2450 + None, # 2451 + None, # 2452 + None, # 2453 + None, # 2454 + None, # 2455 + None, # 2456 + None, # 2457 + None, # 2458 + None, # 2459 + None, # 2460 + None, # 2461 + None, # 2462 + None, # 2463 + None, # 2464 + None, # 2465 + None, # 2466 + None, # 2467 + None, # 2468 + None, # 2469 + None, # 2470 + None, # 2471 + None, # 2472 + None, # 2473 + None, # 2474 + None, # 2475 + None, # 2476 + None, # 2477 + None, # 2478 + None, # 2479 + None, # 2480 + None, # 2481 + None, # 2482 + None, # 2483 + None, # 2484 + None, # 2485 + None, # 2486 + None, # 2487 + None, # 2488 + None, # 2489 + None, # 2490 + None, # 2491 + None, # 2492 + None, # 2493 + None, # 2494 + None, # 2495 + None, # 2496 + None, # 2497 + None, # 2498 + None, # 2499 + None, # 2500 + None, # 2501 + None, # 2502 + None, # 2503 + None, # 2504 + None, # 2505 + None, # 2506 + None, # 2507 + None, # 2508 + None, # 2509 + None, # 2510 + None, # 2511 + None, # 2512 + None, # 2513 + None, # 2514 + None, # 2515 + None, # 2516 + None, # 2517 + None, # 2518 + None, # 2519 + None, # 2520 + None, # 2521 + None, # 2522 + None, # 2523 + None, # 2524 + None, # 2525 + None, # 2526 + None, # 2527 + None, # 2528 + None, # 2529 + None, # 2530 + None, # 2531 + None, # 2532 + None, # 2533 + None, # 2534 + None, # 2535 + None, # 2536 + None, # 2537 + None, # 2538 + None, # 2539 + None, # 2540 + None, # 2541 + None, # 2542 + None, # 2543 + None, # 2544 + None, # 2545 + None, # 2546 + None, # 2547 + None, # 2548 + None, # 2549 + None, # 2550 + None, # 2551 + None, # 2552 + None, # 2553 + None, # 2554 + None, # 2555 + None, # 2556 + None, # 2557 + None, # 2558 + None, # 2559 + None, # 2560 + None, # 2561 + None, # 2562 + None, # 2563 + None, # 2564 + None, # 2565 + None, # 2566 + None, # 2567 + None, # 2568 + None, # 2569 + None, # 2570 + None, # 2571 + None, # 2572 + None, # 2573 + None, # 2574 + None, # 2575 + None, # 2576 + None, # 2577 + None, # 2578 + None, # 2579 + None, # 2580 + None, # 2581 + None, # 2582 + None, # 2583 + None, # 2584 + None, # 2585 + None, # 2586 + None, # 2587 + None, # 2588 + None, # 2589 + None, # 2590 + None, # 2591 + None, # 2592 + None, # 2593 + None, # 2594 + None, # 2595 + None, # 2596 + None, # 2597 + None, # 2598 + None, # 2599 + None, # 2600 + None, # 2601 + None, # 2602 + None, # 2603 + None, # 2604 + None, # 2605 + None, # 2606 + None, # 2607 + None, # 2608 + None, # 2609 + None, # 2610 + None, # 2611 + None, # 2612 + None, # 2613 + None, # 2614 + None, # 2615 + None, # 2616 + None, # 2617 + None, # 2618 + None, # 2619 + None, # 2620 + None, # 2621 + None, # 2622 + None, # 2623 + None, # 2624 + None, # 2625 + None, # 2626 + None, # 2627 + None, # 2628 + None, # 2629 + None, # 2630 + None, # 2631 + None, # 2632 + None, # 2633 + None, # 2634 + None, # 2635 + None, # 2636 + None, # 2637 + None, # 2638 + None, # 2639 + None, # 2640 + None, # 2641 + None, # 2642 + None, # 2643 + None, # 2644 + None, # 2645 + None, # 2646 + None, # 2647 + None, # 2648 + None, # 2649 + None, # 2650 + None, # 2651 + None, # 2652 + None, # 2653 + None, # 2654 + None, # 2655 + None, # 2656 + None, # 2657 + None, # 2658 + None, # 2659 + None, # 2660 + None, # 2661 + None, # 2662 + None, # 2663 + None, # 2664 + None, # 2665 + None, # 2666 + None, # 2667 + None, # 2668 + None, # 2669 + None, # 2670 + None, # 2671 + None, # 2672 + None, # 2673 + None, # 2674 + None, # 2675 + None, # 2676 + None, # 2677 + None, # 2678 + None, # 2679 + None, # 2680 + None, # 2681 + None, # 2682 + None, # 2683 + None, # 2684 + None, # 2685 + None, # 2686 + None, # 2687 + None, # 2688 + None, # 2689 + None, # 2690 + None, # 2691 + None, # 2692 + None, # 2693 + None, # 2694 + None, # 2695 + None, # 2696 + None, # 2697 + None, # 2698 + None, # 2699 + None, # 2700 + None, # 2701 + None, # 2702 + None, # 2703 + None, # 2704 + None, # 2705 + None, # 2706 + None, # 2707 + None, # 2708 + None, # 2709 + None, # 2710 + None, # 2711 + None, # 2712 + None, # 2713 + None, # 2714 + None, # 2715 + None, # 2716 + None, # 2717 + None, # 2718 + None, # 2719 + None, # 2720 + None, # 2721 + None, # 2722 + None, # 2723 + None, # 2724 + None, # 2725 + None, # 2726 + None, # 2727 + None, # 2728 + None, # 2729 + None, # 2730 + None, # 2731 + None, # 2732 + None, # 2733 + None, # 2734 + None, # 2735 + None, # 2736 + None, # 2737 + None, # 2738 + None, # 2739 + None, # 2740 + None, # 2741 + None, # 2742 + None, # 2743 + None, # 2744 + None, # 2745 + None, # 2746 + None, # 2747 + None, # 2748 + None, # 2749 + None, # 2750 + None, # 2751 + None, # 2752 + None, # 2753 + None, # 2754 + None, # 2755 + None, # 2756 + None, # 2757 + None, # 2758 + None, # 2759 + None, # 2760 + None, # 2761 + None, # 2762 + None, # 2763 + None, # 2764 + None, # 2765 + None, # 2766 + None, # 2767 + None, # 2768 + None, # 2769 + None, # 2770 + None, # 2771 + None, # 2772 + None, # 2773 + None, # 2774 + None, # 2775 + None, # 2776 + None, # 2777 + None, # 2778 + None, # 2779 + None, # 2780 + None, # 2781 + None, # 2782 + None, # 2783 + None, # 2784 + None, # 2785 + None, # 2786 + None, # 2787 + None, # 2788 + None, # 2789 + None, # 2790 + None, # 2791 + None, # 2792 + None, # 2793 + None, # 2794 + None, # 2795 + None, # 2796 + None, # 2797 + None, # 2798 + None, # 2799 + None, # 2800 + None, # 2801 + None, # 2802 + None, # 2803 + None, # 2804 + None, # 2805 + None, # 2806 + None, # 2807 + None, # 2808 + None, # 2809 + None, # 2810 + None, # 2811 + None, # 2812 + None, # 2813 + None, # 2814 + None, # 2815 + None, # 2816 + None, # 2817 + None, # 2818 + None, # 2819 + None, # 2820 + None, # 2821 + None, # 2822 + None, # 2823 + None, # 2824 + None, # 2825 + None, # 2826 + None, # 2827 + None, # 2828 + None, # 2829 + None, # 2830 + None, # 2831 + None, # 2832 + None, # 2833 + None, # 2834 + None, # 2835 + None, # 2836 + None, # 2837 + None, # 2838 + None, # 2839 + None, # 2840 + None, # 2841 + None, # 2842 + None, # 2843 + None, # 2844 + None, # 2845 + None, # 2846 + None, # 2847 + None, # 2848 + None, # 2849 + None, # 2850 + None, # 2851 + None, # 2852 + None, # 2853 + None, # 2854 + None, # 2855 + None, # 2856 + None, # 2857 + None, # 2858 + None, # 2859 + None, # 2860 + None, # 2861 + None, # 2862 + None, # 2863 + None, # 2864 + None, # 2865 + None, # 2866 + None, # 2867 + None, # 2868 + None, # 2869 + None, # 2870 + None, # 2871 + None, # 2872 + None, # 2873 + None, # 2874 + None, # 2875 + None, # 2876 + None, # 2877 + None, # 2878 + None, # 2879 + None, # 2880 + None, # 2881 + None, # 2882 + None, # 2883 + None, # 2884 + None, # 2885 + None, # 2886 + None, # 2887 + None, # 2888 + None, # 2889 + None, # 2890 + None, # 2891 + None, # 2892 + None, # 2893 + None, # 2894 + None, # 2895 + None, # 2896 + None, # 2897 + None, # 2898 + None, # 2899 + None, # 2900 + None, # 2901 + None, # 2902 + None, # 2903 + None, # 2904 + None, # 2905 + None, # 2906 + None, # 2907 + None, # 2908 + None, # 2909 + None, # 2910 + None, # 2911 + None, # 2912 + None, # 2913 + None, # 2914 + None, # 2915 + None, # 2916 + None, # 2917 + None, # 2918 + None, # 2919 + None, # 2920 + None, # 2921 + None, # 2922 + None, # 2923 + None, # 2924 + None, # 2925 + None, # 2926 + None, # 2927 + None, # 2928 + None, # 2929 + None, # 2930 + None, # 2931 + None, # 2932 + None, # 2933 + None, # 2934 + None, # 2935 + None, # 2936 + None, # 2937 + None, # 2938 + None, # 2939 + None, # 2940 + None, # 2941 + None, # 2942 + None, # 2943 + None, # 2944 + None, # 2945 + None, # 2946 + None, # 2947 + None, # 2948 + None, # 2949 + None, # 2950 + None, # 2951 + None, # 2952 + None, # 2953 + None, # 2954 + None, # 2955 + None, # 2956 + None, # 2957 + None, # 2958 + None, # 2959 + None, # 2960 + None, # 2961 + None, # 2962 + None, # 2963 + None, # 2964 + None, # 2965 + None, # 2966 + None, # 2967 + None, # 2968 + None, # 2969 + None, # 2970 + None, # 2971 + None, # 2972 + None, # 2973 + None, # 2974 + None, # 2975 + None, # 2976 + None, # 2977 + None, # 2978 + None, # 2979 + None, # 2980 + None, # 2981 + None, # 2982 + None, # 2983 + None, # 2984 + None, # 2985 + None, # 2986 + None, # 2987 + None, # 2988 + None, # 2989 + None, # 2990 + None, # 2991 + None, # 2992 + None, # 2993 + None, # 2994 + None, # 2995 + None, # 2996 + None, # 2997 + None, # 2998 + None, # 2999 + None, # 3000 + None, # 3001 + None, # 3002 + None, # 3003 + None, # 3004 + None, # 3005 + None, # 3006 + None, # 3007 + None, # 3008 + None, # 3009 + None, # 3010 + None, # 3011 + None, # 3012 + None, # 3013 + None, # 3014 + None, # 3015 + None, # 3016 + None, # 3017 + None, # 3018 + None, # 3019 + None, # 3020 + None, # 3021 + None, # 3022 + None, # 3023 + None, # 3024 + None, # 3025 + None, # 3026 + None, # 3027 + None, # 3028 + None, # 3029 + None, # 3030 + None, # 3031 + None, # 3032 + None, # 3033 + None, # 3034 + None, # 3035 + None, # 3036 + None, # 3037 + None, # 3038 + None, # 3039 + None, # 3040 + None, # 3041 + None, # 3042 + None, # 3043 + None, # 3044 + None, # 3045 + None, # 3046 + None, # 3047 + None, # 3048 + None, # 3049 + None, # 3050 + None, # 3051 + None, # 3052 + None, # 3053 + None, # 3054 + None, # 3055 + None, # 3056 + None, # 3057 + None, # 3058 + None, # 3059 + None, # 3060 + None, # 3061 + None, # 3062 + None, # 3063 + None, # 3064 + None, # 3065 + None, # 3066 + None, # 3067 + None, # 3068 + None, # 3069 + None, # 3070 + None, # 3071 + None, # 3072 + None, # 3073 + None, # 3074 + None, # 3075 + None, # 3076 + None, # 3077 + None, # 3078 + None, # 3079 + None, # 3080 + None, # 3081 + None, # 3082 + None, # 3083 + None, # 3084 + None, # 3085 + None, # 3086 + None, # 3087 + None, # 3088 + None, # 3089 + None, # 3090 + None, # 3091 + None, # 3092 + None, # 3093 + None, # 3094 + None, # 3095 + None, # 3096 + None, # 3097 + None, # 3098 + None, # 3099 + None, # 3100 + None, # 3101 + None, # 3102 + None, # 3103 + None, # 3104 + None, # 3105 + None, # 3106 + None, # 3107 + None, # 3108 + None, # 3109 + None, # 3110 + None, # 3111 + None, # 3112 + None, # 3113 + None, # 3114 + None, # 3115 + None, # 3116 + None, # 3117 + None, # 3118 + None, # 3119 + None, # 3120 + None, # 3121 + None, # 3122 + None, # 3123 + None, # 3124 + None, # 3125 + None, # 3126 + None, # 3127 + None, # 3128 + None, # 3129 + None, # 3130 + None, # 3131 + None, # 3132 + None, # 3133 + None, # 3134 + None, # 3135 + None, # 3136 + None, # 3137 + None, # 3138 + None, # 3139 + None, # 3140 + None, # 3141 + None, # 3142 + None, # 3143 + None, # 3144 + None, # 3145 + None, # 3146 + None, # 3147 + None, # 3148 + None, # 3149 + None, # 3150 + None, # 3151 + None, # 3152 + None, # 3153 + None, # 3154 + None, # 3155 + None, # 3156 + None, # 3157 + None, # 3158 + None, # 3159 + None, # 3160 + None, # 3161 + None, # 3162 + None, # 3163 + None, # 3164 + None, # 3165 + None, # 3166 + None, # 3167 + None, # 3168 + None, # 3169 + None, # 3170 + None, # 3171 + None, # 3172 + None, # 3173 + None, # 3174 + None, # 3175 + None, # 3176 + None, # 3177 + None, # 3178 + None, # 3179 + None, # 3180 + None, # 3181 + None, # 3182 + None, # 3183 + None, # 3184 + None, # 3185 + None, # 3186 + None, # 3187 + None, # 3188 + None, # 3189 + None, # 3190 + None, # 3191 + None, # 3192 + None, # 3193 + None, # 3194 + None, # 3195 + None, # 3196 + None, # 3197 + None, # 3198 + None, # 3199 + None, # 3200 + None, # 3201 + None, # 3202 + None, # 3203 + None, # 3204 + None, # 3205 + None, # 3206 + None, # 3207 + None, # 3208 + None, # 3209 + None, # 3210 + None, # 3211 + None, # 3212 + None, # 3213 + None, # 3214 + None, # 3215 + None, # 3216 + None, # 3217 + None, # 3218 + None, # 3219 + None, # 3220 + None, # 3221 + None, # 3222 + None, # 3223 + None, # 3224 + None, # 3225 + None, # 3226 + None, # 3227 + None, # 3228 + None, # 3229 + None, # 3230 + None, # 3231 + None, # 3232 + None, # 3233 + None, # 3234 + None, # 3235 + None, # 3236 + None, # 3237 + None, # 3238 + None, # 3239 + None, # 3240 + None, # 3241 + None, # 3242 + None, # 3243 + None, # 3244 + None, # 3245 + None, # 3246 + None, # 3247 + None, # 3248 + None, # 3249 + None, # 3250 + None, # 3251 + None, # 3252 + None, # 3253 + None, # 3254 + None, # 3255 + None, # 3256 + None, # 3257 + None, # 3258 + None, # 3259 + None, # 3260 + None, # 3261 + None, # 3262 + None, # 3263 + None, # 3264 + None, # 3265 + None, # 3266 + None, # 3267 + None, # 3268 + None, # 3269 + None, # 3270 + None, # 3271 + None, # 3272 + None, # 3273 + None, # 3274 + None, # 3275 + None, # 3276 + None, # 3277 + None, # 3278 + None, # 3279 + None, # 3280 + None, # 3281 + None, # 3282 + None, # 3283 + None, # 3284 + None, # 3285 + None, # 3286 + None, # 3287 + None, # 3288 + None, # 3289 + None, # 3290 + None, # 3291 + None, # 3292 + None, # 3293 + None, # 3294 + None, # 3295 + None, # 3296 + None, # 3297 + None, # 3298 + None, # 3299 + None, # 3300 + None, # 3301 + None, # 3302 + None, # 3303 + None, # 3304 + None, # 3305 + None, # 3306 + None, # 3307 + None, # 3308 + None, # 3309 + None, # 3310 + None, # 3311 + None, # 3312 + None, # 3313 + None, # 3314 + None, # 3315 + None, # 3316 + None, # 3317 + None, # 3318 + None, # 3319 + None, # 3320 + None, # 3321 + None, # 3322 + None, # 3323 + None, # 3324 + None, # 3325 + None, # 3326 + None, # 3327 + None, # 3328 + (3329, TType.STRING, 'responseValidation', 'BINARY', None, ), # 3329 + (3330, TType.I32, 'idempotencyType', None, None, ), # 3330 + (3331, TType.I64, 'statementTimeout', None, None, ), # 3331 + (3332, TType.I32, 'statementTimeoutLevel', None, None, ), # 3332 ) -all_structs.append(TGetOperationStatusReq) -TGetOperationStatusReq.thrift_spec = ( +all_structs.append(TCancelOperationReq) +TCancelOperationReq.thrift_spec = ( None, # 0 (1, TType.STRUCT, 'operationHandle', [TOperationHandle, None], None, ), # 1 - (2, TType.BOOL, 'getProgressUpdate', None, None, ), # 2 -) -all_structs.append(TGetOperationStatusResp) -TGetOperationStatusResp.thrift_spec = ( - None, # 0 - (1, TType.STRUCT, 'status', [TStatus, None], None, ), # 1 - (2, TType.I32, 'operationState', None, None, ), # 2 - (3, TType.STRING, 'sqlState', 'UTF8', None, ), # 3 - (4, TType.I32, 'errorCode', None, None, ), # 4 - (5, TType.STRING, 'errorMessage', 'UTF8', None, ), # 5 - (6, TType.STRING, 'taskStatus', 'UTF8', None, ), # 6 - (7, TType.I64, 'operationStarted', None, None, ), # 7 - (8, TType.I64, 'operationCompleted', None, None, ), # 8 - (9, TType.BOOL, 'hasResultSet', None, None, ), # 9 - (10, TType.STRUCT, 'progressUpdateResponse', [TProgressUpdateResp, None], None, ), # 10 - (11, TType.I64, 'numModifiedRows', None, None, ), # 11 + None, # 2 + None, # 3 + None, # 4 + None, # 5 + None, # 6 + None, # 7 + None, # 8 + None, # 9 + None, # 10 + None, # 11 None, # 12 None, # 13 None, # 14 @@ -78997,8 +85049,8 @@ def __ne__(self, other): None, # 1278 None, # 1279 None, # 1280 - (1281, TType.STRING, 'displayMessage', 'UTF8', None, ), # 1281 - (1282, TType.STRING, 'diagnosticInfo', 'UTF8', None, ), # 1282 + None, # 1281 + None, # 1282 None, # 1283 None, # 1284 None, # 1285 @@ -81045,13 +87097,16 @@ def __ne__(self, other): None, # 3326 None, # 3327 None, # 3328 - (3329, TType.STRING, 'responseValidation', 'BINARY', None, ), # 3329 - (3330, TType.I32, 'idempotencyType', None, None, ), # 3330 - (3331, TType.I64, 'statementTimeout', None, None, ), # 3331 - (3332, TType.I32, 'statementTimeoutLevel', None, None, ), # 3332 + (3329, TType.I16, 'executionVersion', None, None, ), # 3329 + (3330, TType.BOOL, 'replacedByNextAttempt', None, None, ), # 3330 ) -all_structs.append(TCancelOperationReq) -TCancelOperationReq.thrift_spec = ( +all_structs.append(TCancelOperationResp) +TCancelOperationResp.thrift_spec = ( + None, # 0 + (1, TType.STRUCT, 'status', [TStatus, None], None, ), # 1 +) +all_structs.append(TCloseOperationReq) +TCloseOperationReq.thrift_spec = ( None, # 0 (1, TType.STRUCT, 'operationHandle', [TOperationHandle, None], None, ), # 1 None, # 2 @@ -84381,18 +90436,7 @@ def __ne__(self, other): None, # 3326 None, # 3327 None, # 3328 - (3329, TType.I16, 'executionVersion', None, None, ), # 3329 - (3330, TType.BOOL, 'replacedByNextAttempt', None, None, ), # 3330 -) -all_structs.append(TCancelOperationResp) -TCancelOperationResp.thrift_spec = ( - None, # 0 - (1, TType.STRUCT, 'status', [TStatus, None], None, ), # 1 -) -all_structs.append(TCloseOperationReq) -TCloseOperationReq.thrift_spec = ( - None, # 0 - (1, TType.STRUCT, 'operationHandle', [TOperationHandle, None], None, ), # 1 + (3329, TType.I32, 'closeReason', None, 0, ), # 3329 ) all_structs.append(TCloseOperationResp) TCloseOperationResp.thrift_spec = ( @@ -91066,7 +97110,21 @@ def __ne__(self, other): (3329, TType.I32, 'reasonForNoCloudFetch', None, None, ), # 3329 (3330, TType.LIST, 'resultFiles', (TType.STRUCT, [TDBSqlCloudResultFile, None], False), None, ), # 3330 (3331, TType.STRING, 'manifestFile', 'UTF8', None, ), # 3331 - (3332, TType.STRING, 'manifestFileFormat', 'UTF8', None, ), # 3332 + (3332, TType.I32, 'manifestFileFormat', None, None, ), # 3332 + (3333, TType.I64, 'cacheLookupLatency', None, None, ), # 3333 + (3334, TType.STRING, 'remoteCacheMissReason', 'UTF8', None, ), # 3334 + (3335, TType.I32, 'fetchDisposition', None, None, ), # 3335 + (3336, TType.BOOL, 'remoteResultCacheEnabled', None, None, ), # 3336 + (3337, TType.BOOL, 'isServerless', None, None, ), # 3337 + None, # 3338 + None, # 3339 + None, # 3340 + None, # 3341 + None, # 3342 + None, # 3343 + (3344, TType.STRUCT, 'resultDataFormat', [TDBSqlResultFormat, None], None, ), # 3344 + (3345, TType.BOOL, 'truncatedByThriftLimit', None, None, ), # 3345 + (3346, TType.I64, 'resultByteLimit', None, None, ), # 3346 ) all_structs.append(TFetchResultsReq) TFetchResultsReq.thrift_spec = ( @@ -105713,50 +111771,5 @@ def __ne__(self, other): (5, TType.STRING, 'footerSummary', 'UTF8', None, ), # 5 (6, TType.I64, 'startTime', None, None, ), # 6 ) -all_structs.append(TDBSqlClusterMetrics) -TDBSqlClusterMetrics.thrift_spec = ( - None, # 0 - (1, TType.I32, 'clusterCapacity', None, None, ), # 1 - (2, TType.I32, 'numRunningTasks', None, None, ), # 2 - (3, TType.I32, 'numPendingTasks', None, None, ), # 3 - (4, TType.DOUBLE, 'rejectionThreshold', None, None, ), # 4 - (5, TType.DOUBLE, 'tasksCompletedPerMinute', None, None, ), # 5 -) -all_structs.append(TDBSqlQueryLaneMetrics) -TDBSqlQueryLaneMetrics.thrift_spec = ( - None, # 0 - (1, TType.I32, 'fastLaneReservation', None, None, ), # 1 - (2, TType.I32, 'numFastLaneRunningTasks', None, None, ), # 2 - (3, TType.I32, 'numFastLanePendingTasks', None, None, ), # 3 - (4, TType.I32, 'slowLaneReservation', None, None, ), # 4 - (5, TType.I32, 'numSlowLaneRunningTasks', None, None, ), # 5 - (6, TType.I32, 'numSlowLanePendingTasks', None, None, ), # 6 -) -all_structs.append(TDBSqlQueryMetrics) -TDBSqlQueryMetrics.thrift_spec = ( - None, # 0 - (1, TType.STRUCT, 'status', [TStatus, None], None, ), # 1 - (2, TType.STRUCT, 'operationHandle', [TOperationHandle, None], None, ), # 2 - (3, TType.I32, 'idempotencyType', None, None, ), # 3 - (4, TType.STRUCT, 'sessionHandle', [TSessionHandle, None], None, ), # 4 - (5, TType.I64, 'operationStarted', None, None, ), # 5 - (6, TType.DOUBLE, 'queryCost', None, None, ), # 6 - (7, TType.I32, 'numRunningTasks', None, None, ), # 7 - (8, TType.I32, 'numPendingTasks', None, None, ), # 8 - (9, TType.I32, 'numCompletedTasks', None, None, ), # 9 -) -all_structs.append(TDBSqlGetLoadInformationReq) -TDBSqlGetLoadInformationReq.thrift_spec = ( - None, # 0 - (1, TType.BOOL, 'includeQueryMetrics', None, False, ), # 1 -) -all_structs.append(TDBSqlGetLoadInformationResp) -TDBSqlGetLoadInformationResp.thrift_spec = ( - None, # 0 - (1, TType.STRUCT, 'status', [TStatus, None], None, ), # 1 - (2, TType.STRUCT, 'clusterMetrics', [TDBSqlClusterMetrics, None], None, ), # 2 - (3, TType.STRUCT, 'queryLaneMetrics', [TDBSqlQueryLaneMetrics, None], None, ), # 3 - (4, TType.LIST, 'queryMetrics', (TType.STRUCT, [TDBSqlQueryMetrics, None], False), None, ), # 4 -) fix_spec(all_structs) del all_structs diff --git a/src/databricks/sql/thrift_backend.py b/src/databricks/sql/thrift_backend.py index 935c7711..7f6ada9d 100644 --- a/src/databricks/sql/thrift_backend.py +++ b/src/databricks/sql/thrift_backend.py @@ -3,40 +3,61 @@ import logging import math import time +import uuid import threading -import lz4.frame -from ssl import CERT_NONE, CERT_REQUIRED, create_default_context from typing import List, Union -import pyarrow +try: + import pyarrow +except ImportError: + pyarrow = None import thrift.transport.THttpClient import thrift.protocol.TBinaryProtocol import thrift.transport.TSocket import thrift.transport.TTransport +import urllib3.exceptions + import databricks.sql.auth.thrift_http_client +from databricks.sql.auth.thrift_http_client import CommandType from databricks.sql.auth.authenticators import AuthProvider from databricks.sql.thrift_api.TCLIService import TCLIService, ttypes from databricks.sql import * +from databricks.sql.exc import MaxRetryDurationError from databricks.sql.thrift_api.TCLIService.TCLIService import ( Client as TCLIServiceClient, ) from databricks.sql.utils import ( - ArrowQueue, ExecuteResponse, _bound, RequestErrorInfo, NoRetryReason, + ResultSetQueueFactory, + convert_arrow_based_set_to_arrow_table, + convert_decimals_in_arrow_table, + convert_column_based_set_to_arrow_table, ) +from databricks.sql.types import SSLOptions logger = logging.getLogger(__name__) +unsafe_logger = logging.getLogger("databricks.sql.unsafe") +unsafe_logger.setLevel(logging.DEBUG) + +# To capture these logs in client code, add a non-NullHandler. +# See our e2e test suite for an example with logging.FileHandler +unsafe_logger.addHandler(logging.NullHandler()) + +# Disable propagation so that handlers for `databricks.sql` don't pick up these messages +unsafe_logger.propagate = False + THRIFT_ERROR_MESSAGE_HEADER = "x-thriftserver-error-message" DATABRICKS_ERROR_OR_REDIRECT_HEADER = "x-databricks-error-or-redirect-message" DATABRICKS_REASON_HEADER = "x-databricks-reason-phrase" TIMESTAMP_AS_STRING_CONFIG = "spark.thriftserver.arrowBasedRowSet.timestampAsString" +DEFAULT_SOCKET_TIMEOUT = float(900) # see Connection.__init__ for parameter descriptions. # - Min/Max avoids unsustainable configs (sane values are far more constrained) @@ -53,7 +74,12 @@ class ThriftBackend: CLOSED_OP_STATE = ttypes.TOperationState.CLOSED_STATE ERROR_OP_STATE = ttypes.TOperationState.ERROR_STATE - BIT_MASKS = [1, 2, 4, 8, 16, 32, 64, 128] + + _retry_delay_min: float + _retry_delay_max: float + _retry_stop_after_attempts_count: int + _retry_stop_after_attempts_duration: float + _retry_delay_default: float def __init__( self, @@ -62,6 +88,7 @@ def __init__( http_path: str, http_headers, auth_provider: AuthProvider, + ssl_options: SSLOptions, staging_allowed_local_path: Union[None, str, List[str]] = None, **kwargs, ): @@ -70,16 +97,6 @@ def __init__( # Tag to add to User-Agent header. For use by partners. # _username, _password # Username and password Basic authentication (no official support) - # _tls_no_verify - # Set to True (Boolean) to completely disable SSL verification. - # _tls_verify_hostname - # Set to False (Boolean) to disable SSL hostname verification, but check certificate. - # _tls_trusted_ca_file - # Set to the path of the file containing trusted CA certificates for server certificate - # verification. If not provide, uses system truststore. - # _tls_client_cert_file, _tls_client_cert_key_file, _tls_client_cert_key_password - # Set client SSL certificate. - # See https://docs.python.org/3/library/ssl.html#ssl.SSLContext.load_cert_chain # _connection_uri # Overrides server_hostname and http_path. # RETRY/ATTEMPT POLICY @@ -98,17 +115,31 @@ def __init__( # # _retry_stop_after_attempts_count # The maximum number of times we should retry retryable requests (defaults to 24) + # _retry_dangerous_codes + # An iterable of integer HTTP status codes. ExecuteStatement commands will be retried if these codes are received. + # (defaults to []) # _socket_timeout - # The timeout in seconds for socket send, recv and connect operations. Defaults to None for - # no timeout. Should be a positive float or integer. + # The timeout in seconds for socket send, recv and connect operations. Should be a positive float or integer. + # (defaults to 900) + # _enable_v3_retries + # Whether to use the DatabricksRetryPolicy implemented in urllib3 + # (defaults to True) + # _retry_max_redirects + # An integer representing the maximum number of redirects to follow for a request. + # This number must be <= _retry_stop_after_attempts_count. + # (defaults to None) + # max_download_threads + # Number of threads for handling cloud fetch downloads. Defaults to 10 port = port or 443 if kwargs.get("_connection_uri"): uri = kwargs.get("_connection_uri") elif server_hostname and http_path: - uri = "https://{host}:{port}/{path}".format( - host=server_hostname, port=port, path=http_path.lstrip("/") + uri = "{host}:{port}/{path}".format( + host=server_hostname.rstrip("/"), port=port, path=http_path.lstrip("/") ) + if not uri.startswith("https://"): + uri = "https://" + uri else: raise ValueError("No valid connection settings.") @@ -122,38 +153,56 @@ def __init__( "_use_arrow_native_timestamps", True ) - # Configure tls context - ssl_context = create_default_context(cafile=kwargs.get("_tls_trusted_ca_file")) - if kwargs.get("_tls_no_verify") is True: - ssl_context.check_hostname = False - ssl_context.verify_mode = CERT_NONE - elif kwargs.get("_tls_verify_hostname") is False: - ssl_context.check_hostname = False - ssl_context.verify_mode = CERT_REQUIRED + # Cloud fetch + self.max_download_threads = kwargs.get("max_download_threads", 10) + + self._ssl_options = ssl_options + + self._auth_provider = auth_provider + + # Connector version 3 retry approach + self.enable_v3_retries = kwargs.get("_enable_v3_retries", True) + + if not self.enable_v3_retries: + logger.warning( + "Legacy retry behavior is enabled for this connection." + " This behaviour is deprecated and will be removed in a future release." + ) + self.force_dangerous_codes = kwargs.get("_retry_dangerous_codes", []) + + additional_transport_args = {} + _max_redirects: Union[None, int] = kwargs.get("_retry_max_redirects") + + if _max_redirects: + if _max_redirects > self._retry_stop_after_attempts_count: + logger.warn( + "_retry_max_redirects > _retry_stop_after_attempts_count so it will have no affect!" + ) + urllib3_kwargs = {"redirect": _max_redirects} else: - ssl_context.check_hostname = True - ssl_context.verify_mode = CERT_REQUIRED - - tls_client_cert_file = kwargs.get("_tls_client_cert_file") - tls_client_cert_key_file = kwargs.get("_tls_client_cert_key_file") - tls_client_cert_key_password = kwargs.get("_tls_client_cert_key_password") - if tls_client_cert_file: - ssl_context.load_cert_chain( - certfile=tls_client_cert_file, - keyfile=tls_client_cert_key_file, - password=tls_client_cert_key_password, + urllib3_kwargs = {} + if self.enable_v3_retries: + self.retry_policy = databricks.sql.auth.thrift_http_client.DatabricksRetryPolicy( + delay_min=self._retry_delay_min, + delay_max=self._retry_delay_max, + stop_after_attempts_count=self._retry_stop_after_attempts_count, + stop_after_attempts_duration=self._retry_stop_after_attempts_duration, + delay_default=self._retry_delay_default, + force_dangerous_codes=self.force_dangerous_codes, + urllib3_kwargs=urllib3_kwargs, ) - self._auth_provider = auth_provider + additional_transport_args["retry_policy"] = self.retry_policy self._transport = databricks.sql.auth.thrift_http_client.THttpClient( auth_provider=self._auth_provider, uri_or_host=uri, - ssl_context=ssl_context, + ssl_options=self._ssl_options, + **additional_transport_args, # type: ignore ) - timeout = kwargs.get("_socket_timeout") - # setTimeout defaults to None (i.e. no timeout), and is expected in ms + timeout = kwargs.get("_socket_timeout", DEFAULT_SOCKET_TIMEOUT) + # setTimeout defaults to 15 minutes and is expected in ms self._transport.setTimeout(timeout and (float(timeout) * 1000.0)) self._transport.setCustomHeaders(dict(http_headers)) @@ -168,10 +217,11 @@ def __init__( self._request_lock = threading.RLock() + # TODO: Move this bounding logic into DatabricksRetryPolicy for v3 (PECO-918) def _initialize_retry_args(self, kwargs): # Configure retries & timing: use user-settings or defaults, and bound # by policy. Log.warn when given param gets restricted. - for (key, (type_, default, min, max)) in _retry_policy.items(): + for key, (type_, default, min, max) in _retry_policy.items(): given_or_default = type_(kwargs.get(key, default)) bound = _bound(min, max, given_or_default) setattr(self, key, bound) @@ -300,8 +350,8 @@ def extract_retry_delay(attempt): # encapsulate retry checks, returns None || delay-in-secs # Retry IFF 429/503 code + Retry-After header set http_code = getattr(self._transport, "code", None) - retry_after = getattr(self._transport, "headers", {}).get("Retry-After") - if http_code in [429, 503] and retry_after: + retry_after = getattr(self._transport, "headers", {}).get("Retry-After", 1) + if http_code in [429, 503]: # bound delay (seconds) by [min_delay*1.5^(attempt-1), max_delay] return bound_retry_delay(attempt, int(retry_after)) return None @@ -315,10 +365,44 @@ def attempt_request(attempt): error, error_message, retry_delay = None, None, None try: - logger.debug("Sending request: {}".format(request)) + this_method_name = getattr(method, "__name__") + + logger.debug("Sending request: {}()".format(this_method_name)) + unsafe_logger.debug("Sending request: {}".format(request)) + + # These three lines are no-ops if the v3 retry policy is not in use + if self.enable_v3_retries: + this_command_type = CommandType.get(this_method_name) + self._transport.set_retry_command_type(this_command_type) + self._transport.startRetryTimer() + response = method(request) - logger.debug("Received response: {}".format(response)) + + # We need to call type(response) here because thrift doesn't implement __name__ attributes for thrift responses + logger.debug( + "Received response: {}()".format(type(response).__name__) + ) + unsafe_logger.debug("Received response: {}".format(response)) return response + + except urllib3.exceptions.HTTPError as err: + # retry on timeout. Happens a lot in Azure and it is safe as data has not been sent to server yet + + # TODO: don't use exception handling for GOS polling... + + gos_name = TCLIServiceClient.GetOperationStatus.__name__ + if method.__name__ == gos_name: + delay_default = ( + self.enable_v3_retries + and self.retry_policy.delay_default + or self._retry_delay_default + ) + retry_delay = bound_retry_delay(attempt, delay_default) + logger.info( + f"GetOperationStatus failed with HTTP error and will be retried: {str(err)}" + ) + else: + raise err except OSError as err: error = err error_message = str(err) @@ -327,11 +411,11 @@ def attempt_request(attempt): # log.info for errors we believe are not unusual or unexpected. log.warn for # for others like EEXIST, EBADF, ERANGE which are not expected in this context. # - # I manually tested this retry behaviour using mitmweb and confirmed that + # I manually tested this retry behaviour using mitmweb and confirmed that # GetOperationStatus requests are retried when I forced network connection # interruptions / timeouts / reconnects. See #24 for more info. # | Debian | Darwin | - info_errs = [ # |--------|--------| + info_errs = [ # |--------|--------| errno.ESHUTDOWN, # | 32 | 32 | errno.EAFNOSUPPORT, # | 97 | 47 | errno.ECONNRESET, # | 104 | 54 | @@ -355,6 +439,10 @@ def attempt_request(attempt): error_message = ThriftBackend._extract_error_message_from_headers( getattr(self._transport, "headers", {}) ) + finally: + # Calling `close()` here releases the active HTTP connection back to the pool + self._transport.close() + return RequestErrorInfo( error=error, error_message=error_message, @@ -464,7 +552,7 @@ def open_session(self, session_configuration, catalog, schema): response = self.make_request(self._client.OpenSession, open_session_req) self._check_initial_namespace(catalog, schema, response) self._check_protocol_version(response) - return response.sessionHandle + return response except: self._transport.close() raise @@ -484,7 +572,8 @@ def _check_command_not_in_error_or_closed_state( raise ServerOperationError( get_operations_resp.displayMessage, { - "operation-id": op_handle and op_handle.operationId.guid, + "operation-id": op_handle + and self.guid_to_hex_id(op_handle.operationId.guid), "diagnostic-info": get_operations_resp.diagnosticInfo, }, ) @@ -492,16 +581,20 @@ def _check_command_not_in_error_or_closed_state( raise ServerOperationError( get_operations_resp.errorMessage, { - "operation-id": op_handle and op_handle.operationId.guid, + "operation-id": op_handle + and self.guid_to_hex_id(op_handle.operationId.guid), "diagnostic-info": None, }, ) elif get_operations_resp.operationState == ttypes.TOperationState.CLOSED_STATE: raise DatabaseError( "Command {} unexpectedly closed server side".format( - op_handle and op_handle.operationId.guid + op_handle and self.guid_to_hex_id(op_handle.operationId.guid) ), - {"operation-id": op_handle and op_handle.operationId.guid}, + { + "operation-id": op_handle + and self.guid_to_hex_id(op_handle.operationId.guid) + }, ) def _poll_for_status(self, op_handle): @@ -516,108 +609,14 @@ def _create_arrow_table(self, t_row_set, lz4_compressed, schema_bytes, descripti ( arrow_table, num_rows, - ) = ThriftBackend._convert_column_based_set_to_arrow_table( - t_row_set.columns, description - ) + ) = convert_column_based_set_to_arrow_table(t_row_set.columns, description) elif t_row_set.arrowBatches is not None: - ( - arrow_table, - num_rows, - ) = ThriftBackend._convert_arrow_based_set_to_arrow_table( + (arrow_table, num_rows,) = convert_arrow_based_set_to_arrow_table( t_row_set.arrowBatches, lz4_compressed, schema_bytes ) else: raise OperationalError("Unsupported TRowSet instance {}".format(t_row_set)) - return self._convert_decimals_in_arrow_table(arrow_table, description), num_rows - - @staticmethod - def _convert_decimals_in_arrow_table(table, description): - for (i, col) in enumerate(table.itercolumns()): - if description[i][1] == "decimal": - decimal_col = col.to_pandas().apply( - lambda v: v if v is None else Decimal(v) - ) - precision, scale = description[i][4], description[i][5] - assert scale is not None - assert precision is not None - # Spark limits decimal to a maximum scale of 38, - # so 128 is guaranteed to be big enough - dtype = pyarrow.decimal128(precision, scale) - col_data = pyarrow.array(decimal_col, type=dtype) - field = table.field(i).with_type(dtype) - table = table.set_column(i, field, col_data) - return table - - @staticmethod - def _convert_arrow_based_set_to_arrow_table( - arrow_batches, lz4_compressed, schema_bytes - ): - ba = bytearray() - ba += schema_bytes - n_rows = 0 - if lz4_compressed: - for arrow_batch in arrow_batches: - n_rows += arrow_batch.rowCount - ba += lz4.frame.decompress(arrow_batch.batch) - else: - for arrow_batch in arrow_batches: - n_rows += arrow_batch.rowCount - ba += arrow_batch.batch - arrow_table = pyarrow.ipc.open_stream(ba).read_all() - return arrow_table, n_rows - - @staticmethod - def _convert_column_based_set_to_arrow_table(columns, description): - arrow_table = pyarrow.Table.from_arrays( - [ThriftBackend._convert_column_to_arrow_array(c) for c in columns], - # Only use the column names from the schema, the types are determined by the - # physical types used in column based set, as they can differ from the - # mapping used in _hive_schema_to_arrow_schema. - names=[c[0] for c in description], - ) - return arrow_table, arrow_table.num_rows - - @staticmethod - def _convert_column_to_arrow_array(t_col): - """ - Return a pyarrow array from the values in a TColumn instance. - Note that ColumnBasedSet has no native support for complex types, so they will be converted - to strings server-side. - """ - field_name_to_arrow_type = { - "boolVal": pyarrow.bool_(), - "byteVal": pyarrow.int8(), - "i16Val": pyarrow.int16(), - "i32Val": pyarrow.int32(), - "i64Val": pyarrow.int64(), - "doubleVal": pyarrow.float64(), - "stringVal": pyarrow.string(), - "binaryVal": pyarrow.binary(), - } - for field in field_name_to_arrow_type.keys(): - wrapper = getattr(t_col, field) - if wrapper: - return ThriftBackend._create_arrow_array( - wrapper, field_name_to_arrow_type[field] - ) - - raise OperationalError("Empty TColumn instance {}".format(t_col)) - - @staticmethod - def _create_arrow_array(t_col_value_wrapper, arrow_type): - result = t_col_value_wrapper.values - nulls = t_col_value_wrapper.nulls # bitfield describing which values are null - assert isinstance(nulls, bytes) - - # The number of bits in nulls can be both larger or smaller than the number of - # elements in result, so take the minimum of both to iterate over. - length = min(len(result), len(nulls) * 8) - - for i in range(length): - if nulls[i >> 3] & ThriftBackend.BIT_MASKS[i & 0x7]: - result[i] = None - - return pyarrow.array(result, type=arrow_type) + return convert_decimals_in_arrow_table(arrow_table, description), num_rows def _get_metadata_resp(self, op_handle): req = ttypes.TGetResultSetMetadataReq(operationHandle=op_handle) @@ -625,6 +624,7 @@ def _get_metadata_resp(self, op_handle): @staticmethod def _hive_schema_to_arrow_schema(t_table_schema): + def map_type(t_type_entry): if t_type_entry.primitiveEntry: return { @@ -710,6 +710,7 @@ def _results_message_to_execute_response(self, resp, operation_state): if t_result_set_metadata_resp.resultFormat not in [ ttypes.TSparkRowSetType.ARROW_BASED_SET, ttypes.TSparkRowSetType.COLUMN_BASED_SET, + ttypes.TSparkRowSetType.URL_BASED_SET, ]: raise OperationalError( "Expected results to be in Arrow or column based format, " @@ -729,25 +730,32 @@ def _results_message_to_execute_response(self, resp, operation_state): description = self._hive_schema_to_description( t_result_set_metadata_resp.schema ) - schema_bytes = ( - t_result_set_metadata_resp.arrowSchema - or self._hive_schema_to_arrow_schema(t_result_set_metadata_resp.schema) - .serialize() - .to_pybytes() - ) + + if pyarrow: + schema_bytes = ( + t_result_set_metadata_resp.arrowSchema + or self._hive_schema_to_arrow_schema(t_result_set_metadata_resp.schema) + .serialize() + .to_pybytes() + ) + else: + schema_bytes = None + lz4_compressed = t_result_set_metadata_resp.lz4Compressed is_staging_operation = t_result_set_metadata_resp.isStagingOperation if direct_results and direct_results.resultSet: assert direct_results.resultSet.results.startRowOffset == 0 assert direct_results.resultSetMetadata - arrow_results, n_rows = self._create_arrow_table( - direct_results.resultSet.results, - lz4_compressed, - schema_bytes, - description, + arrow_queue_opt = ResultSetQueueFactory.build_queue( + row_set_type=t_result_set_metadata_resp.resultFormat, + t_row_set=direct_results.resultSet.results, + arrow_schema_bytes=schema_bytes, + max_download_threads=self.max_download_threads, + lz4_compressed=lz4_compressed, + description=description, + ssl_options=self._ssl_options, ) - arrow_queue_opt = ArrowQueue(arrow_results, n_rows, 0) else: arrow_queue_opt = None return ExecuteResponse( @@ -801,7 +809,15 @@ def _check_direct_results_for_error(t_spark_direct_results): ) def execute_command( - self, operation, session_handle, max_rows, max_bytes, lz4_compression, cursor + self, + operation, + session_handle, + max_rows, + max_bytes, + lz4_compression, + cursor, + use_cloud_fetch=True, + parameters=[], ): assert session_handle is not None @@ -820,14 +836,15 @@ def execute_command( getDirectResults=ttypes.TSparkGetDirectResults( maxRows=max_rows, maxBytes=max_bytes ), - canReadArrowResult=True, + canReadArrowResult=True if pyarrow else False, canDecompressLZ4Result=lz4_compression, - canDownloadResult=False, + canDownloadResult=use_cloud_fetch, confOverlay={ # We want to receive proper Timestamp arrow types. "spark.thriftserver.arrowBasedRowSet.timestampAsString": "false" }, useArrowNativeTypes=spark_arrow_types, + parameters=parameters, ) resp = self.make_request(self._client.ExecuteStatement, req) return self._handle_execute_response(resp, cursor) @@ -951,6 +968,7 @@ def fetch_results( maxRows=max_rows, maxBytes=max_bytes, orientation=ttypes.TFetchOrientation.FETCH_NEXT, + includeResultSetMetadata=True, ) resp = self.make_request(self._client.FetchResults, req) @@ -960,12 +978,18 @@ def fetch_results( expected_row_start_offset, resp.results.startRowOffset ) ) - arrow_results, n_rows = self._create_arrow_table( - resp.results, lz4_compressed, arrow_schema_bytes, description + + queue = ResultSetQueueFactory.build_queue( + row_set_type=resp.resultSetMetadata.resultFormat, + t_row_set=resp.results, + arrow_schema_bytes=arrow_schema_bytes, + max_download_threads=self.max_download_threads, + lz4_compressed=lz4_compressed, + description=description, + ssl_options=self._ssl_options, ) - arrow_queue = ArrowQueue(arrow_results, n_rows) - return arrow_queue, resp.hasMoreRows + return queue, resp.hasMoreRows def close_command(self, op_handle): req = ttypes.TCloseOperationReq(operationHandle=op_handle) @@ -973,10 +997,39 @@ def close_command(self, op_handle): return resp.status def cancel_command(self, active_op_handle): - logger.debug("Cancelling command {}".format(active_op_handle.operationId.guid)) + logger.debug( + "Cancelling command {}".format( + self.guid_to_hex_id(active_op_handle.operationId.guid) + ) + ) req = ttypes.TCancelOperationReq(active_op_handle) self.make_request(self._client.CancelOperation, req) @staticmethod def handle_to_id(session_handle): return session_handle.sessionId.guid + + @staticmethod + def handle_to_hex_id(session_handle: TCLIService.TSessionHandle): + this_uuid = uuid.UUID(bytes=session_handle.sessionId.guid) + return str(this_uuid) + + @staticmethod + def guid_to_hex_id(guid: bytes) -> str: + """Return a hexadecimal string instead of bytes + + Example: + IN b'\x01\xee\x1d)\xa4\x19\x1d\xb6\xa9\xc0\x8d\xf1\xfe\xbaB\xdd' + OUT '01ee1d29-a419-1db6-a9c0-8df1feba42dd' + + If conversion to hexadecimal fails, the original bytes are returned + """ + + this_uuid: Union[bytes, uuid.UUID] + + try: + this_uuid = uuid.UUID(bytes=guid) + except Exception as e: + logger.debug(f"Unable to convert bytes to UUID: {bytes} -- {str(e)}") + this_uuid = guid + return str(this_uuid) diff --git a/src/databricks/sql/types.py b/src/databricks/sql/types.py index b44704cd..fef22cd9 100644 --- a/src/databricks/sql/types.py +++ b/src/databricks/sql/types.py @@ -16,7 +16,57 @@ # # Row class was taken from Apache Spark pyspark. -from typing import Any, Dict, List, Optional, Tuple, Union +from typing import Any, Dict, List, Optional, Tuple, Union, TypeVar +import datetime +import decimal +from ssl import SSLContext, CERT_NONE, CERT_REQUIRED, create_default_context + + +class SSLOptions: + tls_verify: bool + tls_verify_hostname: bool + tls_trusted_ca_file: Optional[str] + tls_client_cert_file: Optional[str] + tls_client_cert_key_file: Optional[str] + tls_client_cert_key_password: Optional[str] + + def __init__( + self, + tls_verify: bool = True, + tls_verify_hostname: bool = True, + tls_trusted_ca_file: Optional[str] = None, + tls_client_cert_file: Optional[str] = None, + tls_client_cert_key_file: Optional[str] = None, + tls_client_cert_key_password: Optional[str] = None, + ): + self.tls_verify = tls_verify + self.tls_verify_hostname = tls_verify_hostname + self.tls_trusted_ca_file = tls_trusted_ca_file + self.tls_client_cert_file = tls_client_cert_file + self.tls_client_cert_key_file = tls_client_cert_key_file + self.tls_client_cert_key_password = tls_client_cert_key_password + + def create_ssl_context(self) -> SSLContext: + ssl_context = create_default_context(cafile=self.tls_trusted_ca_file) + + if self.tls_verify is False: + ssl_context.check_hostname = False + ssl_context.verify_mode = CERT_NONE + elif self.tls_verify_hostname is False: + ssl_context.check_hostname = False + ssl_context.verify_mode = CERT_REQUIRED + else: + ssl_context.check_hostname = True + ssl_context.verify_mode = CERT_REQUIRED + + if self.tls_client_cert_file: + ssl_context.load_cert_chain( + certfile=self.tls_client_cert_file, + keyfile=self.tls_client_cert_key_file, + password=self.tls_client_cert_key_password, + ) + + return ssl_context class Row(tuple): diff --git a/src/databricks/sql/utils.py b/src/databricks/sql/utils.py index ed558136..ffeaeaf0 100644 --- a/src/databricks/sql/utils.py +++ b/src/databricks/sql/utils.py @@ -1,16 +1,159 @@ -from collections import namedtuple, OrderedDict +from __future__ import annotations + +import pytz +import datetime +import decimal +from abc import ABC, abstractmethod +from collections import OrderedDict, namedtuple from collections.abc import Iterable -import datetime, decimal +from decimal import Decimal from enum import Enum -from typing import Dict -import pyarrow +from typing import Any, Dict, List, Optional, Union +import re + +import lz4.frame +try: + import pyarrow +except ImportError: + pyarrow = None + +from databricks.sql import OperationalError, exc +from databricks.sql.cloudfetch.download_manager import ResultFileDownloadManager +from databricks.sql.thrift_api.TCLIService.ttypes import ( + TRowSet, + TSparkArrowResultLink, + TSparkRowSetType, +) +from databricks.sql.types import SSLOptions + +from databricks.sql.parameters.native import ParameterStructure, TDbsqlParameter + +import logging + +BIT_MASKS = [1, 2, 4, 8, 16, 32, 64, 128] +DEFAULT_ERROR_CONTEXT = "Unknown error" + +logger = logging.getLogger(__name__) + + +class ResultSetQueue(ABC): + @abstractmethod + def next_n_rows(self, num_rows: int): + pass + + @abstractmethod + def remaining_rows(self): + pass + + +class ResultSetQueueFactory(ABC): + @staticmethod + def build_queue( + row_set_type: TSparkRowSetType, + t_row_set: TRowSet, + arrow_schema_bytes: bytes, + max_download_threads: int, + ssl_options: SSLOptions, + lz4_compressed: bool = True, + description: Optional[List[List[Any]]] = None, + ) -> ResultSetQueue: + """ + Factory method to build a result set queue. + + Args: + row_set_type (enum): Row set type (Arrow, Column, or URL). + t_row_set (TRowSet): Result containing arrow batches, columns, or cloud fetch links. + arrow_schema_bytes (bytes): Bytes representing the arrow schema. + lz4_compressed (bool): Whether result data has been lz4 compressed. + description (List[List[Any]]): Hive table schema description. + max_download_threads (int): Maximum number of downloader thread pool threads. + ssl_options (SSLOptions): SSLOptions object for CloudFetchQueue + + Returns: + ResultSetQueue + """ + if row_set_type == TSparkRowSetType.ARROW_BASED_SET: + arrow_table, n_valid_rows = convert_arrow_based_set_to_arrow_table( + t_row_set.arrowBatches, lz4_compressed, arrow_schema_bytes + ) + converted_arrow_table = convert_decimals_in_arrow_table( + arrow_table, description + ) + return ArrowQueue(converted_arrow_table, n_valid_rows) + elif row_set_type == TSparkRowSetType.COLUMN_BASED_SET: + column_table, column_names = convert_column_based_set_to_column_table( + t_row_set.columns, description + ) + + converted_column_table = convert_to_assigned_datatypes_in_column_table( + column_table, description + ) + + return ColumnQueue(ColumnTable(converted_column_table, column_names)) + elif row_set_type == TSparkRowSetType.URL_BASED_SET: + return CloudFetchQueue( + schema_bytes=arrow_schema_bytes, + start_row_offset=t_row_set.startRowOffset, + result_links=t_row_set.resultLinks, + lz4_compressed=lz4_compressed, + description=description, + max_download_threads=max_download_threads, + ssl_options=ssl_options, + ) + else: + raise AssertionError("Row set type is not valid") + +class ColumnTable: + def __init__(self, column_table, column_names): + self.column_table = column_table + self.column_names = column_names + + @property + def num_rows(self): + if len(self.column_table) == 0: + return 0 + else: + return len(self.column_table[0]) + + @property + def num_columns(self): + return len(self.column_names) + + def get_item(self, col_index, row_index): + return self.column_table[col_index][row_index] + + def slice(self, curr_index, length): + sliced_column_table = [column[curr_index : curr_index + length] for column in self.column_table] + return ColumnTable(sliced_column_table, self.column_names) + + def __eq__(self, other): + return self.column_table == other.column_table and self.column_names == other.column_names + +class ColumnQueue(ResultSetQueue): + def __init__(self, column_table: ColumnTable): + self.column_table = column_table + self.cur_row_index = 0 + self.n_valid_rows = column_table.num_rows + + def next_n_rows(self, num_rows): + length = min(num_rows, self.n_valid_rows - self.cur_row_index) + + slice = self.column_table.slice(self.cur_row_index, length) + self.cur_row_index += slice.num_rows + return slice -from databricks.sql import exc + def remaining_rows(self): + slice = self.column_table.slice(self.cur_row_index, self.n_valid_rows - self.cur_row_index) + self.cur_row_index += slice.num_rows + return slice -class ArrowQueue: +class ArrowQueue(ResultSetQueue): def __init__( - self, arrow_table: pyarrow.Table, n_valid_rows: int, start_row_index: int = 0 + self, + arrow_table: "pyarrow.Table", + n_valid_rows: int, + start_row_index: int = 0, ): """ A queue-like wrapper over an Arrow table @@ -23,7 +166,7 @@ def __init__( self.arrow_table = arrow_table self.n_valid_rows = n_valid_rows - def next_n_rows(self, num_rows: int) -> pyarrow.Table: + def next_n_rows(self, num_rows: int) -> "pyarrow.Table": """Get upto the next n rows of the Arrow dataframe""" length = min(num_rows, self.n_valid_rows - self.cur_row_index) # Note that the table.slice API is not the same as Python's slice @@ -32,7 +175,7 @@ def next_n_rows(self, num_rows: int) -> pyarrow.Table: self.cur_row_index += slice.num_rows return slice - def remaining_rows(self) -> pyarrow.Table: + def remaining_rows(self) -> "pyarrow.Table": slice = self.arrow_table.slice( self.cur_row_index, self.n_valid_rows - self.cur_row_index ) @@ -40,6 +183,155 @@ def remaining_rows(self) -> pyarrow.Table: return slice +class CloudFetchQueue(ResultSetQueue): + def __init__( + self, + schema_bytes, + max_download_threads: int, + ssl_options: SSLOptions, + start_row_offset: int = 0, + result_links: Optional[List[TSparkArrowResultLink]] = None, + lz4_compressed: bool = True, + description: Optional[List[List[Any]]] = None, + ): + """ + A queue-like wrapper over CloudFetch arrow batches. + + Attributes: + schema_bytes (bytes): Table schema in bytes. + max_download_threads (int): Maximum number of downloader thread pool threads. + start_row_offset (int): The offset of the first row of the cloud fetch links. + result_links (List[TSparkArrowResultLink]): Links containing the downloadable URL and metadata. + lz4_compressed (bool): Whether the files are lz4 compressed. + description (List[List[Any]]): Hive table schema description. + """ + self.schema_bytes = schema_bytes + self.max_download_threads = max_download_threads + self.start_row_index = start_row_offset + self.result_links = result_links + self.lz4_compressed = lz4_compressed + self.description = description + self._ssl_options = ssl_options + + logger.debug( + "Initialize CloudFetch loader, row set start offset: {}, file list:".format( + start_row_offset + ) + ) + if result_links is not None: + for result_link in result_links: + logger.debug( + "- start row offset: {}, row count: {}".format( + result_link.startRowOffset, result_link.rowCount + ) + ) + self.download_manager = ResultFileDownloadManager( + links=result_links or [], + max_download_threads=self.max_download_threads, + lz4_compressed=self.lz4_compressed, + ssl_options=self._ssl_options, + ) + + self.table = self._create_next_table() + self.table_row_index = 0 + + def next_n_rows(self, num_rows: int) -> "pyarrow.Table": + """ + Get up to the next n rows of the cloud fetch Arrow dataframes. + + Args: + num_rows (int): Number of rows to retrieve. + + Returns: + pyarrow.Table + """ + if not self.table: + logger.debug("CloudFetchQueue: no more rows available") + # Return empty pyarrow table to cause retry of fetch + return self._create_empty_table() + logger.debug("CloudFetchQueue: trying to get {} next rows".format(num_rows)) + results = self.table.slice(0, 0) + while num_rows > 0 and self.table: + # Get remaining of num_rows or the rest of the current table, whichever is smaller + length = min(num_rows, self.table.num_rows - self.table_row_index) + table_slice = self.table.slice(self.table_row_index, length) + results = pyarrow.concat_tables([results, table_slice]) + self.table_row_index += table_slice.num_rows + + # Replace current table with the next table if we are at the end of the current table + if self.table_row_index == self.table.num_rows: + self.table = self._create_next_table() + self.table_row_index = 0 + num_rows -= table_slice.num_rows + + logger.debug("CloudFetchQueue: collected {} next rows".format(results.num_rows)) + return results + + def remaining_rows(self) -> "pyarrow.Table": + """ + Get all remaining rows of the cloud fetch Arrow dataframes. + + Returns: + pyarrow.Table + """ + if not self.table: + # Return empty pyarrow table to cause retry of fetch + return self._create_empty_table() + results = self.table.slice(0, 0) + while self.table: + table_slice = self.table.slice( + self.table_row_index, self.table.num_rows - self.table_row_index + ) + results = pyarrow.concat_tables([results, table_slice]) + self.table_row_index += table_slice.num_rows + self.table = self._create_next_table() + self.table_row_index = 0 + return results + + def _create_next_table(self) -> Union["pyarrow.Table", None]: + logger.debug( + "CloudFetchQueue: Trying to get downloaded file for row {}".format( + self.start_row_index + ) + ) + # Create next table by retrieving the logical next downloaded file, or return None to signal end of queue + downloaded_file = self.download_manager.get_next_downloaded_file( + self.start_row_index + ) + if not downloaded_file: + logger.debug( + "CloudFetchQueue: Cannot find downloaded file for row {}".format( + self.start_row_index + ) + ) + # None signals no more Arrow tables can be built from the remaining handlers if any remain + return None + arrow_table = create_arrow_table_from_arrow_file( + downloaded_file.file_bytes, self.description + ) + + # The server rarely prepares the exact number of rows requested by the client in cloud fetch. + # Subsequently, we drop the extraneous rows in the last file if more rows are retrieved than requested + if arrow_table.num_rows > downloaded_file.row_count: + arrow_table = arrow_table.slice(0, downloaded_file.row_count) + + # At this point, whether the file has extraneous rows or not, the arrow table should have the correct num rows + assert downloaded_file.row_count == arrow_table.num_rows + self.start_row_index += arrow_table.num_rows + + logger.debug( + "CloudFetchQueue: Found downloaded file, row count: {}, new start offset: {}".format( + arrow_table.num_rows, self.start_row_index + ) + ) + + return arrow_table + + def _create_empty_table(self) -> "pyarrow.Table": + # Create a 0-row table with just the schema bytes + return create_arrow_table_from_arrow_file(self.schema_bytes, self.description) + + ExecuteResponse = namedtuple( "ExecuteResponse", "status has_been_closed_server_side has_more_rows description lz4_compressed is_staging_operation " @@ -116,7 +408,12 @@ def user_friendly_error_message(self, no_retry_reason, attempt, elapsed): user_friendly_error_message = "{}: {}".format( user_friendly_error_message, self.error_message ) - return user_friendly_error_message + try: + error_context = str(self.error) + except: + error_context = DEFAULT_ERROR_CONTEXT + + return user_friendly_error_message + ". " + error_context # Taken from PyHive @@ -183,3 +480,258 @@ def escape_item(self, item): def inject_parameters(operation: str, parameters: Dict[str, str]): return operation % parameters + + +def _dbsqlparameter_names(params: List[TDbsqlParameter]) -> list[str]: + return [p.name if p.name else "" for p in params] + + +def _generate_named_interpolation_values( + params: List[TDbsqlParameter], +) -> dict[str, str]: + """Returns a dictionary of the form {name: ":name"} for each parameter in params""" + + names = _dbsqlparameter_names(params) + + return {name: f":{name}" for name in names} + + +def _may_contain_inline_positional_markers(operation: str) -> bool: + """Check for the presence of `%s` in the operation string.""" + + interpolated = operation.replace("%s", "?") + return interpolated != operation + + +def _interpolate_named_markers( + operation: str, parameters: List[TDbsqlParameter] +) -> str: + """Replace all instances of `%(param)s` in `operation` with `:param`. + + If `operation` contains no instances of `%(param)s` then the input string is returned unchanged. + + ``` + "SELECT * FROM table WHERE field = %(field)s and other_field = %(other_field)s" + ``` + + Yields + + ``` + SELECT * FROM table WHERE field = :field and other_field = :other_field + ``` + """ + + _output_operation = operation + + PYFORMAT_PARAMSTYLE_REGEX = r"%\((\w+)\)s" + pat = re.compile(PYFORMAT_PARAMSTYLE_REGEX) + NAMED_PARAMSTYLE_FMT = ":{}" + PYFORMAT_PARAMSTYLE_FMT = "%({})s" + + pyformat_markers = pat.findall(operation) + for marker in pyformat_markers: + pyformat_marker = PYFORMAT_PARAMSTYLE_FMT.format(marker) + named_marker = NAMED_PARAMSTYLE_FMT.format(marker) + _output_operation = _output_operation.replace(pyformat_marker, named_marker) + + return _output_operation + + +def transform_paramstyle( + operation: str, + parameters: List[TDbsqlParameter], + param_structure: ParameterStructure, +) -> str: + """ + Performs a Python string interpolation such that any occurence of `%(param)s` will be replaced with `:param` + + This utility function is built to assist users in the transition between the default paramstyle in + this connector prior to version 3.0.0 (`pyformat`) and the new default paramstyle (`named`). + + Args: + operation: The operation or SQL text to transform. + parameters: The parameters to use for the transformation. + + Returns: + str + """ + output = operation + if ( + param_structure == ParameterStructure.POSITIONAL + and _may_contain_inline_positional_markers(operation) + ): + logger.warning( + "It looks like this query may contain un-named query markers like `%s`" + " This format is not supported when use_inline_params=False." + " Use `?` instead or set use_inline_params=True" + ) + elif param_structure == ParameterStructure.NAMED: + output = _interpolate_named_markers(operation, parameters) + + return output + + +def create_arrow_table_from_arrow_file(file_bytes: bytes, description) -> "pyarrow.Table": + arrow_table = convert_arrow_based_file_to_arrow_table(file_bytes) + return convert_decimals_in_arrow_table(arrow_table, description) + + +def convert_arrow_based_file_to_arrow_table(file_bytes: bytes): + try: + return pyarrow.ipc.open_stream(file_bytes).read_all() + except Exception as e: + raise RuntimeError("Failure to convert arrow based file to arrow table", e) + + +def convert_arrow_based_set_to_arrow_table(arrow_batches, lz4_compressed, schema_bytes): + ba = bytearray() + ba += schema_bytes + n_rows = 0 + for arrow_batch in arrow_batches: + n_rows += arrow_batch.rowCount + ba += ( + lz4.frame.decompress(arrow_batch.batch) + if lz4_compressed + else arrow_batch.batch + ) + arrow_table = pyarrow.ipc.open_stream(ba).read_all() + return arrow_table, n_rows + + +def convert_decimals_in_arrow_table(table, description) -> "pyarrow.Table": + for i, col in enumerate(table.itercolumns()): + if description[i][1] == "decimal": + decimal_col = col.to_pandas().apply( + lambda v: v if v is None else Decimal(v) + ) + precision, scale = description[i][4], description[i][5] + assert scale is not None + assert precision is not None + # Spark limits decimal to a maximum scale of 38, + # so 128 is guaranteed to be big enough + dtype = pyarrow.decimal128(precision, scale) + col_data = pyarrow.array(decimal_col, type=dtype) + field = table.field(i).with_type(dtype) + table = table.set_column(i, field, col_data) + return table + + +def convert_to_assigned_datatypes_in_column_table(column_table, description): + + converted_column_table = [] + for i, col in enumerate(column_table): + if description[i][1] == "decimal": + converted_column_table.append(tuple(v if v is None else Decimal(v) for v in col)) + elif description[i][1] == "date": + converted_column_table.append(tuple( + v if v is None else datetime.date.fromisoformat(v) for v in col + )) + elif description[i][1] == "timestamp": + converted_column_table.append(tuple( + ( + v + if v is None + else datetime.datetime.strptime(v, "%Y-%m-%d %H:%M:%S.%f").replace( + tzinfo=pytz.UTC + ) + ) + for v in col + )) + else: + converted_column_table.append(col) + + return converted_column_table + + +def convert_column_based_set_to_arrow_table(columns, description): + arrow_table = pyarrow.Table.from_arrays( + [_convert_column_to_arrow_array(c) for c in columns], + # Only use the column names from the schema, the types are determined by the + # physical types used in column based set, as they can differ from the + # mapping used in _hive_schema_to_arrow_schema. + names=[c[0] for c in description], + ) + return arrow_table, arrow_table.num_rows + + +def convert_column_based_set_to_column_table(columns, description): + column_names = [c[0] for c in description] + column_table = [_convert_column_to_list(c) for c in columns] + + return column_table, column_names + + +def _convert_column_to_arrow_array(t_col): + """ + Return a pyarrow array from the values in a TColumn instance. + Note that ColumnBasedSet has no native support for complex types, so they will be converted + to strings server-side. + """ + field_name_to_arrow_type = { + "boolVal": pyarrow.bool_(), + "byteVal": pyarrow.int8(), + "i16Val": pyarrow.int16(), + "i32Val": pyarrow.int32(), + "i64Val": pyarrow.int64(), + "doubleVal": pyarrow.float64(), + "stringVal": pyarrow.string(), + "binaryVal": pyarrow.binary(), + } + for field in field_name_to_arrow_type.keys(): + wrapper = getattr(t_col, field) + if wrapper: + return _create_arrow_array(wrapper, field_name_to_arrow_type[field]) + + raise OperationalError("Empty TColumn instance {}".format(t_col)) + + +def _convert_column_to_list(t_col): + SUPPORTED_FIELD_TYPES = ( + "boolVal", + "byteVal", + "i16Val", + "i32Val", + "i64Val", + "doubleVal", + "stringVal", + "binaryVal", + ) + + for field in SUPPORTED_FIELD_TYPES: + wrapper = getattr(t_col, field) + if wrapper: + return _create_python_tuple(wrapper) + + raise OperationalError("Empty TColumn instance {}".format(t_col)) + + +def _create_arrow_array(t_col_value_wrapper, arrow_type): + result = t_col_value_wrapper.values + nulls = t_col_value_wrapper.nulls # bitfield describing which values are null + assert isinstance(nulls, bytes) + + # The number of bits in nulls can be both larger or smaller than the number of + # elements in result, so take the minimum of both to iterate over. + length = min(len(result), len(nulls) * 8) + + for i in range(length): + if nulls[i >> 3] & BIT_MASKS[i & 0x7]: + result[i] = None + + return pyarrow.array(result, type=arrow_type) + + +def _create_python_tuple(t_col_value_wrapper): + result = t_col_value_wrapper.values + nulls = t_col_value_wrapper.nulls # bitfield describing which values are null + assert isinstance(nulls, bytes) + + # The number of bits in nulls can be both larger or smaller than the number of + # elements in result, so take the minimum of both to iterate over. + length = min(len(result), len(nulls) * 8) + + for i in range(length): + if nulls[i >> 3] & BIT_MASKS[i & 0x7]: + result[i] = None + + return tuple(result) \ No newline at end of file diff --git a/src/databricks/sqlalchemy/README.sqlalchemy.md b/src/databricks/sqlalchemy/README.sqlalchemy.md new file mode 100644 index 00000000..8aa51973 --- /dev/null +++ b/src/databricks/sqlalchemy/README.sqlalchemy.md @@ -0,0 +1,203 @@ +## Databricks dialect for SQLALchemy 2.0 + +The Databricks dialect for SQLAlchemy serves as bridge between [SQLAlchemy](https://www.sqlalchemy.org/) and the Databricks SQL Python driver. The dialect is included with `databricks-sql-connector==3.0.0` and above. A working example demonstrating usage can be found in `examples/sqlalchemy.py`. + +## Usage with SQLAlchemy <= 2.0 +A SQLAlchemy 1.4 compatible dialect was first released in connector [version 2.4](https://github.com/databricks/databricks-sql-python/releases/tag/v2.4.0). Support for SQLAlchemy 1.4 was dropped from the dialect as part of `databricks-sql-connector==3.0.0`. To continue using the dialect with SQLAlchemy 1.x, you can use `databricks-sql-connector^2.4.0`. + + +## Installation + +To install the dialect and its dependencies: + +```shell +pip install databricks-sql-connector[sqlalchemy] +``` + +If you also plan to use `alembic` you can alternatively run: + +```shell +pip install databricks-sql-connector[alembic] +``` + +## Connection String + +Every SQLAlchemy application that connects to a database needs to use an [Engine](https://docs.sqlalchemy.org/en/20/tutorial/engine.html#tutorial-engine), which you can create by passing a connection string to `create_engine`. The connection string must include these components: + +1. Host +2. HTTP Path for a compute resource +3. API access token +4. Initial catalog for the connection +5. Initial schema for the connection + +**Note: Our dialect is built and tested on workspaces with Unity Catalog enabled. Support for the `hive_metastore` catalog is untested.** + +For example: + +```python +import os +from sqlalchemy import create_engine + +host = os.getenv("DATABRICKS_SERVER_HOSTNAME") +http_path = os.getenv("DATABRICKS_HTTP_PATH") +access_token = os.getenv("DATABRICKS_TOKEN") +catalog = os.getenv("DATABRICKS_CATALOG") +schema = os.getenv("DATABRICKS_SCHEMA") + +engine = create_engine( + f"databricks://token:{access_token}@{host}?http_path={http_path}&catalog={catalog}&schema={schema}" + ) +``` + +## Types + +The [SQLAlchemy type hierarchy](https://docs.sqlalchemy.org/en/20/core/type_basics.html) contains backend-agnostic type implementations (represented in CamelCase) and backend-specific types (represented in UPPERCASE). The majority of SQLAlchemy's [CamelCase](https://docs.sqlalchemy.org/en/20/core/type_basics.html#the-camelcase-datatypes) types are supported. This means that a SQLAlchemy application using these types should "just work" with Databricks. + +|SQLAlchemy Type|Databricks SQL Type| +|-|-| +[`BigInteger`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.BigInteger)| [`BIGINT`](https://docs.databricks.com/en/sql/language-manual/data-types/bigint-type.html) +[`LargeBinary`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.LargeBinary)| (not supported)| +[`Boolean`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Boolean)| [`BOOLEAN`](https://docs.databricks.com/en/sql/language-manual/data-types/boolean-type.html) +[`Date`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Date)| [`DATE`](https://docs.databricks.com/en/sql/language-manual/data-types/date-type.html) +[`DateTime`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.DateTime)| [`TIMESTAMP_NTZ`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html)| +[`Double`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Double)| [`DOUBLE`](https://docs.databricks.com/en/sql/language-manual/data-types/double-type.html) +[`Enum`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Enum)| (not supported)| +[`Float`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Float)| [`FLOAT`](https://docs.databricks.com/en/sql/language-manual/data-types/float-type.html) +[`Integer`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Integer)| [`INT`](https://docs.databricks.com/en/sql/language-manual/data-types/int-type.html) +[`Numeric`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Numeric)| [`DECIMAL`](https://docs.databricks.com/en/sql/language-manual/data-types/decimal-type.html)| +[`PickleType`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.PickleType)| (not supported)| +[`SmallInteger`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.SmallInteger)| [`SMALLINT`](https://docs.databricks.com/en/sql/language-manual/data-types/smallint-type.html) +[`String`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.String)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| +[`Text`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Text)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| +[`Time`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Time)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| +[`Unicode`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Unicode)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| +[`UnicodeText`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.UnicodeText)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html)| +[`Uuid`](https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.Uuid)| [`STRING`](https://docs.databricks.com/en/sql/language-manual/data-types/string-type.html) + +In addition, the dialect exposes three UPPERCASE SQLAlchemy types which are specific to Databricks: + +- [`databricks.sqlalchemy.TINYINT`](https://docs.databricks.com/en/sql/language-manual/data-types/tinyint-type.html) +- [`databricks.sqlalchemy.TIMESTAMP`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-type.html) +- [`databricks.sqlalchemy.TIMESTAMP_NTZ`](https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html) + + +### `LargeBinary()` and `PickleType()` + +Databricks Runtime doesn't currently support binding of binary values in SQL queries, which is a pre-requisite for this functionality in SQLAlchemy. + +## `Enum()` and `CHECK` constraints + +Support for `CHECK` constraints is not implemented in this dialect. Support is planned for a future release. + +SQLAlchemy's `Enum()` type depends on `CHECK` constraints and is therefore not yet supported. + +### `DateTime()`, `TIMESTAMP_NTZ()`, and `TIMESTAMP()` + +Databricks Runtime provides two datetime-like types: `TIMESTAMP` which is always timezone-aware and `TIMESTAMP_NTZ` which is timezone agnostic. Both types can be imported from `databricks.sqlalchemy` and used in your models. + +The SQLAlchemy documentation indicates that `DateTime()` is not timezone-aware by default. So our dialect maps this type to `TIMESTAMP_NTZ()`. In practice, you should never need to use `TIMESTAMP_NTZ()` directly. Just use `DateTime()`. + +If you need your field to be timezone-aware, you can import `TIMESTAMP()` and use it instead. + +_Note that SQLAlchemy documentation suggests that you can declare a `DateTime()` with `timezone=True` on supported backends. However, if you do this with the Databricks dialect, the `timezone` argument will be ignored._ + +```python +from sqlalchemy import DateTime +from databricks.sqlalchemy import TIMESTAMP + +class SomeModel(Base): + some_date_without_timezone = DateTime() + some_date_with_timezone = TIMESTAMP() +``` + +### `String()`, `Text()`, `Unicode()`, and `UnicodeText()` + +Databricks Runtime doesn't support length limitations for `STRING` fields. Therefore `String()` or `String(1)` or `String(255)` will all produce identical DDL. Since `Text()`, `Unicode()`, `UnicodeText()` all use the same underlying type in Databricks SQL, they will generate equivalent DDL. + +### `Time()` + +Databricks Runtime doesn't have a native time-like data type. To implement this type in SQLAlchemy, our dialect stores SQLAlchemy `Time()` values in a `STRING` field. Unlike `DateTime` above, this type can optionally support timezone awareness (since the dialect is in complete control of the strings that we write to the Delta table). + +```python +from sqlalchemy import Time + +class SomeModel(Base): + time_tz = Time(timezone=True) + time_ntz = Time() +``` + + +# Usage Notes + +## `Identity()` and `autoincrement` + +Identity and generated value support is currently limited in this dialect. + +When defining models, SQLAlchemy types can accept an [`autoincrement`](https://docs.sqlalchemy.org/en/20/core/metadata.html#sqlalchemy.schema.Column.params.autoincrement) argument. In our dialect, this argument is currently ignored. To create an auto-incrementing field in your model you can pass in an explicit [`Identity()`](https://docs.sqlalchemy.org/en/20/core/defaults.html#identity-ddl) instead. + +Furthermore, in Databricks Runtime, only `BIGINT` fields can be configured to auto-increment. So in SQLAlchemy, you must use the `BigInteger()` type. + +```python +from sqlalchemy import Identity, String + +class SomeModel(Base): + id = BigInteger(Identity()) + value = String() +``` + +When calling `Base.metadata.create_all()`, the executed DDL will include `GENERATED ALWAYS AS IDENTITY` for the `id` column. This is useful when using SQLAlchemy to generate tables. However, as of this writing, `Identity()` constructs are not captured when SQLAlchemy reflects a table's metadata (support for this is planned). + +## Parameters + +`databricks-sql-connector` supports two approaches to parameterizing SQL queries: native and inline. Our SQLAlchemy 2.0 dialect always uses the native approach and is therefore limited to DBR 14.2 and above. If you are writing parameterized queries to be executed by SQLAlchemy, you must use the "named" paramstyle (`:param`). Read more about parameterization in `docs/parameters.md`. + +## Usage with pandas + +Use [`pandas.DataFrame.to_sql`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.DataFrame.to_sql.html) and [`pandas.read_sql`](https://pandas.pydata.org/pandas-docs/stable/reference/api/pandas.read_sql.html#pandas.read_sql) to write and read from Databricks SQL. These methods both accept a SQLAlchemy connection to interact with Databricks. + +### Read from Databricks SQL into pandas +```python +from sqlalchemy import create_engine +import pandas as pd + +engine = create_engine("databricks://token:dapi***@***.cloud.databricks.com?http_path=***&catalog=main&schema=test") +with engine.connect() as conn: + # This will read the contents of `main.test.some_table` + df = pd.read_sql("some_table", conn) +``` + +### Write to Databricks SQL from pandas + +```python +from sqlalchemy import create_engine +import pandas as pd + +engine = create_engine("databricks://token:dapi***@***.cloud.databricks.com?http_path=***&catalog=main&schema=test") +squares = [(i, i * i) for i in range(100)] +df = pd.DataFrame(data=squares,columns=['x','x_squared']) + +with engine.connect() as conn: + # This will write the contents of `df` to `main.test.squares` + df.to_sql('squares',conn) +``` + +## [`PrimaryKey()`](https://docs.sqlalchemy.org/en/20/core/constraints.html#sqlalchemy.schema.PrimaryKeyConstraint) and [`ForeignKey()`](https://docs.sqlalchemy.org/en/20/core/constraints.html#defining-foreign-keys) + +Unity Catalog workspaces in Databricks support PRIMARY KEY and FOREIGN KEY constraints. _Note that Databricks Runtime does not enforce the integrity of FOREIGN KEY constraints_. You can establish a primary key by setting `primary_key=True` when defining a column. + +When building `ForeignKey` or `ForeignKeyConstraint` objects, you must specify a `name` for the constraint. + +If your model definition requires a self-referential FOREIGN KEY constraint, you must include `use_alter=True` when defining the relationship. + +```python +from sqlalchemy import Table, Column, ForeignKey, BigInteger, String + +users = Table( + "users", + metadata_obj, + Column("id", BigInteger, primary_key=True), + Column("name", String(), nullable=False), + Column("email", String()), + Column("manager_id", ForeignKey("users.id", name="fk_users_manager_id_x_users_id", use_alter=True)) +) +``` diff --git a/src/databricks/sqlalchemy/README.tests.md b/src/databricks/sqlalchemy/README.tests.md new file mode 100644 index 00000000..3ed92aba --- /dev/null +++ b/src/databricks/sqlalchemy/README.tests.md @@ -0,0 +1,44 @@ +## SQLAlchemy Dialect Compliance Test Suite with Databricks + +The contents of the `test/` directory follow the SQLAlchemy developers' [guidance] for running the reusable dialect compliance test suite. Since not every test in the suite is applicable to every dialect, two options are provided to skip tests: + +- Any test can be skipped by subclassing its parent class, re-declaring the test-case and adding a `pytest.mark.skip` directive. +- Any test that is decorated with a `@requires` decorator can be skipped by marking the indicated requirement as `.closed()` in `requirements.py` + +We prefer to skip test cases directly with the first method wherever possible. We only mark requirements as `closed()` if there is no easier option to avoid a test failure. This principally occurs in test cases where the same test in the suite is parametrized, and some parameter combinations are conditionally skipped depending on `requirements.py`. If we skip the entire test method, then we skip _all_ permutations, not just the combinations we don't support. + +## Regression, Unsupported, and Future test cases + +We maintain three files of test cases that we import from the SQLAlchemy source code: + +* **`_regression.py`** contains all the tests cases with tests that we expect to pass for our dialect. Each one is marked with `pytest.mark.reiewed` to indicate that we've evaluated it for relevance. This file only contains base class declarations. +* **`_unsupported.py`** contains test cases that fail because of missing features in Databricks. We mark them as skipped with a `SkipReason` enumeration. If Databricks comes to support these features, those test or entire classes can be moved to `_regression.py`. +* **`_future.py`** contains test cases that fail because of missing features in the dialect itself, but which _are_ supported by Databricks generally. We mark them as skipped with a `FutureFeature` enumeration. These are features that have not been prioritised or that do not violate our acceptance criteria. All of these test cases will eventually move to either `_regression.py`. + +In some cases, only certain tests in class should be skipped with a `SkipReason` or `FutureFeature` justification. In those cases, we import the class into `_regression.py`, then import it from there into one or both of `_future.py` and `_unsupported.py`. If a class needs to be "touched" by regression, unsupported, and future, the class will be imported in that order. If an entire class should be skipped, then we do not import it into `_regression.py` at all. + +We maintain `_extra.py` with test cases that depend on SQLAlchemy's reusable dialect test fixtures but which are specific to Databricks (e.g TinyIntegerTest). + +## Running the reusable dialect tests + +``` +poetry shell +cd src/databricks/sqlalchemy/test +python -m pytest test_suite.py --dburi \ + "databricks://token:$access_token@$host?http_path=$http_path&catalog=$catalog&schema=$schema" +``` + +Whatever schema you pass in the `dburi` argument should be empty. Some tests also require the presence of an empty schema named `test_schema`. Note that we plan to implement our own `provision.py` which SQLAlchemy can automatically use to create an empty schema for testing. But for now this is a manual process. + +You can run only reviewed tests by appending `-m "reviewed"` to the test runner invocation. + +You can run only the unreviewed tests by appending `-m "not reviewed"` instead. + +Note that because these tests depend on SQLAlchemy's custom pytest plugin, they are not discoverable by IDE-based test runners like VSCode or PyCharm and must be invoked from a CLI. + +## Running local unit and e2e tests + +Apart from the SQLAlchemy reusable suite, we maintain our own unit and e2e tests under the `test_local/` directory. These can be invoked from a VSCode or Pycharm since they don't depend on a custom pytest plugin. Due to pytest's lookup order, the `pytest.ini` which is required for running the reusable dialect tests, also conflicts with VSCode and Pycharm's default pytest implementation and overrides the settings in `pyproject.toml`. So to run these tests, you can delete or rename `pytest.ini`. + + +[guidance]: "https://github.com/sqlalchemy/sqlalchemy/blob/rel_2_0_22/README.dialects.rst" diff --git a/src/databricks/sqlalchemy/__init__.py b/src/databricks/sqlalchemy/__init__.py index 1df1e1d4..2a17ac3e 100644 --- a/src/databricks/sqlalchemy/__init__.py +++ b/src/databricks/sqlalchemy/__init__.py @@ -1 +1,4 @@ -from databricks.sqlalchemy.dialect import DatabricksDialect +from databricks.sqlalchemy.base import DatabricksDialect +from databricks.sqlalchemy._types import TINYINT, TIMESTAMP, TIMESTAMP_NTZ + +__all__ = ["TINYINT", "TIMESTAMP", "TIMESTAMP_NTZ"] diff --git a/src/databricks/sqlalchemy/_ddl.py b/src/databricks/sqlalchemy/_ddl.py new file mode 100644 index 00000000..d5d0bf87 --- /dev/null +++ b/src/databricks/sqlalchemy/_ddl.py @@ -0,0 +1,100 @@ +import re +from sqlalchemy.sql import compiler, sqltypes +import logging + +logger = logging.getLogger(__name__) + + +class DatabricksIdentifierPreparer(compiler.IdentifierPreparer): + """https://docs.databricks.com/en/sql/language-manual/sql-ref-identifiers.html""" + + legal_characters = re.compile(r"^[A-Z0-9_]+$", re.I) + + def __init__(self, dialect): + super().__init__(dialect, initial_quote="`") + + +class DatabricksDDLCompiler(compiler.DDLCompiler): + def post_create_table(self, table): + post = [" USING DELTA"] + if table.comment: + comment = self.sql_compiler.render_literal_value( + table.comment, sqltypes.String() + ) + post.append("COMMENT " + comment) + + post.append("TBLPROPERTIES('delta.feature.allowColumnDefaults' = 'enabled')") + return "\n".join(post) + + def visit_unique_constraint(self, constraint, **kw): + logger.warning("Databricks does not support unique constraints") + pass + + def visit_check_constraint(self, constraint, **kw): + logger.warning("This dialect does not support check constraints") + pass + + def visit_identity_column(self, identity, **kw): + """When configuring an Identity() with Databricks, only the always option is supported. + All other options are ignored. + + Note: IDENTITY columns must always be defined as BIGINT. An exception will be raised if INT is used. + + https://www.databricks.com/blog/2022/08/08/identity-columns-to-generate-surrogate-keys-are-now-available-in-a-lakehouse-near-you.html + """ + text = "GENERATED %s AS IDENTITY" % ( + "ALWAYS" if identity.always else "BY DEFAULT", + ) + return text + + def visit_set_column_comment(self, create, **kw): + return "ALTER TABLE %s ALTER COLUMN %s COMMENT %s" % ( + self.preparer.format_table(create.element.table), + self.preparer.format_column(create.element), + self.sql_compiler.render_literal_value( + create.element.comment, sqltypes.String() + ), + ) + + def visit_drop_column_comment(self, create, **kw): + return "ALTER TABLE %s ALTER COLUMN %s COMMENT ''" % ( + self.preparer.format_table(create.element.table), + self.preparer.format_column(create.element), + ) + + def get_column_specification(self, column, **kwargs): + """ + Emit a log message if a user attempts to set autoincrement=True on a column. + See comments in test_suite.py. We may implement implicit IDENTITY using this + feature in the future, similar to the Microsoft SQL Server dialect. + """ + if column is column.table._autoincrement_column or column.autoincrement is True: + logger.warning( + "Databricks dialect ignores SQLAlchemy's autoincrement semantics. Use explicit Identity() instead." + ) + + colspec = super().get_column_specification(column, **kwargs) + if column.comment is not None: + literal = self.sql_compiler.render_literal_value( + column.comment, sqltypes.STRINGTYPE + ) + colspec += " COMMENT " + literal + + return colspec + + +class DatabricksStatementCompiler(compiler.SQLCompiler): + def limit_clause(self, select, **kw): + """Identical to the default implementation of SQLCompiler.limit_clause except it writes LIMIT ALL instead of LIMIT -1, + since Databricks SQL doesn't support the latter. + + https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-qry-select-limit.html + """ + text = "" + if select._limit_clause is not None: + text += "\n LIMIT " + self.process(select._limit_clause, **kw) + if select._offset_clause is not None: + if select._limit_clause is None: + text += "\n LIMIT ALL" + text += " OFFSET " + self.process(select._offset_clause, **kw) + return text diff --git a/src/databricks/sqlalchemy/_parse.py b/src/databricks/sqlalchemy/_parse.py new file mode 100644 index 00000000..6d38e1e6 --- /dev/null +++ b/src/databricks/sqlalchemy/_parse.py @@ -0,0 +1,385 @@ +from typing import List, Optional, Dict +import re + +import sqlalchemy +from sqlalchemy.engine import CursorResult +from sqlalchemy.engine.interfaces import ReflectedColumn + +from databricks.sqlalchemy import _types as type_overrides + +""" +This module contains helper functions that can parse the contents +of metadata and exceptions received from DBR. These are mostly just +wrappers around regexes. +""" + + +class DatabricksSqlAlchemyParseException(Exception): + pass + + +def _match_table_not_found_string(message: str) -> bool: + """Return True if the message contains a substring indicating that a table was not found""" + + DBR_LTE_12_NOT_FOUND_STRING = "Table or view not found" + DBR_GT_12_NOT_FOUND_STRING = "TABLE_OR_VIEW_NOT_FOUND" + return any( + [ + DBR_LTE_12_NOT_FOUND_STRING in message, + DBR_GT_12_NOT_FOUND_STRING in message, + ] + ) + + +def _describe_table_extended_result_to_dict_list( + result: CursorResult, +) -> List[Dict[str, str]]: + """Transform the CursorResult of DESCRIBE TABLE EXTENDED into a list of Dictionaries""" + + rows_to_return = [] + for row in result.all(): + this_row = {"col_name": row.col_name, "data_type": row.data_type} + rows_to_return.append(this_row) + + return rows_to_return + + +def extract_identifiers_from_string(input_str: str) -> List[str]: + """For a string input resembling (`a`, `b`, `c`) return a list of identifiers ['a', 'b', 'c']""" + + # This matches the valid character list contained in DatabricksIdentifierPreparer + pattern = re.compile(r"`([A-Za-z0-9_]+)`") + matches = pattern.findall(input_str) + return [i for i in matches] + + +def extract_identifier_groups_from_string(input_str: str) -> List[str]: + """For a string input resembling : + + FOREIGN KEY (`pname`, `pid`, `pattr`) REFERENCES `main`.`pysql_sqlalchemy`.`tb1` (`name`, `id`, `attr`) + + Return ['(`pname`, `pid`, `pattr`)', '(`name`, `id`, `attr`)'] + """ + pattern = re.compile(r"\([`A-Za-z0-9_,\s]*\)") + matches = pattern.findall(input_str) + return [i for i in matches] + + +def extract_three_level_identifier_from_constraint_string(input_str: str) -> dict: + """For a string input resembling : + FOREIGN KEY (`parent_user_id`) REFERENCES `main`.`pysql_dialect_compliance`.`users` (`user_id`) + + Return a dict like + { + "catalog": "main", + "schema": "pysql_dialect_compliance", + "table": "users" + } + + Raise a DatabricksSqlAlchemyParseException if a 3L namespace isn't found + """ + pat = re.compile(r"REFERENCES\s+(.*?)\s*\(") + matches = pat.findall(input_str) + + if not matches: + raise DatabricksSqlAlchemyParseException( + "3L namespace not found in constraint string" + ) + + first_match = matches[0] + parts = first_match.split(".") + + def strip_backticks(input: str): + return input.replace("`", "") + + try: + return { + "catalog": strip_backticks(parts[0]), + "schema": strip_backticks(parts[1]), + "table": strip_backticks(parts[2]), + } + except IndexError: + raise DatabricksSqlAlchemyParseException( + "Incomplete 3L namespace found in constraint string: " + ".".join(parts) + ) + + +def _parse_fk_from_constraint_string(constraint_str: str) -> dict: + """Build a dictionary of foreign key constraint information from a constraint string. + + For example: + + ``` + FOREIGN KEY (`pname`, `pid`, `pattr`) REFERENCES `main`.`pysql_dialect_compliance`.`tb1` (`name`, `id`, `attr`) + ``` + + Return a dictionary like: + + ``` + { + "constrained_columns": ["pname", "pid", "pattr"], + "referred_table": "tb1", + "referred_schema": "pysql_dialect_compliance", + "referred_columns": ["name", "id", "attr"] + } + ``` + + Note that the constraint name doesn't appear in the constraint string so it will not + be present in the output of this function. + """ + + referred_table_dict = extract_three_level_identifier_from_constraint_string( + constraint_str + ) + referred_table = referred_table_dict["table"] + referred_schema = referred_table_dict["schema"] + + # _extracted is a tuple of two lists of identifiers + # we assume the first immediately follows "FOREIGN KEY" and the second + # immediately follows REFERENCES $tableName + _extracted = extract_identifier_groups_from_string(constraint_str) + constrained_columns_str, referred_columns_str = ( + _extracted[0], + _extracted[1], + ) + + constrained_columns = extract_identifiers_from_string(constrained_columns_str) + referred_columns = extract_identifiers_from_string(referred_columns_str) + + return { + "constrained_columns": constrained_columns, + "referred_table": referred_table, + "referred_columns": referred_columns, + "referred_schema": referred_schema, + } + + +def build_fk_dict( + fk_name: str, fk_constraint_string: str, schema_name: Optional[str] +) -> dict: + """ + Given a foriegn key name and a foreign key constraint string, return a dictionary + with the following keys: + + name + the name of the foreign key constraint + constrained_columns + a list of column names that make up the foreign key + referred_table + the name of the table that the foreign key references + referred_columns + a list of column names that are referenced by the foreign key + referred_schema + the name of the schema that the foreign key references. + + referred schema will be None if the schema_name argument is None. + This is required by SQLAlchey's ComponentReflectionTest::test_get_foreign_keys + """ + + # The foreign key name is not contained in the constraint string so we + # need to add it manually + base_fk_dict = _parse_fk_from_constraint_string(fk_constraint_string) + + if not schema_name: + schema_override_dict = dict(referred_schema=None) + else: + schema_override_dict = {} + + # mypy doesn't like this method of conditionally adding a key to a dictionary + # while keeping everything immutable + complete_foreign_key_dict = { + "name": fk_name, + **base_fk_dict, + **schema_override_dict, # type: ignore + } + + return complete_foreign_key_dict + + +def _parse_pk_columns_from_constraint_string(constraint_str: str) -> List[str]: + """Build a list of constrained columns from a constraint string returned by DESCRIBE TABLE EXTENDED + + For example: + + PRIMARY KEY (`id`, `name`, `email_address`) + + Returns a list like + + ["id", "name", "email_address"] + """ + + _extracted = extract_identifiers_from_string(constraint_str) + + return _extracted + + +def build_pk_dict(pk_name: str, pk_constraint_string: str) -> dict: + """Given a primary key name and a primary key constraint string, return a dictionary + with the following keys: + + constrained_columns + A list of string column names that make up the primary key + + name + The name of the primary key constraint + """ + + constrained_columns = _parse_pk_columns_from_constraint_string(pk_constraint_string) + + return {"constrained_columns": constrained_columns, "name": pk_name} + + +def match_dte_rows_by_value(dte_output: List[Dict[str, str]], match: str) -> List[dict]: + """Return a list of dictionaries containing only the col_name:data_type pairs where the `data_type` + value contains the match argument. + + Today, DESCRIBE TABLE EXTENDED doesn't give a deterministic name to the fields + a constraint will be found in its output. So we cycle through its output looking + for a match. This is brittle. We could optionally make two roundtrips: the first + would query information_schema for the name of the constraint on this table, and + a second to DESCRIBE TABLE EXTENDED, at which point we would know the name of the + constraint. But for now we instead assume that Python list comprehension is faster + than a network roundtrip + """ + + output_rows = [] + + for row_dict in dte_output: + if match in row_dict["data_type"]: + output_rows.append(row_dict) + + return output_rows + + +def match_dte_rows_by_key(dte_output: List[Dict[str, str]], match: str) -> List[dict]: + """Return a list of dictionaries containing only the col_name:data_type pairs where the `col_name` + value contains the match argument. + """ + + output_rows = [] + + for row_dict in dte_output: + if match in row_dict["col_name"]: + output_rows.append(row_dict) + + return output_rows + + +def get_fk_strings_from_dte_output(dte_output: List[Dict[str, str]]) -> List[dict]: + """If the DESCRIBE TABLE EXTENDED output contains foreign key constraints, return a list of dictionaries, + one dictionary per defined constraint + """ + + output = match_dte_rows_by_value(dte_output, "FOREIGN KEY") + + return output + + +def get_pk_strings_from_dte_output( + dte_output: List[Dict[str, str]] +) -> Optional[List[dict]]: + """If the DESCRIBE TABLE EXTENDED output contains primary key constraints, return a list of dictionaries, + one dictionary per defined constraint. + + Returns None if no primary key constraints are found. + """ + + output = match_dte_rows_by_value(dte_output, "PRIMARY KEY") + + return output + + +def get_comment_from_dte_output(dte_output: List[Dict[str, str]]) -> Optional[str]: + """Returns the value of the first "Comment" col_name data in dte_output""" + output = match_dte_rows_by_key(dte_output, "Comment") + if not output: + return None + else: + return output[0]["data_type"] + + +# The keys of this dictionary are the values we expect to see in a +# TGetColumnsRequest's .TYPE_NAME attribute. +# These are enumerated in ttypes.py as class TTypeId. +# TODO: confirm that all types in TTypeId are included here. +GET_COLUMNS_TYPE_MAP = { + "boolean": sqlalchemy.types.Boolean, + "smallint": sqlalchemy.types.SmallInteger, + "tinyint": type_overrides.TINYINT, + "int": sqlalchemy.types.Integer, + "bigint": sqlalchemy.types.BigInteger, + "float": sqlalchemy.types.Float, + "double": sqlalchemy.types.Float, + "string": sqlalchemy.types.String, + "varchar": sqlalchemy.types.String, + "char": sqlalchemy.types.String, + "binary": sqlalchemy.types.String, + "array": sqlalchemy.types.String, + "map": sqlalchemy.types.String, + "struct": sqlalchemy.types.String, + "uniontype": sqlalchemy.types.String, + "decimal": sqlalchemy.types.Numeric, + "timestamp": type_overrides.TIMESTAMP, + "timestamp_ntz": type_overrides.TIMESTAMP_NTZ, + "date": sqlalchemy.types.Date, +} + + +def parse_numeric_type_precision_and_scale(type_name_str): + """Return an intantiated sqlalchemy Numeric() type that preserves the precision and scale indicated + in the output from TGetColumnsRequest. + + type_name_str + The value of TGetColumnsReq.TYPE_NAME. + + If type_name_str is "DECIMAL(18,5) returns sqlalchemy.types.Numeric(18,5) + """ + + pattern = re.compile(r"DECIMAL\((\d+,\d+)\)") + match = re.search(pattern, type_name_str) + precision_and_scale = match.group(1) + precision, scale = tuple(precision_and_scale.split(",")) + + return sqlalchemy.types.Numeric(int(precision), int(scale)) + + +def parse_column_info_from_tgetcolumnsresponse(thrift_resp_row) -> ReflectedColumn: + """Returns a dictionary of the ReflectedColumn schema parsed from + a single of the result of a TGetColumnsRequest thrift RPC + """ + + pat = re.compile(r"^\w+") + + # This method assumes a valid TYPE_NAME field in the response. + # TODO: add error handling in case TGetColumnsResponse format changes + + _raw_col_type = re.search(pat, thrift_resp_row.TYPE_NAME).group(0).lower() # type: ignore + _col_type = GET_COLUMNS_TYPE_MAP[_raw_col_type] + + if _raw_col_type == "decimal": + final_col_type = parse_numeric_type_precision_and_scale( + thrift_resp_row.TYPE_NAME + ) + else: + final_col_type = _col_type + + # See comments about autoincrement in test_suite.py + # Since Databricks SQL doesn't currently support inline AUTOINCREMENT declarations + # the autoincrement must be manually declared with an Identity() construct in SQLAlchemy + # Other dialects can perform this extra Identity() step automatically. But that is not + # implemented in the Databricks dialect right now. So autoincrement is currently always False. + # It's not clear what IS_AUTO_INCREMENT in the thrift response actually reflects or whether + # it ever returns a `YES`. + + # Per the guidance in SQLAlchemy's docstrings, we prefer to not even include an autoincrement + # key in this dictionary. + this_column = { + "name": thrift_resp_row.COLUMN_NAME, + "type": final_col_type, + "nullable": bool(thrift_resp_row.NULLABLE), + "default": thrift_resp_row.COLUMN_DEF, + "comment": thrift_resp_row.REMARKS or None, + } + + # TODO: figure out how to return sqlalchemy.interfaces in a way that mypy respects + return this_column # type: ignore diff --git a/src/databricks/sqlalchemy/_types.py b/src/databricks/sqlalchemy/_types.py new file mode 100644 index 00000000..5fc14a70 --- /dev/null +++ b/src/databricks/sqlalchemy/_types.py @@ -0,0 +1,323 @@ +from datetime import datetime, time, timezone +from itertools import product +from typing import Any, Union, Optional + +import sqlalchemy +from sqlalchemy.engine.interfaces import Dialect +from sqlalchemy.ext.compiler import compiles + +from databricks.sql.utils import ParamEscaper + + +def process_literal_param_hack(value: Any): + """This method is supposed to accept a Python type and return a string representation of that type. + But due to some weirdness in the way SQLAlchemy's literal rendering works, we have to return + the value itself because, by the time it reaches our custom type code, it's already been converted + into a string. + + TimeTest + DateTimeTest + DateTimeTZTest + + This dynamic only seems to affect the literal rendering of datetime and time objects. + + All fail without this hack in-place. I'm not sure why. But it works. + """ + return value + + +@compiles(sqlalchemy.types.Enum, "databricks") +@compiles(sqlalchemy.types.String, "databricks") +@compiles(sqlalchemy.types.Text, "databricks") +@compiles(sqlalchemy.types.Time, "databricks") +@compiles(sqlalchemy.types.Unicode, "databricks") +@compiles(sqlalchemy.types.UnicodeText, "databricks") +@compiles(sqlalchemy.types.Uuid, "databricks") +def compile_string_databricks(type_, compiler, **kw): + """ + We override the default compilation for Enum(), String(), Text(), and Time() because SQLAlchemy + defaults to incompatible / abnormal compiled names + + Enum -> VARCHAR + String -> VARCHAR[LENGTH] + Text -> VARCHAR[LENGTH] + Time -> TIME + Unicode -> VARCHAR[LENGTH] + UnicodeText -> TEXT + Uuid -> CHAR[32] + + But all of these types will be compiled to STRING in Databricks SQL + """ + return "STRING" + + +@compiles(sqlalchemy.types.Integer, "databricks") +def compile_integer_databricks(type_, compiler, **kw): + """ + We need to override the default Integer compilation rendering because Databricks uses "INT" instead of "INTEGER" + """ + return "INT" + + +@compiles(sqlalchemy.types.LargeBinary, "databricks") +def compile_binary_databricks(type_, compiler, **kw): + """ + We need to override the default LargeBinary compilation rendering because Databricks uses "BINARY" instead of "BLOB" + """ + return "BINARY" + + +@compiles(sqlalchemy.types.Numeric, "databricks") +def compile_numeric_databricks(type_, compiler, **kw): + """ + We need to override the default Numeric compilation rendering because Databricks uses "DECIMAL" instead of "NUMERIC" + + The built-in visit_DECIMAL behaviour captures the precision and scale. Here we're just mapping calls to compile Numeric + to the SQLAlchemy Decimal() implementation + """ + return compiler.visit_DECIMAL(type_, **kw) + + +@compiles(sqlalchemy.types.DateTime, "databricks") +def compile_datetime_databricks(type_, compiler, **kw): + """ + We need to override the default DateTime compilation rendering because Databricks uses "TIMESTAMP_NTZ" instead of "DATETIME" + """ + return "TIMESTAMP_NTZ" + + +@compiles(sqlalchemy.types.ARRAY, "databricks") +def compile_array_databricks(type_, compiler, **kw): + """ + SQLAlchemy's default ARRAY can't compile as it's only implemented for Postgresql. + The Postgres implementation works for Databricks SQL, so we duplicate that here. + + :type_: + This is an instance of sqlalchemy.types.ARRAY which always includes an item_type attribute + which is itself an instance of TypeEngine + + https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.ARRAY + """ + + inner = compiler.process(type_.item_type, **kw) + + return f"ARRAY<{inner}>" + + +class TIMESTAMP_NTZ(sqlalchemy.types.TypeDecorator): + """Represents values comprising values of fields year, month, day, hour, minute, and second. + All operations are performed without taking any time zone into account. + + Our dialect maps sqlalchemy.types.DateTime() to this type, which means that all DateTime() + objects are stored without tzinfo. To read and write timezone-aware datetimes use + databricks.sql.TIMESTAMP instead. + + https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-ntz-type.html + """ + + impl = sqlalchemy.types.DateTime + + cache_ok = True + + def process_result_value(self, value: Union[None, datetime], dialect): + if value is None: + return None + return value.replace(tzinfo=None) + + +class TIMESTAMP(sqlalchemy.types.TypeDecorator): + """Represents values comprising values of fields year, month, day, hour, minute, and second, + with the session local time-zone. + + Our dialect maps sqlalchemy.types.DateTime() to TIMESTAMP_NTZ, which means that all DateTime() + objects are stored without tzinfo. To read and write timezone-aware datetimes use + this type instead. + + ```python + # This won't work + `Column(sqlalchemy.DateTime(timezone=True))` + + # But this does + `Column(TIMESTAMP)` + ```` + + https://docs.databricks.com/en/sql/language-manual/data-types/timestamp-type.html + """ + + impl = sqlalchemy.types.DateTime + + cache_ok = True + + def process_result_value(self, value: Union[None, datetime], dialect): + if value is None: + return None + + if not value.tzinfo: + return value.replace(tzinfo=timezone.utc) + return value + + def process_bind_param( + self, value: Union[datetime, None], dialect + ) -> Optional[datetime]: + """pysql can pass datetime.datetime() objects directly to DBR""" + return value + + def process_literal_param( + self, value: Union[datetime, None], dialect: Dialect + ) -> str: + """ """ + return process_literal_param_hack(value) + + +@compiles(TIMESTAMP, "databricks") +def compile_timestamp_databricks(type_, compiler, **kw): + """ + We need to override the default DateTime compilation rendering because Databricks uses "TIMESTAMP_NTZ" instead of "DATETIME" + """ + return "TIMESTAMP" + + +class DatabricksTimeType(sqlalchemy.types.TypeDecorator): + """Databricks has no native TIME type. So we store it as a string.""" + + impl = sqlalchemy.types.Time + cache_ok = True + + BASE_FMT = "%H:%M:%S" + MICROSEC_PART = ".%f" + TIMEZONE_PART = "%z" + + def _generate_fmt_string(self, ms: bool, tz: bool) -> str: + """Return a format string for datetime.strptime() that includes or excludes microseconds and timezone.""" + _ = lambda x, y: x if y else "" + return f"{self.BASE_FMT}{_(self.MICROSEC_PART,ms)}{_(self.TIMEZONE_PART,tz)}" + + @property + def allowed_fmt_strings(self): + """Time strings can be read with or without microseconds and with or without a timezone.""" + + if not hasattr(self, "_allowed_fmt_strings"): + ms_switch = tz_switch = [True, False] + self._allowed_fmt_strings = [ + self._generate_fmt_string(x, y) + for x, y in product(ms_switch, tz_switch) + ] + + return self._allowed_fmt_strings + + def _parse_result_string(self, value: str) -> time: + """Parse a string into a time object. Try all allowed formats until one works.""" + for fmt in self.allowed_fmt_strings: + try: + # We use timetz() here because we want to preserve the timezone information + # Calling .time() will strip the timezone information + return datetime.strptime(value, fmt).timetz() + except ValueError: + pass + + raise ValueError(f"Could not parse time string {value}") + + def _determine_fmt_string(self, value: time) -> str: + """Determine which format string to use to render a time object as a string.""" + ms_bool = value.microsecond > 0 + tz_bool = value.tzinfo is not None + return self._generate_fmt_string(ms_bool, tz_bool) + + def process_bind_param(self, value: Union[time, None], dialect) -> Union[None, str]: + """Values sent to the database are converted to %:H:%M:%S strings.""" + if value is None: + return None + fmt_string = self._determine_fmt_string(value) + return value.strftime(fmt_string) + + # mypy doesn't like this workaround because TypeEngine wants process_literal_param to return a string + def process_literal_param(self, value, dialect) -> time: # type: ignore + """ """ + return process_literal_param_hack(value) + + def process_result_value( + self, value: Union[None, str], dialect + ) -> Union[time, None]: + """Values received from the database are parsed into datetime.time() objects""" + if value is None: + return None + + return self._parse_result_string(value) + + +class DatabricksStringType(sqlalchemy.types.TypeDecorator): + """We have to implement our own String() type because SQLAlchemy's default implementation + wants to escape single-quotes with a doubled single-quote. Databricks uses a backslash for + escaping of literal strings. And SQLAlchemy's default escaping breaks Databricks SQL. + """ + + impl = sqlalchemy.types.String + cache_ok = True + pe = ParamEscaper() + + def process_literal_param(self, value, dialect) -> str: + """SQLAlchemy's default string escaping for backslashes doesn't work for databricks. The logic here + implements the same logic as our legacy inline escaping logic. + """ + + return self.pe.escape_string(value) + + def literal_processor(self, dialect): + """We manually override this method to prevent further processing of the string literal beyond + what happens in the process_literal_param() method. + + The SQLAlchemy docs _specifically_ say to not override this method. + + It appears that any processing that happens from TypeEngine.process_literal_param happens _before_ + and _in addition to_ whatever the class's impl.literal_processor() method does. The String.literal_processor() + method performs a string replacement that doubles any single-quote in the contained string. This raises a syntax + error in Databricks. And it's not necessary because ParamEscaper() already implements all the escaping we need. + + We should consider opening an issue on the SQLAlchemy project to see if I'm using it wrong. + + See type_api.py::TypeEngine.literal_processor: + + ```python + def process(value: Any) -> str: + return fixed_impl_processor( + fixed_process_literal_param(value, dialect) + ) + ``` + + That call to fixed_impl_processor wraps the result of fixed_process_literal_param (which is the + process_literal_param defined in our Databricks dialect) + + https://docs.sqlalchemy.org/en/20/core/custom_types.html#sqlalchemy.types.TypeDecorator.literal_processor + """ + + def process(value): + """This is a copy of the default String.literal_processor() method but stripping away + its double-escaping behaviour for single-quotes. + """ + + _step1 = self.process_literal_param(value, dialect="databricks") + if dialect.identifier_preparer._double_percents: + _step2 = _step1.replace("%", "%%") + else: + _step2 = _step1 + + return "%s" % _step2 + + return process + + +class TINYINT(sqlalchemy.types.TypeDecorator): + """Represents 1-byte signed integers + + Acts like a sqlalchemy SmallInteger() in Python but writes to a TINYINT field in Databricks + + https://docs.databricks.com/en/sql/language-manual/data-types/tinyint-type.html + """ + + impl = sqlalchemy.types.SmallInteger + cache_ok = True + + +@compiles(TINYINT, "databricks") +def compile_tinyint(type_, compiler, **kw): + return "TINYINT" diff --git a/src/databricks/sqlalchemy/base.py b/src/databricks/sqlalchemy/base.py new file mode 100644 index 00000000..9148de7f --- /dev/null +++ b/src/databricks/sqlalchemy/base.py @@ -0,0 +1,436 @@ +from typing import Any, List, Optional, Dict, Union + +import databricks.sqlalchemy._ddl as dialect_ddl_impl +import databricks.sqlalchemy._types as dialect_type_impl +from databricks import sql +from databricks.sqlalchemy._parse import ( + _describe_table_extended_result_to_dict_list, + _match_table_not_found_string, + build_fk_dict, + build_pk_dict, + get_fk_strings_from_dte_output, + get_pk_strings_from_dte_output, + get_comment_from_dte_output, + parse_column_info_from_tgetcolumnsresponse, +) + +import sqlalchemy +from sqlalchemy import DDL, event +from sqlalchemy.engine import Connection, Engine, default, reflection +from sqlalchemy.engine.interfaces import ( + ReflectedForeignKeyConstraint, + ReflectedPrimaryKeyConstraint, + ReflectedColumn, + ReflectedTableComment, +) +from sqlalchemy.engine.reflection import ReflectionDefaults +from sqlalchemy.exc import DatabaseError, SQLAlchemyError + +try: + import alembic +except ImportError: + pass +else: + from alembic.ddl import DefaultImpl + + class DatabricksImpl(DefaultImpl): + __dialect__ = "databricks" + + +import logging + +logger = logging.getLogger(__name__) + + +class DatabricksDialect(default.DefaultDialect): + """This dialect implements only those methods required to pass our e2e tests""" + + # See sqlalchemy.engine.interfaces for descriptions of each of these properties + name: str = "databricks" + driver: str = "databricks" + default_schema_name: str = "default" + preparer = dialect_ddl_impl.DatabricksIdentifierPreparer # type: ignore + ddl_compiler = dialect_ddl_impl.DatabricksDDLCompiler + statement_compiler = dialect_ddl_impl.DatabricksStatementCompiler + supports_statement_cache: bool = True + supports_multivalues_insert: bool = True + supports_native_decimal: bool = True + supports_sane_rowcount: bool = False + non_native_boolean_check_constraint: bool = False + supports_identity_columns: bool = True + supports_schemas: bool = True + default_paramstyle: str = "named" + div_is_floordiv: bool = False + supports_default_values: bool = False + supports_server_side_cursors: bool = False + supports_sequences: bool = False + supports_native_boolean: bool = True + + colspecs = { + sqlalchemy.types.DateTime: dialect_type_impl.TIMESTAMP_NTZ, + sqlalchemy.types.Time: dialect_type_impl.DatabricksTimeType, + sqlalchemy.types.String: dialect_type_impl.DatabricksStringType, + } + + # SQLAlchemy requires that a table with no primary key + # constraint return a dictionary that looks like this. + EMPTY_PK: Dict[str, Any] = {"constrained_columns": [], "name": None} + + # SQLAlchemy requires that a table with no foreign keys + # defined return an empty list. Same for indexes. + EMPTY_FK: List + EMPTY_INDEX: List + EMPTY_FK = EMPTY_INDEX = [] + + @classmethod + def import_dbapi(cls): + return sql + + def _force_paramstyle_to_native_mode(self): + """This method can be removed after databricks-sql-connector wholly switches to NATIVE ParamApproach. + + This is a hack to trick SQLAlchemy into using a different paramstyle + than the one declared by this module in src/databricks/sql/__init__.py + + This method is called _after_ the dialect has been initialised, which is important because otherwise + our users would need to include a `paramstyle` argument in their SQLAlchemy connection string. + + This dialect is written to support NATIVE queries. Although the INLINE approach can technically work, + the same behaviour can be achieved within SQLAlchemy itself using its literal_processor methods. + """ + + self.paramstyle = self.default_paramstyle + + def create_connect_args(self, url): + # TODO: can schema be provided after HOST? + # Expected URI format is: databricks+thrift://token:dapi***@***.cloud.databricks.com?http_path=/sql/*** + + kwargs = { + "server_hostname": url.host, + "access_token": url.password, + "http_path": url.query.get("http_path"), + "catalog": url.query.get("catalog"), + "schema": url.query.get("schema"), + "use_inline_params": False, + } + + self.schema = kwargs["schema"] + self.catalog = kwargs["catalog"] + + self._force_paramstyle_to_native_mode() + + return [], kwargs + + def get_columns( + self, connection, table_name, schema=None, **kwargs + ) -> List[ReflectedColumn]: + """Return information about columns in `table_name`.""" + + with self.get_connection_cursor(connection) as cur: + resp = cur.columns( + catalog_name=self.catalog, + schema_name=schema or self.schema, + table_name=table_name, + ).fetchall() + + if not resp: + # TGetColumnsRequest will not raise an exception if passed a table that doesn't exist + # But Databricks supports tables with no columns. So if the result is an empty list, + # we need to check if the table exists (and raise an exception if not) or simply return + # an empty list. + self._describe_table_extended( + connection, + table_name, + self.catalog, + schema or self.schema, + expect_result=False, + ) + return resp + columns = [] + for col in resp: + row_dict = parse_column_info_from_tgetcolumnsresponse(col) + columns.append(row_dict) + + return columns + + def _describe_table_extended( + self, + connection: Connection, + table_name: str, + catalog_name: Optional[str] = None, + schema_name: Optional[str] = None, + expect_result=True, + ) -> Union[List[Dict[str, str]], None]: + """Run DESCRIBE TABLE EXTENDED on a table and return a list of dictionaries of the result. + + This method is the fastest way to check for the presence of a table in a schema. + + If expect_result is False, this method returns None as the output dict isn't required. + + Raises NoSuchTableError if the table is not present in the schema. + """ + + _target_catalog = catalog_name or self.catalog + _target_schema = schema_name or self.schema + _target = f"`{_target_catalog}`.`{_target_schema}`.`{table_name}`" + + # sql injection risk? + # DESCRIBE TABLE EXTENDED in DBR doesn't support parameterised inputs :( + stmt = DDL(f"DESCRIBE TABLE EXTENDED {_target}") + + try: + result = connection.execute(stmt) + except DatabaseError as e: + if _match_table_not_found_string(str(e)): + raise sqlalchemy.exc.NoSuchTableError( + f"No such table {table_name}" + ) from e + raise e + + if not expect_result: + return None + + fmt_result = _describe_table_extended_result_to_dict_list(result) + return fmt_result + + @reflection.cache + def get_pk_constraint( + self, + connection, + table_name: str, + schema: Optional[str] = None, + **kw: Any, + ) -> ReflectedPrimaryKeyConstraint: + """Fetch information about the primary key constraint on table_name. + + Returns a dictionary with these keys: + constrained_columns + a list of column names that make up the primary key. Results is an empty list + if no PRIMARY KEY is defined. + + name + the name of the primary key constraint + """ + + result = self._describe_table_extended( + connection=connection, + table_name=table_name, + schema_name=schema, + ) + + # Type ignore is because mypy knows that self._describe_table_extended *can* + # return None (even though it never will since expect_result defaults to True) + raw_pk_constraints: List = get_pk_strings_from_dte_output(result) # type: ignore + if not any(raw_pk_constraints): + return self.EMPTY_PK # type: ignore + + if len(raw_pk_constraints) > 1: + logger.warning( + "Found more than one primary key constraint in DESCRIBE TABLE EXTENDED output. " + "This is unexpected. Please report this as a bug. " + "Only the first primary key constraint will be returned." + ) + + first_pk_constraint = raw_pk_constraints[0] + pk_name = first_pk_constraint.get("col_name") + pk_constraint_string = first_pk_constraint.get("data_type") + + # TODO: figure out how to return sqlalchemy.interfaces in a way that mypy respects + return build_pk_dict(pk_name, pk_constraint_string) # type: ignore + + def get_foreign_keys( + self, connection, table_name, schema=None, **kw + ) -> List[ReflectedForeignKeyConstraint]: + """Return information about foreign_keys in `table_name`.""" + + result = self._describe_table_extended( + connection=connection, + table_name=table_name, + schema_name=schema, + ) + + # Type ignore is because mypy knows that self._describe_table_extended *can* + # return None (even though it never will since expect_result defaults to True) + raw_fk_constraints: List = get_fk_strings_from_dte_output(result) # type: ignore + + if not any(raw_fk_constraints): + return self.EMPTY_FK + + fk_constraints = [] + for constraint_dict in raw_fk_constraints: + fk_name = constraint_dict.get("col_name") + fk_constraint_string = constraint_dict.get("data_type") + this_constraint_dict = build_fk_dict( + fk_name, fk_constraint_string, schema_name=schema + ) + fk_constraints.append(this_constraint_dict) + + # TODO: figure out how to return sqlalchemy.interfaces in a way that mypy respects + return fk_constraints # type: ignore + + def get_indexes(self, connection, table_name, schema=None, **kw): + """SQLAlchemy requires this method. Databricks doesn't support indexes.""" + return self.EMPTY_INDEX + + @reflection.cache + def get_table_names(self, connection: Connection, schema=None, **kwargs): + """Return a list of tables in the current schema.""" + + _target_catalog = self.catalog + _target_schema = schema or self.schema + _target = f"`{_target_catalog}`.`{_target_schema}`" + + stmt = DDL(f"SHOW TABLES FROM {_target}") + + tables_result = connection.execute(stmt).all() + views_result = self.get_view_names(connection=connection, schema=schema) + + # In Databricks, SHOW TABLES FROM returns both tables and views. + # Potential optimisation: rewrite this to instead query information_schema + tables_minus_views = [ + row.tableName for row in tables_result if row.tableName not in views_result + ] + + return tables_minus_views + + @reflection.cache + def get_view_names( + self, + connection, + schema=None, + only_materialized=False, + only_temp=False, + **kwargs, + ) -> List[str]: + """Returns a list of string view names contained in the schema, if any.""" + + _target_catalog = self.catalog + _target_schema = schema or self.schema + _target = f"`{_target_catalog}`.`{_target_schema}`" + + stmt = DDL(f"SHOW VIEWS FROM {_target}") + result = connection.execute(stmt).all() + + return [ + row.viewName + for row in result + if (not only_materialized or row.isMaterialized) + and (not only_temp or row.isTemporary) + ] + + @reflection.cache + def get_materialized_view_names( + self, connection: Connection, schema: Optional[str] = None, **kw: Any + ) -> List[str]: + """A wrapper around get_view_names that fetches only the names of materialized views""" + return self.get_view_names(connection, schema, only_materialized=True) + + @reflection.cache + def get_temp_view_names( + self, connection: Connection, schema: Optional[str] = None, **kw: Any + ) -> List[str]: + """A wrapper around get_view_names that fetches only the names of temporary views""" + return self.get_view_names(connection, schema, only_temp=True) + + def do_rollback(self, dbapi_connection): + # Databricks SQL Does not support transactions + pass + + @reflection.cache + def has_table( + self, connection, table_name, schema=None, catalog=None, **kwargs + ) -> bool: + """For internal dialect use, check the existence of a particular table + or view in the database. + """ + + try: + self._describe_table_extended( + connection=connection, + table_name=table_name, + catalog_name=catalog, + schema_name=schema, + ) + return True + except sqlalchemy.exc.NoSuchTableError as e: + return False + + def get_connection_cursor(self, connection): + """Added for backwards compatibility with 1.3.x""" + if hasattr(connection, "_dbapi_connection"): + return connection._dbapi_connection.dbapi_connection.cursor() + elif hasattr(connection, "raw_connection"): + return connection.raw_connection().cursor() + elif hasattr(connection, "connection"): + return connection.connection.cursor() + + raise SQLAlchemyError( + "Databricks dialect can't obtain a cursor context manager from the dbapi" + ) + + @reflection.cache + def get_schema_names(self, connection, **kw): + """Return a list of all schema names available in the database.""" + stmt = DDL("SHOW SCHEMAS") + result = connection.execute(stmt) + schema_list = [row[0] for row in result] + return schema_list + + @reflection.cache + def get_table_comment( + self, + connection: Connection, + table_name: str, + schema: Optional[str] = None, + **kw: Any, + ) -> ReflectedTableComment: + result = self._describe_table_extended( + connection=connection, + table_name=table_name, + schema_name=schema, + ) + + if result is None: + return ReflectionDefaults.table_comment() + + comment = get_comment_from_dte_output(result) + + if comment: + return dict(text=comment) + else: + return ReflectionDefaults.table_comment() + + +@event.listens_for(Engine, "do_connect") +def receive_do_connect(dialect, conn_rec, cargs, cparams): + """Helpful for DS on traffic from clients using SQLAlchemy in particular""" + + # Ignore connect invocations that don't use our dialect + if not dialect.name == "databricks": + return + + ua = cparams.get("_user_agent_entry", "") + + def add_sqla_tag_if_not_present(val: str): + if not val: + output = "sqlalchemy" + + if val and "sqlalchemy" in val: + output = val + + else: + output = f"sqlalchemy + {val}" + + return output + + cparams["_user_agent_entry"] = add_sqla_tag_if_not_present(ua) + + if sqlalchemy.__version__.startswith("1.3"): + # SQLAlchemy 1.3.x fails to parse the http_path, catalog, and schema from our connection string + # These should be passed in as connect_args when building the Engine + + if "schema" in cparams: + dialect.schema = cparams["schema"] + + if "catalog" in cparams: + dialect.catalog = cparams["catalog"] diff --git a/src/databricks/sqlalchemy/py.typed b/src/databricks/sqlalchemy/py.typed new file mode 100755 index 00000000..e69de29b diff --git a/src/databricks/sqlalchemy/requirements.py b/src/databricks/sqlalchemy/requirements.py new file mode 100644 index 00000000..5c70c029 --- /dev/null +++ b/src/databricks/sqlalchemy/requirements.py @@ -0,0 +1,249 @@ +""" +The complete list of requirements is provided by SQLAlchemy here: + +https://github.com/sqlalchemy/sqlalchemy/blob/main/lib/sqlalchemy/testing/requirements.py + +When SQLAlchemy skips a test because a requirement is closed() it gives a generic skip message. +To make these failures more actionable, we only define requirements in this file that we wish to +force to be open(). If a test should be skipped on Databricks, it will be specifically marked skip +in test_suite.py with a Databricks-specific reason. + +See the special note about the array_type exclusion below. +See special note about has_temp_table exclusion below. +""" + +import sqlalchemy.testing.requirements +import sqlalchemy.testing.exclusions + + +class Requirements(sqlalchemy.testing.requirements.SuiteRequirements): + @property + def date_historic(self): + """target dialect supports representation of Python + datetime.datetime() objects with historic (pre 1970) values.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def datetime_historic(self): + """target dialect supports representation of Python + datetime.datetime() objects with historic (pre 1970) values.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def datetime_literals(self): + """target dialect supports rendering of a date, time, or datetime as a + literal string, e.g. via the TypeEngine.literal_processor() method. + + """ + + return sqlalchemy.testing.exclusions.open() + + @property + def timestamp_microseconds(self): + """target dialect supports representation of Python + datetime.datetime() with microsecond objects but only + if TIMESTAMP is used.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def time_microseconds(self): + """target dialect supports representation of Python + datetime.time() with microsecond objects. + + This requirement declaration isn't needed but I've included it here for completeness. + Since Databricks doesn't have a TIME type, SQLAlchemy will compile Time() columns + as STRING Databricks data types. And we use a custom time type to render those strings + between str() and time.time() representations. Therefore we can store _any_ precision + that SQLAlchemy needs. The time_microseconds requirement defaults to ON for all dialects + except mssql, mysql, mariadb, and oracle. + """ + + return sqlalchemy.testing.exclusions.open() + + @property + def infinity_floats(self): + """The Float type can persist and load float('inf'), float('-inf').""" + + return sqlalchemy.testing.exclusions.open() + + @property + def precision_numerics_retains_significant_digits(self): + """A precision numeric type will return empty significant digits, + i.e. a value such as 10.000 will come back in Decimal form with + the .000 maintained.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def precision_numerics_many_significant_digits(self): + """target backend supports values with many digits on both sides, + such as 319438950232418390.273596, 87673.594069654243 + + """ + return sqlalchemy.testing.exclusions.open() + + @property + def array_type(self): + """While Databricks does support ARRAY types, pysql cannot bind them. So + we cannot use them with SQLAlchemy + + Due to a bug in SQLAlchemy, we _must_ define this exclusion as closed() here or else the + test runner will crash the pytest process due to an AttributeError + """ + + # TODO: Implement array type using inline? + return sqlalchemy.testing.exclusions.closed() + + @property + def table_ddl_if_exists(self): + """target platform supports IF NOT EXISTS / IF EXISTS for tables.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def identity_columns(self): + """If a backend supports GENERATED { ALWAYS | BY DEFAULT } + AS IDENTITY""" + return sqlalchemy.testing.exclusions.open() + + @property + def identity_columns_standard(self): + """If a backend supports GENERATED { ALWAYS | BY DEFAULT } + AS IDENTITY with a standard syntax. + This is mainly to exclude MSSql. + """ + return sqlalchemy.testing.exclusions.open() + + @property + def has_temp_table(self): + """target dialect supports checking a single temp table name + + unfortunately this is not the same as temp_table_names + + SQLAlchemy's HasTableTest is not normalised in such a way that temp table tests + are separate from temp view and normal table tests. If those tests were split out, + we would just add detailed skip markers in test_suite.py. But since we'd like to + run the HasTableTest group for the features we support, we must set this exclusinon + to closed(). + + It would be ideal if there were a separate requirement for has_temp_view. Without it, + we're in a bind. + """ + return sqlalchemy.testing.exclusions.closed() + + @property + def temporary_views(self): + """target database supports temporary views""" + return sqlalchemy.testing.exclusions.open() + + @property + def views(self): + """Target database must support VIEWs.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def temporary_tables(self): + """target database supports temporary tables + + ComponentReflection test is intricate and simply cannot function without this exclusion being defined here. + This happens because we cannot skip individual combinations used in ComponentReflection test. + """ + return sqlalchemy.testing.exclusions.closed() + + @property + def table_reflection(self): + """target database has general support for table reflection""" + return sqlalchemy.testing.exclusions.open() + + @property + def comment_reflection(self): + """Indicates if the database support table comment reflection""" + return sqlalchemy.testing.exclusions.open() + + @property + def comment_reflection_full_unicode(self): + """Indicates if the database support table comment reflection in the + full unicode range, including emoji etc. + """ + return sqlalchemy.testing.exclusions.open() + + @property + def temp_table_reflection(self): + """ComponentReflection test is intricate and simply cannot function without this exclusion being defined here. + This happens because we cannot skip individual combinations used in ComponentReflection test. + """ + return sqlalchemy.testing.exclusions.closed() + + @property + def index_reflection(self): + """ComponentReflection test is intricate and simply cannot function without this exclusion being defined here. + This happens because we cannot skip individual combinations used in ComponentReflection test. + """ + return sqlalchemy.testing.exclusions.closed() + + @property + def unique_constraint_reflection(self): + """ComponentReflection test is intricate and simply cannot function without this exclusion being defined here. + This happens because we cannot skip individual combinations used in ComponentReflection test. + + Databricks doesn't support UNIQUE constraints. + """ + return sqlalchemy.testing.exclusions.closed() + + @property + def reflects_pk_names(self): + """Target driver reflects the name of primary key constraints.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def datetime_implicit_bound(self): + """target dialect when given a datetime object will bind it such + that the database server knows the object is a date, and not + a plain string. + """ + + return sqlalchemy.testing.exclusions.open() + + @property + def tuple_in(self): + return sqlalchemy.testing.exclusions.open() + + @property + def ctes(self): + return sqlalchemy.testing.exclusions.open() + + @property + def ctes_with_update_delete(self): + return sqlalchemy.testing.exclusions.open() + + @property + def delete_from(self): + """Target must support DELETE FROM..FROM or DELETE..USING syntax""" + return sqlalchemy.testing.exclusions.open() + + @property + def table_value_constructor(self): + return sqlalchemy.testing.exclusions.open() + + @property + def reflect_tables_no_columns(self): + return sqlalchemy.testing.exclusions.open() + + @property + def denormalized_names(self): + """Target database must have 'denormalized', i.e. + UPPERCASE as case insensitive names.""" + + return sqlalchemy.testing.exclusions.open() + + @property + def time_timezone(self): + """target dialect supports representation of Python + datetime.time() with tzinfo with Time(timezone=True).""" + + return sqlalchemy.testing.exclusions.open() diff --git a/src/databricks/sqlalchemy/setup.cfg b/src/databricks/sqlalchemy/setup.cfg new file mode 100644 index 00000000..ab89d17d --- /dev/null +++ b/src/databricks/sqlalchemy/setup.cfg @@ -0,0 +1,4 @@ + +[sqla_testing] +requirement_cls=databricks.sqlalchemy.requirements:Requirements +profile_file=profiles.txt diff --git a/src/databricks/sqlalchemy/test/_extra.py b/src/databricks/sqlalchemy/test/_extra.py new file mode 100644 index 00000000..2f3e7a7d --- /dev/null +++ b/src/databricks/sqlalchemy/test/_extra.py @@ -0,0 +1,70 @@ +"""Additional tests authored by Databricks that use SQLAlchemy's test fixtures +""" + +import datetime + +from sqlalchemy.testing.suite.test_types import ( + _LiteralRoundTripFixture, + fixtures, + testing, + eq_, + select, + Table, + Column, + config, + _DateFixture, + literal, +) +from databricks.sqlalchemy import TINYINT, TIMESTAMP + + +class TinyIntegerTest(_LiteralRoundTripFixture, fixtures.TestBase): + __backend__ = True + + def test_literal(self, literal_round_trip): + literal_round_trip(TINYINT, [5], [5]) + + @testing.fixture + def integer_round_trip(self, metadata, connection): + def run(datatype, data): + int_table = Table( + "tiny_integer_table", + metadata, + Column( + "id", + TINYINT, + primary_key=True, + test_needs_autoincrement=False, + ), + Column("tiny_integer_data", datatype), + ) + + metadata.create_all(config.db) + + connection.execute(int_table.insert(), {"id": 1, "integer_data": data}) + + row = connection.execute(select(int_table.c.integer_data)).first() + + eq_(row, (data,)) + + assert isinstance(row[0], int) + + return run + + +class DateTimeTZTestCustom(_DateFixture, fixtures.TablesTest): + """This test confirms that when a user uses the TIMESTAMP + type to store a datetime object, it retains its timezone + """ + + __backend__ = True + datatype = TIMESTAMP + data = datetime.datetime(2012, 10, 15, 12, 57, 18, tzinfo=datetime.timezone.utc) + + @testing.requires.datetime_implicit_bound + def test_select_direct(self, connection): + + # We need to pass the TIMESTAMP type to the literal function + # so that the value is processed correctly. + result = connection.scalar(select(literal(self.data, TIMESTAMP))) + eq_(result, self.data) diff --git a/src/databricks/sqlalchemy/test/_future.py b/src/databricks/sqlalchemy/test/_future.py new file mode 100644 index 00000000..6e470f60 --- /dev/null +++ b/src/databricks/sqlalchemy/test/_future.py @@ -0,0 +1,331 @@ +# type: ignore + +from enum import Enum + +import pytest +from databricks.sqlalchemy.test._regression import ( + ExpandingBoundInTest, + IdentityAutoincrementTest, + LikeFunctionsTest, + NormalizedNameTest, +) +from databricks.sqlalchemy.test._unsupported import ( + ComponentReflectionTest, + ComponentReflectionTestExtra, + CTETest, + InsertBehaviorTest, +) +from sqlalchemy.testing.suite import ( + ArrayTest, + BinaryTest, + BizarroCharacterFKResolutionTest, + CollateTest, + ComputedColumnTest, + ComputedReflectionTest, + DifficultParametersTest, + FutureWeCanSetDefaultSchemaWEventsTest, + IdentityColumnTest, + IdentityReflectionTest, + JSONLegacyStringCastIndexTest, + JSONTest, + NativeUUIDTest, + QuotedNameArgumentTest, + RowCountTest, + SimpleUpdateDeleteTest, + WeCanSetDefaultSchemaWEventsTest, +) + + +class FutureFeature(Enum): + ARRAY = "ARRAY column type handling" + BINARY = "BINARY column type handling" + CHECK = "CHECK constraint handling" + COLLATE = "COLLATE DDL generation" + CTE_FEAT = "required CTE features" + EMPTY_INSERT = "empty INSERT support" + FK_OPTS = "foreign key option checking" + GENERATED_COLUMNS = "Delta computed / generated columns support" + IDENTITY = "identity reflection" + JSON = "JSON column type handling" + MULTI_PK = "get_multi_pk_constraint method" + PROVISION = "event-driven engine configuration" + REGEXP = "_visit_regexp" + SANE_ROWCOUNT = "sane_rowcount support" + TBL_OPTS = "get_table_options method" + TEST_DESIGN = "required test-fixture overrides" + TUPLE_LITERAL = "tuple-like IN markers completely" + UUID = "native Uuid() type" + VIEW_DEF = "get_view_definition method" + + +def render_future_feature(rsn: FutureFeature, extra=False) -> str: + postfix = " More detail in _future.py" if extra else "" + return f"[FUTURE][{rsn.name}]: This dialect doesn't implement {rsn.value}.{postfix}" + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.BINARY)) +class BinaryTest(BinaryTest): + """Databricks doesn't support binding of BINARY type values. When DBR supports this, we can implement + in this dialect. + """ + + pass + + +class ExpandingBoundInTest(ExpandingBoundInTest): + @pytest.mark.skip(render_future_feature(FutureFeature.TUPLE_LITERAL)) + def test_empty_heterogeneous_tuples_bindparam(self): + pass + + @pytest.mark.skip(render_future_feature(FutureFeature.TUPLE_LITERAL)) + def test_empty_heterogeneous_tuples_direct(self): + pass + + @pytest.mark.skip(render_future_feature(FutureFeature.TUPLE_LITERAL)) + def test_empty_homogeneous_tuples_bindparam(self): + pass + + @pytest.mark.skip(render_future_feature(FutureFeature.TUPLE_LITERAL)) + def test_empty_homogeneous_tuples_direct(self): + pass + + +class NormalizedNameTest(NormalizedNameTest): + @pytest.mark.skip(render_future_feature(FutureFeature.TEST_DESIGN, True)) + def test_get_table_names(self): + """I'm not clear how this test can ever pass given that it's assertion looks like this: + + ```python + eq_(tablenames[0].upper(), tablenames[0].lower()) + eq_(tablenames[1].upper(), tablenames[1].lower()) + ``` + + It's forcibly calling .upper() and .lower() on the same string and expecting them to be equal. + """ + pass + + +class CTETest(CTETest): + @pytest.mark.skip(render_future_feature(FutureFeature.CTE_FEAT, True)) + def test_delete_from_round_trip(self): + """Databricks dialect doesn't implement multiple-table criteria within DELETE""" + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.TEST_DESIGN, True)) +class IdentityColumnTest(IdentityColumnTest): + """Identity works. Test needs rewrite for Databricks. See comments in test_suite.py + + The setup for these tests tries to create a table with a DELTA IDENTITY column but has two problems: + 1. It uses an Integer() type for the column. Whereas DELTA IDENTITY columns must be BIGINT. + 2. It tries to set the start == 42, which Databricks doesn't support + + I can get the tests to _run_ by patching the table fixture to use BigInteger(). But it asserts that the + identity of two rows are 42 and 43, which is not possible since they will be rows 1 and 2 instead. + + I'm satisified through manual testing that our implementation of visit_identity_column works but a better test is needed. + """ + + pass + + +class IdentityAutoincrementTest(IdentityAutoincrementTest): + @pytest.mark.skip(render_future_feature(FutureFeature.TEST_DESIGN, True)) + def test_autoincrement_with_identity(self): + """This test has the same issue as IdentityColumnTest.test_select_all in that it creates a table with identity + using an Integer() rather than a BigInteger(). If I override this behaviour to use a BigInteger() instead, the + test passes. + """ + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.TEST_DESIGN)) +class BizarroCharacterFKResolutionTest(BizarroCharacterFKResolutionTest): + """Some of the combinations in this test pass. Others fail. Given the esoteric nature of these failures, + we have opted to defer implementing fixes to a later time, guided by customer feedback. Passage of + these tests is not an acceptance criteria for our dialect. + """ + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.TEST_DESIGN)) +class DifficultParametersTest(DifficultParametersTest): + """Some of the combinations in this test pass. Others fail. Given the esoteric nature of these failures, + we have opted to defer implementing fixes to a later time, guided by customer feedback. Passage of + these tests is not an acceptance criteria for our dialect. + """ + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.IDENTITY, True)) +class IdentityReflectionTest(IdentityReflectionTest): + """It's not clear _how_ to implement this for SQLAlchemy. Columns created with GENERATED ALWAYS AS IDENTITY + are not specially demarked in the output of TGetColumnsResponse or DESCRIBE TABLE EXTENDED. + + We could theoretically parse this from the contents of `SHOW CREATE TABLE` but that feels like a hack. + """ + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.JSON)) +class JSONTest(JSONTest): + """Databricks supports JSON path expressions in queries it's just not implemented in this dialect.""" + + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.JSON)) +class JSONLegacyStringCastIndexTest(JSONLegacyStringCastIndexTest): + """Same comment applies as JSONTest""" + + pass + + +class LikeFunctionsTest(LikeFunctionsTest): + @pytest.mark.skip(render_future_feature(FutureFeature.REGEXP)) + def test_not_regexp_match(self): + """The defaul dialect doesn't implement _visit_regexp methods so we don't get them automatically.""" + pass + + @pytest.mark.skip(render_future_feature(FutureFeature.REGEXP)) + def test_regexp_match(self): + """The defaul dialect doesn't implement _visit_regexp methods so we don't get them automatically.""" + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.COLLATE)) +class CollateTest(CollateTest): + """This is supported in Databricks. Not implemented here.""" + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.UUID, True)) +class NativeUUIDTest(NativeUUIDTest): + """Type implementation will be straightforward. Since Databricks doesn't have a native UUID type we can use + a STRING field, create a custom TypeDecorator for sqlalchemy.types.Uuid and add it to the dialect's colspecs. + + Then mark requirements.uuid_data_type as open() so this test can run. + """ + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.SANE_ROWCOUNT)) +class RowCountTest(RowCountTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.SANE_ROWCOUNT)) +class SimpleUpdateDeleteTest(SimpleUpdateDeleteTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.PROVISION, True)) +class WeCanSetDefaultSchemaWEventsTest(WeCanSetDefaultSchemaWEventsTest): + """provision.py allows us to define event listeners that emit DDL for things like setting up a test schema + or, in this case, changing the default schema for the connection after it's been built. This would override + the schema defined in the sqlalchemy connection string. This support is possible but is not implemented + in the dialect. Deferred for now. + """ + + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.PROVISION, True)) +class FutureWeCanSetDefaultSchemaWEventsTest(FutureWeCanSetDefaultSchemaWEventsTest): + """provision.py allows us to define event listeners that emit DDL for things like setting up a test schema + or, in this case, changing the default schema for the connection after it's been built. This would override + the schema defined in the sqlalchemy connection string. This support is possible but is not implemented + in the dialect. Deferred for now. + """ + + pass + + +class ComponentReflectionTest(ComponentReflectionTest): + @pytest.mark.skip(reason=render_future_feature(FutureFeature.TBL_OPTS, True)) + def test_multi_get_table_options_tables(self): + """It's not clear what the expected ouput from this method would even _be_. Requires research.""" + pass + + @pytest.mark.skip(render_future_feature(FutureFeature.VIEW_DEF)) + def test_get_view_definition(self): + pass + + @pytest.mark.skip(render_future_feature(FutureFeature.VIEW_DEF)) + def test_get_view_definition_does_not_exist(self): + pass + + @pytest.mark.skip(render_future_feature(FutureFeature.MULTI_PK)) + def test_get_multi_pk_constraint(self): + pass + + @pytest.mark.skip(render_future_feature(FutureFeature.CHECK)) + def test_get_multi_check_constraints(self): + pass + + +class ComponentReflectionTestExtra(ComponentReflectionTestExtra): + @pytest.mark.skip(render_future_feature(FutureFeature.CHECK)) + def test_get_check_constraints(self): + pass + + @pytest.mark.skip(render_future_feature(FutureFeature.FK_OPTS)) + def test_get_foreign_key_options(self): + """It's not clear from the test code what the expected output is here. Further research required.""" + pass + + +class InsertBehaviorTest(InsertBehaviorTest): + @pytest.mark.skip(render_future_feature(FutureFeature.EMPTY_INSERT, True)) + def test_empty_insert(self): + """Empty inserts are possible using DEFAULT VALUES on Databricks. To implement it, we need + to hook into the SQLCompiler to render a no-op column list. With SQLAlchemy's default implementation + the request fails with a syntax error + """ + pass + + @pytest.mark.skip(render_future_feature(FutureFeature.EMPTY_INSERT, True)) + def test_empty_insert_multiple(self): + """Empty inserts are possible using DEFAULT VALUES on Databricks. To implement it, we need + to hook into the SQLCompiler to render a no-op column list. With SQLAlchemy's default implementation + the request fails with a syntax error + """ + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.ARRAY)) +class ArrayTest(ArrayTest): + """While Databricks supports ARRAY types, DBR cannot handle bound parameters of this type. + This makes them unusable to SQLAlchemy without some workaround. Potentially we could inline + the values of these parameters (which risks sql injection). + """ + + +@pytest.mark.reviewed +@pytest.mark.skip(render_future_feature(FutureFeature.TEST_DESIGN, True)) +class QuotedNameArgumentTest(QuotedNameArgumentTest): + """These tests are challenging. The whole test setup depends on a table with a name like `quote ' one` + which will never work on Databricks because table names can't contains spaces. But QuotedNamedArgumentTest + also checks the behaviour of DDL identifier preparation process. We need to override some of IdentifierPreparer + methods because these are the ultimate control for whether or not CHECK and UNIQUE constraints are emitted. + """ + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_future_feature(FutureFeature.GENERATED_COLUMNS)) +class ComputedColumnTest(ComputedColumnTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_future_feature(FutureFeature.GENERATED_COLUMNS)) +class ComputedReflectionTest(ComputedReflectionTest): + pass diff --git a/src/databricks/sqlalchemy/test/_regression.py b/src/databricks/sqlalchemy/test/_regression.py new file mode 100644 index 00000000..4dbc5ec2 --- /dev/null +++ b/src/databricks/sqlalchemy/test/_regression.py @@ -0,0 +1,311 @@ +# type: ignore + +import pytest +from sqlalchemy.testing.suite import ( + ArgSignatureTest, + BooleanTest, + CastTypeDecoratorTest, + ComponentReflectionTestExtra, + CompositeKeyReflectionTest, + CompoundSelectTest, + DateHistoricTest, + DateTest, + DateTimeCoercedToDateTimeTest, + DateTimeHistoricTest, + DateTimeMicrosecondsTest, + DateTimeTest, + DeprecatedCompoundSelectTest, + DistinctOnTest, + EscapingTest, + ExistsTest, + ExpandingBoundInTest, + FetchLimitOffsetTest, + FutureTableDDLTest, + HasTableTest, + IdentityAutoincrementTest, + InsertBehaviorTest, + IntegerTest, + IsOrIsNotDistinctFromTest, + JoinTest, + LikeFunctionsTest, + NormalizedNameTest, + NumericTest, + OrderByLabelTest, + PingTest, + PostCompileParamsTest, + ReturningGuardsTest, + RowFetchTest, + SameNamedSchemaTableTest, + StringTest, + TableDDLTest, + TableNoColumnsTest, + TextTest, + TimeMicrosecondsTest, + TimestampMicrosecondsTest, + TimeTest, + TimeTZTest, + TrueDivTest, + UnicodeTextTest, + UnicodeVarcharTest, + UuidTest, + ValuesExpressionTest, +) + +from databricks.sqlalchemy.test.overrides._ctetest import CTETest +from databricks.sqlalchemy.test.overrides._componentreflectiontest import ( + ComponentReflectionTest, +) + + +@pytest.mark.reviewed +class NumericTest(NumericTest): + pass + + +@pytest.mark.reviewed +class HasTableTest(HasTableTest): + pass + + +@pytest.mark.reviewed +class ComponentReflectionTestExtra(ComponentReflectionTestExtra): + pass + + +@pytest.mark.reviewed +class InsertBehaviorTest(InsertBehaviorTest): + pass + + +@pytest.mark.reviewed +class ComponentReflectionTest(ComponentReflectionTest): + """This test requires two schemas be present in the target Databricks workspace: + - The schema set in --dburi + - A second schema named "test_schema" + + Note that test_get_multi_foreign keys is flaky because DBR does not guarantee the order of data returned in DESCRIBE TABLE EXTENDED + + _Most_ of these tests pass if we manually override the bad test setup. + """ + + pass + + +@pytest.mark.reviewed +class TableDDLTest(TableDDLTest): + pass + + +@pytest.mark.reviewed +class FutureTableDDLTest(FutureTableDDLTest): + pass + + +@pytest.mark.reviewed +class FetchLimitOffsetTest(FetchLimitOffsetTest): + pass + + +@pytest.mark.reviewed +class UuidTest(UuidTest): + pass + + +@pytest.mark.reviewed +class ValuesExpressionTest(ValuesExpressionTest): + pass + + +@pytest.mark.reviewed +class BooleanTest(BooleanTest): + pass + + +@pytest.mark.reviewed +class PostCompileParamsTest(PostCompileParamsTest): + pass + + +@pytest.mark.reviewed +class TimeMicrosecondsTest(TimeMicrosecondsTest): + pass + + +@pytest.mark.reviewed +class TextTest(TextTest): + pass + + +@pytest.mark.reviewed +class StringTest(StringTest): + pass + + +@pytest.mark.reviewed +class DateTimeMicrosecondsTest(DateTimeMicrosecondsTest): + pass + + +@pytest.mark.reviewed +class TimestampMicrosecondsTest(TimestampMicrosecondsTest): + pass + + +@pytest.mark.reviewed +class DateTimeCoercedToDateTimeTest(DateTimeCoercedToDateTimeTest): + pass + + +@pytest.mark.reviewed +class TimeTest(TimeTest): + pass + + +@pytest.mark.reviewed +class DateTimeTest(DateTimeTest): + pass + + +@pytest.mark.reviewed +class DateTimeHistoricTest(DateTimeHistoricTest): + pass + + +@pytest.mark.reviewed +class DateTest(DateTest): + pass + + +@pytest.mark.reviewed +class DateHistoricTest(DateHistoricTest): + pass + + +@pytest.mark.reviewed +class RowFetchTest(RowFetchTest): + pass + + +@pytest.mark.reviewed +class CompositeKeyReflectionTest(CompositeKeyReflectionTest): + pass + + +@pytest.mark.reviewed +class TrueDivTest(TrueDivTest): + pass + + +@pytest.mark.reviewed +class ArgSignatureTest(ArgSignatureTest): + pass + + +@pytest.mark.reviewed +class CompoundSelectTest(CompoundSelectTest): + pass + + +@pytest.mark.reviewed +class DeprecatedCompoundSelectTest(DeprecatedCompoundSelectTest): + pass + + +@pytest.mark.reviewed +class CastTypeDecoratorTest(CastTypeDecoratorTest): + pass + + +@pytest.mark.reviewed +class DistinctOnTest(DistinctOnTest): + pass + + +@pytest.mark.reviewed +class EscapingTest(EscapingTest): + pass + + +@pytest.mark.reviewed +class ExistsTest(ExistsTest): + pass + + +@pytest.mark.reviewed +class IntegerTest(IntegerTest): + pass + + +@pytest.mark.reviewed +class IsOrIsNotDistinctFromTest(IsOrIsNotDistinctFromTest): + pass + + +@pytest.mark.reviewed +class JoinTest(JoinTest): + pass + + +@pytest.mark.reviewed +class OrderByLabelTest(OrderByLabelTest): + pass + + +@pytest.mark.reviewed +class PingTest(PingTest): + pass + + +@pytest.mark.reviewed +class ReturningGuardsTest(ReturningGuardsTest): + pass + + +@pytest.mark.reviewed +class SameNamedSchemaTableTest(SameNamedSchemaTableTest): + pass + + +@pytest.mark.reviewed +class UnicodeTextTest(UnicodeTextTest): + pass + + +@pytest.mark.reviewed +class UnicodeVarcharTest(UnicodeVarcharTest): + pass + + +@pytest.mark.reviewed +class TableNoColumnsTest(TableNoColumnsTest): + pass + + +@pytest.mark.reviewed +class ExpandingBoundInTest(ExpandingBoundInTest): + pass + + +@pytest.mark.reviewed +class CTETest(CTETest): + pass + + +@pytest.mark.reviewed +class NormalizedNameTest(NormalizedNameTest): + pass + + +@pytest.mark.reviewed +class IdentityAutoincrementTest(IdentityAutoincrementTest): + pass + + +@pytest.mark.reviewed +class LikeFunctionsTest(LikeFunctionsTest): + pass + + +@pytest.mark.reviewed +class TimeTZTest(TimeTZTest): + pass diff --git a/src/databricks/sqlalchemy/test/_unsupported.py b/src/databricks/sqlalchemy/test/_unsupported.py new file mode 100644 index 00000000..c1f81205 --- /dev/null +++ b/src/databricks/sqlalchemy/test/_unsupported.py @@ -0,0 +1,450 @@ +# type: ignore + +from enum import Enum + +import pytest +from databricks.sqlalchemy.test._regression import ( + ComponentReflectionTest, + ComponentReflectionTestExtra, + CTETest, + FetchLimitOffsetTest, + FutureTableDDLTest, + HasTableTest, + InsertBehaviorTest, + NumericTest, + TableDDLTest, + UuidTest, +) + +# These are test suites that are fully skipped with a SkipReason +from sqlalchemy.testing.suite import ( + AutocommitIsolationTest, + DateTimeTZTest, + ExceptionTest, + HasIndexTest, + HasSequenceTest, + HasSequenceTestEmpty, + IsolationLevelTest, + LastrowidTest, + LongNameBlowoutTest, + PercentSchemaNamesTest, + ReturningTest, + SequenceCompilerTest, + SequenceTest, + ServerSideCursorsTest, + UnicodeSchemaTest, +) + + +class SkipReason(Enum): + AUTO_INC = "implicit AUTO_INCREMENT" + CTE_FEAT = "required CTE features" + CURSORS = "server-side cursors" + DECIMAL_FEAT = "required decimal features" + ENFORCE_KEYS = "enforcing primary or foreign key restraints" + FETCH = "fetch clauses" + IDENTIFIER_LENGTH = "identifiers > 255 characters" + IMPL_FLOAT_PREC = "required implicit float precision" + IMPLICIT_ORDER = "deterministic return order if ORDER BY is not present" + INDEXES = "SQL INDEXes" + RETURNING = "INSERT ... RETURNING syntax" + SEQUENCES = "SQL SEQUENCES" + STRING_FEAT = "required STRING type features" + SYMBOL_CHARSET = "symbols expected by test" + TEMP_TBL = "temporary tables" + TIMEZONE_OPT = "timezone-optional TIMESTAMP fields" + TRANSACTIONS = "transactions" + UNIQUE = "UNIQUE constraints" + + +def render_skip_reason(rsn: SkipReason, setup_error=False, extra=False) -> str: + prefix = "[BADSETUP]" if setup_error else "" + postfix = " More detail in _unsupported.py" if extra else "" + return f"[UNSUPPORTED]{prefix}[{rsn.name}]: Databricks does not support {rsn.value}.{postfix}" + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.ENFORCE_KEYS)) +class ExceptionTest(ExceptionTest): + """Per Databricks documentation, primary and foreign key constraints are informational only + and are not enforced. + + https://docs.databricks.com/api/workspace/tableconstraints + """ + + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.IDENTIFIER_LENGTH)) +class LongNameBlowoutTest(LongNameBlowoutTest): + """These tests all include assertions that the tested name > 255 characters""" + + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.SEQUENCES)) +class HasSequenceTest(HasSequenceTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.SEQUENCES)) +class HasSequenceTestEmpty(HasSequenceTestEmpty): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.INDEXES)) +class HasIndexTest(HasIndexTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.SYMBOL_CHARSET)) +class UnicodeSchemaTest(UnicodeSchemaTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.CURSORS)) +class ServerSideCursorsTest(ServerSideCursorsTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.SYMBOL_CHARSET)) +class PercentSchemaNamesTest(PercentSchemaNamesTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.TRANSACTIONS)) +class IsolationLevelTest(IsolationLevelTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.TRANSACTIONS)) +class AutocommitIsolationTest(AutocommitIsolationTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.RETURNING)) +class ReturningTest(ReturningTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.SEQUENCES)) +class SequenceTest(SequenceTest): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(reason=render_skip_reason(SkipReason.SEQUENCES)) +class SequenceCompilerTest(SequenceCompilerTest): + pass + + +class FetchLimitOffsetTest(FetchLimitOffsetTest): + @pytest.mark.flaky + @pytest.mark.skip(reason=render_skip_reason(SkipReason.IMPLICIT_ORDER, extra=True)) + def test_limit_render_multiple_times(self): + """This test depends on the order that records are inserted into the table. It's passing criteria requires that + a record inserted with id=1 is the first record returned when no ORDER BY clause is specified. But Databricks occasionally + INSERTS in a different order, which makes this test seem to fail. The test is flaky, but the underlying functionality + (can multiple LIMIT clauses be rendered) is not broken. + + Unclear if this is a bug in Databricks, Delta, or some race-condition in the test itself. + """ + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_bound_fetch_offset(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_fetch_offset_no_order(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_fetch_offset_nobinds(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_simple_fetch(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_simple_fetch_offset(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_simple_fetch_percent(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_simple_fetch_percent_ties(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_simple_fetch_ties(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_expr_fetch_offset(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_fetch_offset_percent(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_fetch_offset_percent_ties(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_fetch_offset_ties(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.FETCH)) + def test_fetch_offset_ties_exact_number(self): + pass + + +class UuidTest(UuidTest): + @pytest.mark.skip(reason=render_skip_reason(SkipReason.RETURNING)) + def test_uuid_returning(self): + pass + + +class FutureTableDDLTest(FutureTableDDLTest): + @pytest.mark.skip(render_skip_reason(SkipReason.INDEXES)) + def test_create_index_if_not_exists(self): + """We could use requirements.index_reflection and requirements.index_ddl_if_exists + here to disable this but prefer a more meaningful skip message + """ + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.INDEXES)) + def test_drop_index_if_exists(self): + """We could use requirements.index_reflection and requirements.index_ddl_if_exists + here to disable this but prefer a more meaningful skip message + """ + pass + + +class TableDDLTest(TableDDLTest): + @pytest.mark.skip(reason=render_skip_reason(SkipReason.INDEXES)) + def test_create_index_if_not_exists(self, connection): + """We could use requirements.index_reflection and requirements.index_ddl_if_exists + here to disable this but prefer a more meaningful skip message + """ + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.INDEXES)) + def test_drop_index_if_exists(self, connection): + """We could use requirements.index_reflection and requirements.index_ddl_if_exists + here to disable this but prefer a more meaningful skip message + """ + pass + + +class ComponentReflectionTest(ComponentReflectionTest): + """This test requires two schemas be present in the target Databricks workspace: + - The schema set in --dburi + - A second schema named "test_schema" + + Note that test_get_multi_foreign keys is flaky because DBR does not guarantee the order of data returned in DESCRIBE TABLE EXTENDED + """ + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.UNIQUE)) + def test_get_multi_unique_constraints(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.TEMP_TBL, True, True)) + def test_get_temp_view_names(self): + """While Databricks supports temporary views, this test creates a temp view aimed at a temp table. + Databricks doesn't support temp tables. So the test can never pass. + """ + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.TEMP_TBL)) + def test_get_temp_table_columns(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.TEMP_TBL)) + def test_get_temp_table_indexes(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.TEMP_TBL)) + def test_get_temp_table_names(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.TEMP_TBL)) + def test_get_temp_table_unique_constraints(self): + pass + + @pytest.mark.skip(reason=render_skip_reason(SkipReason.TEMP_TBL)) + def test_reflect_table_temp_table(self): + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.INDEXES)) + def test_get_indexes(self): + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.INDEXES)) + def test_multi_indexes(self): + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.INDEXES)) + def get_noncol_index(self): + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.UNIQUE)) + def test_get_unique_constraints(self): + pass + + +class NumericTest(NumericTest): + @pytest.mark.skip(render_skip_reason(SkipReason.DECIMAL_FEAT)) + def test_enotation_decimal(self): + """This test automatically runs if requirements.precision_numerics_enotation_large is open()""" + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.DECIMAL_FEAT)) + def test_enotation_decimal_large(self): + """This test automatically runs if requirements.precision_numerics_enotation_large is open()""" + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.IMPL_FLOAT_PREC, extra=True)) + def test_float_coerce_round_trip(self): + """ + This automatically runs if requirements.literal_float_coercion is open() + + Without additional work, Databricks returns 15.75629997253418 when you SELECT 15.7563. + This is a potential area where we could override the Float literal processor to add a CAST. + Will leave to a PM to decide if we should do so. + """ + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.IMPL_FLOAT_PREC, extra=True)) + def test_float_custom_scale(self): + """This test automatically runs if requirements.precision_generic_float_type is open()""" + pass + + +class HasTableTest(HasTableTest): + """Databricks does not support temporary tables.""" + + @pytest.mark.skip(render_skip_reason(SkipReason.TEMP_TBL)) + def test_has_table_temp_table(self): + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.TEMP_TBL, True, True)) + def test_has_table_temp_view(self): + """Databricks supports temporary views but this test depends on requirements.has_temp_table, which we + explicitly close so that we can run other tests in this group. See the comment under has_temp_table in + requirements.py for details. + + From what I can see, there is no way to run this test since it will fail during setup if we mark has_temp_table + open(). It _might_ be possible to hijack this behaviour by implementing temp_table_keyword_args in our own + provision.py. Doing so would mean creating a real table during this class setup instead of a temp table. Then + we could just skip the temp table tests but run the temp view tests. But this test fixture doesn't cleanup its + temp tables and has no hook to do so. + + It would be ideal for SQLAlchemy to define a separate requirements.has_temp_views. + """ + pass + + +class ComponentReflectionTestExtra(ComponentReflectionTestExtra): + @pytest.mark.skip(render_skip_reason(SkipReason.INDEXES)) + def test_reflect_covering_index(self): + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.INDEXES)) + def test_reflect_expression_based_indexes(self): + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.STRING_FEAT, extra=True)) + def test_varchar_reflection(self): + """Databricks doesn't enforce string length limitations like STRING(255).""" + pass + + +class InsertBehaviorTest(InsertBehaviorTest): + @pytest.mark.skip(render_skip_reason(SkipReason.AUTO_INC, True, True)) + def test_autoclose_on_insert(self): + """The setup for this test creates a column with implicit autoincrement enabled. + This dialect does not implement implicit autoincrement - users must declare Identity() explicitly. + """ + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.AUTO_INC, True, True)) + def test_insert_from_select_autoinc(self): + """Implicit autoincrement is not implemented in this dialect.""" + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.AUTO_INC, True, True)) + def test_insert_from_select_autoinc_no_rows(self): + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.RETURNING)) + def test_autoclose_on_insert_implicit_returning(self): + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(render_skip_reason(SkipReason.AUTO_INC, extra=True)) +class LastrowidTest(LastrowidTest): + """SQLAlchemy docs describe that a column without an explicit Identity() may implicitly create one if autoincrement=True. + That is what this method tests. Databricks supports auto-incrementing IDENTITY columns but they must be explicitly + declared. This limitation is present in our dialect as well. Which means that SQLAlchemy's autoincrement setting of a column + is ignored. We emit a logging.WARN message if you try it. + + In the future we could handle this autoincrement by implicitly calling the visit_identity_column() method of our DDLCompiler + when autoincrement=True. There is an example of this in the Microsoft SQL Server dialect: MSSDDLCompiler.get_column_specification + + For now, if you need to create a SQLAlchemy column with an auto-incrementing identity, you must set this explicitly in your column + definition by passing an Identity() to the column constructor. + """ + + pass + + +class CTETest(CTETest): + """During the teardown for this test block, it tries to drop a constraint that it never named which raises + a compilation error. This could point to poor constraint reflection but our other constraint reflection + tests pass. Requires investigation. + """ + + @pytest.mark.skip(render_skip_reason(SkipReason.CTE_FEAT, extra=True)) + def test_select_recursive_round_trip(self): + pass + + @pytest.mark.skip(render_skip_reason(SkipReason.CTE_FEAT, extra=True)) + def test_delete_scalar_subq_round_trip(self): + """Error received is [UNSUPPORTED_SUBQUERY_EXPRESSION_CATEGORY.MUST_AGGREGATE_CORRELATED_SCALAR_SUBQUERY] + + This suggests a limitation of the platform. But a workaround may be possible if customers require it. + """ + pass + + +@pytest.mark.reviewed +@pytest.mark.skip(render_skip_reason(SkipReason.TIMEZONE_OPT, True)) +class DateTimeTZTest(DateTimeTZTest): + """Test whether the sqlalchemy.DateTime() type can _optionally_ include timezone info. + This dialect maps DateTime() → TIMESTAMP, which _always_ includes tzinfo. + + Users can use databricks.sqlalchemy.TIMESTAMP_NTZ for a tzinfo-less timestamp. The SQLA docs + acknowledge this is expected for some dialects. + + https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.DateTime + """ + + pass diff --git a/src/databricks/sqlalchemy/test/conftest.py b/src/databricks/sqlalchemy/test/conftest.py new file mode 100644 index 00000000..ea43e8d3 --- /dev/null +++ b/src/databricks/sqlalchemy/test/conftest.py @@ -0,0 +1,13 @@ +from sqlalchemy.dialects import registry +import pytest + +registry.register("databricks", "databricks.sqlalchemy", "DatabricksDialect") +# sqlalchemy's dialect-testing machinery wants an entry like this. +# This seems to be based around dialects maybe having multiple drivers +# and wanting to test driver-specific URLs, but doesn't seem to make +# much sense for dialects with only one driver. +registry.register("databricks.databricks", "databricks.sqlalchemy", "DatabricksDialect") + +pytest.register_assert_rewrite("sqlalchemy.testing.assertions") + +from sqlalchemy.testing.plugin.pytestplugin import * diff --git a/src/databricks/sqlalchemy/test/overrides/_componentreflectiontest.py b/src/databricks/sqlalchemy/test/overrides/_componentreflectiontest.py new file mode 100644 index 00000000..a1f58fa6 --- /dev/null +++ b/src/databricks/sqlalchemy/test/overrides/_componentreflectiontest.py @@ -0,0 +1,189 @@ +"""The default test setup uses self-referential foreign keys and indexes for a test table. +We override to remove these assumptions. + +Note that test_multi_foreign_keys currently does not pass for all combinations due to +an ordering issue. The dialect returns the expected information. But this test makes assertions +on the order of the returned results. We can't guarantee that order at the moment. + +The test fixture actually tries to sort the outputs, but this sort isn't working. Will need +to follow-up on this later. +""" +import sqlalchemy as sa +from sqlalchemy.testing import config +from sqlalchemy.testing.schema import Column +from sqlalchemy.testing.schema import Table +from sqlalchemy import ForeignKey +from sqlalchemy import testing + +from sqlalchemy.testing.suite.test_reflection import ComponentReflectionTest + + +class ComponentReflectionTest(ComponentReflectionTest): # type: ignore + @classmethod + def define_reflected_tables(cls, metadata, schema): + if schema: + schema_prefix = schema + "." + else: + schema_prefix = "" + + if testing.requires.self_referential_foreign_keys.enabled: + parent_id_args = ( + ForeignKey( + "%susers.user_id" % schema_prefix, name="user_id_fk", use_alter=True + ), + ) + else: + parent_id_args = () + users = Table( + "users", + metadata, + Column("user_id", sa.INT, primary_key=True), + Column("test1", sa.CHAR(5), nullable=False), + Column("test2", sa.Float(), nullable=False), + Column("parent_user_id", sa.Integer, *parent_id_args), + sa.CheckConstraint( + "test2 > 0", + name="zz_test2_gt_zero", + comment="users check constraint", + ), + sa.CheckConstraint("test2 <= 1000"), + schema=schema, + test_needs_fk=True, + ) + + Table( + "dingalings", + metadata, + Column("dingaling_id", sa.Integer, primary_key=True), + Column( + "address_id", + sa.Integer, + ForeignKey( + "%semail_addresses.address_id" % schema_prefix, + name="zz_email_add_id_fg", + comment="di fk comment", + ), + ), + Column( + "id_user", + sa.Integer, + ForeignKey("%susers.user_id" % schema_prefix), + ), + Column("data", sa.String(30), unique=True), + sa.CheckConstraint( + "address_id > 0 AND address_id < 1000", + name="address_id_gt_zero", + ), + sa.UniqueConstraint( + "address_id", + "dingaling_id", + name="zz_dingalings_multiple", + comment="di unique comment", + ), + schema=schema, + test_needs_fk=True, + ) + Table( + "email_addresses", + metadata, + Column("address_id", sa.Integer), + Column("remote_user_id", sa.Integer, ForeignKey(users.c.user_id)), + Column("email_address", sa.String(20)), + sa.PrimaryKeyConstraint( + "address_id", name="email_ad_pk", comment="ea pk comment" + ), + schema=schema, + test_needs_fk=True, + ) + Table( + "comment_test", + metadata, + Column("id", sa.Integer, primary_key=True, comment="id comment"), + Column("data", sa.String(20), comment="data % comment"), + Column( + "d2", + sa.String(20), + comment=r"""Comment types type speedily ' " \ '' Fun!""", + ), + Column("d3", sa.String(42), comment="Comment\nwith\rescapes"), + schema=schema, + comment=r"""the test % ' " \ table comment""", + ) + Table( + "no_constraints", + metadata, + Column("data", sa.String(20)), + schema=schema, + comment="no\nconstraints\rhas\fescaped\vcomment", + ) + + if testing.requires.cross_schema_fk_reflection.enabled: + if schema is None: + Table( + "local_table", + metadata, + Column("id", sa.Integer, primary_key=True), + Column("data", sa.String(20)), + Column( + "remote_id", + ForeignKey("%s.remote_table_2.id" % testing.config.test_schema), + ), + test_needs_fk=True, + schema=config.db.dialect.default_schema_name, + ) + else: + Table( + "remote_table", + metadata, + Column("id", sa.Integer, primary_key=True), + Column( + "local_id", + ForeignKey( + "%s.local_table.id" % config.db.dialect.default_schema_name + ), + ), + Column("data", sa.String(20)), + schema=schema, + test_needs_fk=True, + ) + Table( + "remote_table_2", + metadata, + Column("id", sa.Integer, primary_key=True), + Column("data", sa.String(20)), + schema=schema, + test_needs_fk=True, + ) + + if testing.requires.index_reflection.enabled: + Index("users_t_idx", users.c.test1, users.c.test2, unique=True) + Index("users_all_idx", users.c.user_id, users.c.test2, users.c.test1) + + if not schema: + # test_needs_fk is at the moment to force MySQL InnoDB + noncol_idx_test_nopk = Table( + "noncol_idx_test_nopk", + metadata, + Column("q", sa.String(5)), + test_needs_fk=True, + ) + + noncol_idx_test_pk = Table( + "noncol_idx_test_pk", + metadata, + Column("id", sa.Integer, primary_key=True), + Column("q", sa.String(5)), + test_needs_fk=True, + ) + + if ( + testing.requires.indexes_with_ascdesc.enabled + and testing.requires.reflect_indexes_with_ascdesc.enabled + ): + Index("noncol_idx_nopk", noncol_idx_test_nopk.c.q.desc()) + Index("noncol_idx_pk", noncol_idx_test_pk.c.q.desc()) + + if testing.requires.view_column_reflection.enabled: + cls.define_views(metadata, schema) + if not schema and testing.requires.temp_table_reflection.enabled: + cls.define_temp_tables(metadata) diff --git a/src/databricks/sqlalchemy/test/overrides/_ctetest.py b/src/databricks/sqlalchemy/test/overrides/_ctetest.py new file mode 100644 index 00000000..3cdae036 --- /dev/null +++ b/src/databricks/sqlalchemy/test/overrides/_ctetest.py @@ -0,0 +1,33 @@ +"""The default test setup uses a self-referential foreign key. With our dialect this requires +`use_alter=True` and the fk constraint to be named. So we override this to make the test pass. +""" + +from sqlalchemy.testing.suite import CTETest + +from sqlalchemy.testing.schema import Column +from sqlalchemy.testing.schema import Table +from sqlalchemy import ForeignKey +from sqlalchemy import Integer +from sqlalchemy import String + + +class CTETest(CTETest): # type: ignore + @classmethod + def define_tables(cls, metadata): + Table( + "some_table", + metadata, + Column("id", Integer, primary_key=True), + Column("data", String(50)), + Column( + "parent_id", ForeignKey("some_table.id", name="fk_test", use_alter=True) + ), + ) + + Table( + "some_other_table", + metadata, + Column("id", Integer, primary_key=True), + Column("data", String(50)), + Column("parent_id", Integer), + ) diff --git a/src/databricks/sqlalchemy/test/test_suite.py b/src/databricks/sqlalchemy/test/test_suite.py new file mode 100644 index 00000000..2b40a432 --- /dev/null +++ b/src/databricks/sqlalchemy/test/test_suite.py @@ -0,0 +1,13 @@ +""" +The order of these imports is important. Test cases are imported first from SQLAlchemy, +then are overridden by our local skip markers in _regression, _unsupported, and _future. +""" + + +# type: ignore +# fmt: off +from sqlalchemy.testing.suite import * +from databricks.sqlalchemy.test._regression import * +from databricks.sqlalchemy.test._unsupported import * +from databricks.sqlalchemy.test._future import * +from databricks.sqlalchemy.test._extra import TinyIntegerTest, DateTimeTZTestCustom diff --git a/src/databricks/sqlalchemy/test_local/__init__.py b/src/databricks/sqlalchemy/test_local/__init__.py new file mode 100644 index 00000000..eca1cf55 --- /dev/null +++ b/src/databricks/sqlalchemy/test_local/__init__.py @@ -0,0 +1,5 @@ +""" +This module contains tests entirely maintained by Databricks. + +These tests do not rely on SQLAlchemy's custom test runner. +""" diff --git a/src/databricks/sqlalchemy/test_local/conftest.py b/src/databricks/sqlalchemy/test_local/conftest.py new file mode 100644 index 00000000..c8b350be --- /dev/null +++ b/src/databricks/sqlalchemy/test_local/conftest.py @@ -0,0 +1,44 @@ +import os +import pytest + + +@pytest.fixture(scope="session") +def host(): + return os.getenv("DATABRICKS_SERVER_HOSTNAME") + + +@pytest.fixture(scope="session") +def http_path(): + return os.getenv("DATABRICKS_HTTP_PATH") + + +@pytest.fixture(scope="session") +def access_token(): + return os.getenv("DATABRICKS_TOKEN") + + +@pytest.fixture(scope="session") +def ingestion_user(): + return os.getenv("DATABRICKS_USER") + + +@pytest.fixture(scope="session") +def catalog(): + return os.getenv("DATABRICKS_CATALOG") + + +@pytest.fixture(scope="session") +def schema(): + return os.getenv("DATABRICKS_SCHEMA", "default") + + +@pytest.fixture(scope="session", autouse=True) +def connection_details(host, http_path, access_token, ingestion_user, catalog, schema): + return { + "host": host, + "http_path": http_path, + "access_token": access_token, + "ingestion_user": ingestion_user, + "catalog": catalog, + "schema": schema, + } diff --git a/tests/sqlalchemy/demo_data/MOCK_DATA.xlsx b/src/databricks/sqlalchemy/test_local/e2e/MOCK_DATA.xlsx similarity index 100% rename from tests/sqlalchemy/demo_data/MOCK_DATA.xlsx rename to src/databricks/sqlalchemy/test_local/e2e/MOCK_DATA.xlsx diff --git a/src/databricks/sqlalchemy/test_local/e2e/test_basic.py b/src/databricks/sqlalchemy/test_local/e2e/test_basic.py new file mode 100644 index 00000000..ce0b5d89 --- /dev/null +++ b/src/databricks/sqlalchemy/test_local/e2e/test_basic.py @@ -0,0 +1,543 @@ +import datetime +import decimal +from typing import Tuple, Union, List +from unittest import skipIf + +import pytest +from sqlalchemy import ( + Column, + MetaData, + Table, + Text, + create_engine, + insert, + select, + text, +) +from sqlalchemy.engine import Engine +from sqlalchemy.engine.reflection import Inspector +from sqlalchemy.orm import DeclarativeBase, Mapped, Session, mapped_column +from sqlalchemy.schema import DropColumnComment, SetColumnComment +from sqlalchemy.types import BOOLEAN, DECIMAL, Date, Integer, String + +try: + from sqlalchemy.orm import declarative_base +except ImportError: + from sqlalchemy.ext.declarative import declarative_base + + +USER_AGENT_TOKEN = "PySQL e2e Tests" + + +def sqlalchemy_1_3(): + import sqlalchemy + + return sqlalchemy.__version__.startswith("1.3") + + +def version_agnostic_select(object_to_select, *args, **kwargs): + """ + SQLAlchemy==1.3.x requires arguments to select() to be a Python list + + https://docs.sqlalchemy.org/en/20/changelog/migration_14.html#orm-query-is-internally-unified-with-select-update-delete-2-0-style-execution-available + """ + + if sqlalchemy_1_3(): + return select([object_to_select], *args, **kwargs) + else: + return select(object_to_select, *args, **kwargs) + + +def version_agnostic_connect_arguments(connection_details) -> Tuple[str, dict]: + HOST = connection_details["host"] + HTTP_PATH = connection_details["http_path"] + ACCESS_TOKEN = connection_details["access_token"] + CATALOG = connection_details["catalog"] + SCHEMA = connection_details["schema"] + + ua_connect_args = {"_user_agent_entry": USER_AGENT_TOKEN} + + if sqlalchemy_1_3(): + conn_string = f"databricks://token:{ACCESS_TOKEN}@{HOST}" + connect_args = { + **ua_connect_args, + "http_path": HTTP_PATH, + "server_hostname": HOST, + "catalog": CATALOG, + "schema": SCHEMA, + } + + return conn_string, connect_args + else: + return ( + f"databricks://token:{ACCESS_TOKEN}@{HOST}?http_path={HTTP_PATH}&catalog={CATALOG}&schema={SCHEMA}", + ua_connect_args, + ) + + +@pytest.fixture +def db_engine(connection_details) -> Engine: + conn_string, connect_args = version_agnostic_connect_arguments(connection_details) + return create_engine(conn_string, connect_args=connect_args) + + +def run_query(db_engine: Engine, query: Union[str, Text]): + if not isinstance(query, Text): + _query = text(query) # type: ignore + else: + _query = query # type: ignore + with db_engine.begin() as conn: + return conn.execute(_query).fetchall() + + +@pytest.fixture +def samples_engine(connection_details) -> Engine: + details = connection_details.copy() + details["catalog"] = "samples" + details["schema"] = "nyctaxi" + conn_string, connect_args = version_agnostic_connect_arguments(details) + return create_engine(conn_string, connect_args=connect_args) + + +@pytest.fixture() +def base(db_engine): + return declarative_base() + + +@pytest.fixture() +def session(db_engine): + return Session(db_engine) + + +@pytest.fixture() +def metadata_obj(db_engine): + return MetaData() + + +def test_can_connect(db_engine): + simple_query = "SELECT 1" + result = run_query(db_engine, simple_query) + assert len(result) == 1 + + +def test_connect_args(db_engine): + """Verify that extra connect args passed to sqlalchemy.create_engine are passed to DBAPI + + This will most commonly happen when partners supply a user agent entry + """ + + conn = db_engine.connect() + connection_headers = conn.connection.thrift_backend._transport._headers + user_agent = connection_headers["User-Agent"] + + expected = f"(sqlalchemy + {USER_AGENT_TOKEN})" + assert expected in user_agent + + +@pytest.mark.skipif(sqlalchemy_1_3(), reason="Pandas requires SQLAlchemy >= 1.4") +@pytest.mark.skip( + reason="DBR is currently limited to 256 parameters per call to .execute(). Test cannot pass." +) +def test_pandas_upload(db_engine, metadata_obj): + import pandas as pd + + SCHEMA = "default" + try: + df = pd.read_excel( + "src/databricks/sqlalchemy/test_local/e2e/demo_data/MOCK_DATA.xlsx" + ) + df.to_sql( + "mock_data", + db_engine, + schema=SCHEMA, + index=False, + method="multi", + if_exists="replace", + ) + + df_after = pd.read_sql_table("mock_data", db_engine, schema=SCHEMA) + assert len(df) == len(df_after) + except Exception as e: + raise e + finally: + db_engine.execute("DROP TABLE mock_data") + + +def test_create_table_not_null(db_engine, metadata_obj: MetaData): + table_name = "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")) + + SampleTable = Table( + table_name, + metadata_obj, + Column("name", String(255)), + Column("episodes", Integer), + Column("some_bool", BOOLEAN, nullable=False), + ) + + metadata_obj.create_all(db_engine) + + columns = db_engine.dialect.get_columns( + connection=db_engine.connect(), table_name=table_name + ) + + name_column_description = columns[0] + some_bool_column_description = columns[2] + + assert name_column_description.get("nullable") is True + assert some_bool_column_description.get("nullable") is False + + metadata_obj.drop_all(db_engine) + + +def test_column_comment(db_engine, metadata_obj: MetaData): + table_name = "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")) + + column = Column("name", String(255), comment="some comment") + SampleTable = Table(table_name, metadata_obj, column) + + metadata_obj.create_all(db_engine) + connection = db_engine.connect() + + columns = db_engine.dialect.get_columns( + connection=connection, table_name=table_name + ) + + assert columns[0].get("comment") == "some comment" + + column.comment = "other comment" + connection.execute(SetColumnComment(column)) + + columns = db_engine.dialect.get_columns( + connection=connection, table_name=table_name + ) + + assert columns[0].get("comment") == "other comment" + + connection.execute(DropColumnComment(column)) + + columns = db_engine.dialect.get_columns( + connection=connection, table_name=table_name + ) + + assert columns[0].get("comment") == None + + metadata_obj.drop_all(db_engine) + + +def test_bulk_insert_with_core(db_engine, metadata_obj, session): + import random + + # Maximum number of parameter is 256. 256/4 == 64 + num_to_insert = 64 + + table_name = "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")) + + names = ["Bim", "Miki", "Sarah", "Ira"] + + SampleTable = Table( + table_name, metadata_obj, Column("name", String(255)), Column("number", Integer) + ) + + rows = [ + {"name": names[i % 3], "number": random.choice(range(64))} + for i in range(num_to_insert) + ] + + metadata_obj.create_all(db_engine) + with db_engine.begin() as conn: + conn.execute(insert(SampleTable).values(rows)) + + with db_engine.begin() as conn: + rows = conn.execute(version_agnostic_select(SampleTable)).fetchall() + + assert len(rows) == num_to_insert + + +def test_create_insert_drop_table_core(base, db_engine, metadata_obj: MetaData): + """ """ + + SampleTable = Table( + "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")), + metadata_obj, + Column("name", String(255)), + Column("episodes", Integer), + Column("some_bool", BOOLEAN), + Column("dollars", DECIMAL(10, 2)), + ) + + metadata_obj.create_all(db_engine) + + insert_stmt = insert(SampleTable).values( + name="Bim Adewunmi", episodes=6, some_bool=True, dollars=decimal.Decimal(125) + ) + + with db_engine.connect() as conn: + conn.execute(insert_stmt) + + select_stmt = version_agnostic_select(SampleTable) + with db_engine.begin() as conn: + resp = conn.execute(select_stmt) + + result = resp.fetchall() + + assert len(result) == 1 + + metadata_obj.drop_all(db_engine) + + +# ORM tests are made following this tutorial +# https://docs.sqlalchemy.org/en/14/orm/quickstart.html + + +@skipIf(False, "Unity catalog must be supported") +def test_create_insert_drop_table_orm(db_engine): + """ORM classes built on the declarative base class must have a primary key. + This is restricted to Unity Catalog. + """ + + class Base(DeclarativeBase): + pass + + class SampleObject(Base): + __tablename__ = "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")) + + name: Mapped[str] = mapped_column(String(255), primary_key=True) + episodes: Mapped[int] = mapped_column(Integer) + some_bool: Mapped[bool] = mapped_column(BOOLEAN) + + Base.metadata.create_all(db_engine) + + sample_object_1 = SampleObject(name="Bim Adewunmi", episodes=6, some_bool=True) + sample_object_2 = SampleObject(name="Miki Meek", episodes=12, some_bool=False) + + session = Session(db_engine) + session.add(sample_object_1) + session.add(sample_object_2) + session.flush() + + stmt = version_agnostic_select(SampleObject).where( + SampleObject.name.in_(["Bim Adewunmi", "Miki Meek"]) + ) + + if sqlalchemy_1_3(): + output = [i for i in session.execute(stmt)] + else: + output = [i for i in session.scalars(stmt)] + + assert len(output) == 2 + + Base.metadata.drop_all(db_engine) + + +def test_dialect_type_mappings(db_engine, metadata_obj: MetaData): + """Confirms that we get back the same time we declared in a model and inserted using Core""" + + class Base(DeclarativeBase): + pass + + SampleTable = Table( + "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")), + metadata_obj, + Column("string_example", String(255)), + Column("integer_example", Integer), + Column("boolean_example", BOOLEAN), + Column("decimal_example", DECIMAL(10, 2)), + Column("date_example", Date), + ) + + string_example = "" + integer_example = 100 + boolean_example = True + decimal_example = decimal.Decimal(125) + date_example = datetime.date(2013, 1, 1) + + metadata_obj.create_all(db_engine) + + insert_stmt = insert(SampleTable).values( + string_example=string_example, + integer_example=integer_example, + boolean_example=boolean_example, + decimal_example=decimal_example, + date_example=date_example, + ) + + with db_engine.connect() as conn: + conn.execute(insert_stmt) + + select_stmt = version_agnostic_select(SampleTable) + with db_engine.begin() as conn: + resp = conn.execute(select_stmt) + + result = resp.fetchall() + this_row = result[0] + + assert this_row.string_example == string_example + assert this_row.integer_example == integer_example + assert this_row.boolean_example == boolean_example + assert this_row.decimal_example == decimal_example + assert this_row.date_example == date_example + + metadata_obj.drop_all(db_engine) + + +def test_inspector_smoke_test(samples_engine: Engine): + """It does not appear that 3L namespace is supported here""" + + schema, table = "nyctaxi", "trips" + + try: + inspector = Inspector.from_engine(samples_engine) + except Exception as e: + assert False, f"Could not build inspector: {e}" + + # Expect six columns + columns = inspector.get_columns(table, schema=schema) + + # Expect zero views, but the method should return + views = inspector.get_view_names(schema=schema) + + assert ( + len(columns) == 6 + ), "Dialect did not find the expected number of columns in samples.nyctaxi.trips" + assert len(views) == 0, "Views could not be fetched" + + +@pytest.mark.skip(reason="engine.table_names has been removed in sqlalchemy verison 2") +def test_get_table_names_smoke_test(samples_engine: Engine): + with samples_engine.connect() as conn: + _names = samples_engine.table_names(schema="nyctaxi", connection=conn) # type: ignore + _names is not None, "get_table_names did not succeed" + + +def test_has_table_across_schemas( + db_engine: Engine, samples_engine: Engine, catalog: str, schema: str +): + """For this test to pass these conditions must be met: + - Table samples.nyctaxi.trips must exist + - Table samples.tpch.customer must exist + - The `catalog` and `schema` environment variables must be set and valid + """ + + with samples_engine.connect() as conn: + # 1) Check for table within schema declared at engine creation time + assert samples_engine.dialect.has_table(connection=conn, table_name="trips") + + # 2) Check for table within another schema in the same catalog + assert samples_engine.dialect.has_table( + connection=conn, table_name="customer", schema="tpch" + ) + + # 3) Check for a table within a different catalog + # Create a table in a different catalog + with db_engine.connect() as conn: + conn.execute(text("CREATE TABLE test_has_table (numbers_are_cool INT);")) + + try: + # Verify that this table is not found in the samples catalog + assert not samples_engine.dialect.has_table( + connection=conn, table_name="test_has_table" + ) + # Verify that this table is found in a separate catalog + assert samples_engine.dialect.has_table( + connection=conn, + table_name="test_has_table", + schema=schema, + catalog=catalog, + ) + finally: + conn.execute(text("DROP TABLE test_has_table;")) + + +def test_user_agent_adjustment(db_engine): + # If .connect() is called multiple times on an engine, don't keep pre-pending the user agent + # https://github.com/databricks/databricks-sql-python/issues/192 + c1 = db_engine.connect() + c2 = db_engine.connect() + + def get_conn_user_agent(conn): + return conn.connection.dbapi_connection.thrift_backend._transport._headers.get( + "User-Agent" + ) + + ua1 = get_conn_user_agent(c1) + ua2 = get_conn_user_agent(c2) + same_ua = ua1 == ua2 + + c1.close() + c2.close() + + assert same_ua, f"User agents didn't match \n {ua1} \n {ua2}" + + +@pytest.fixture +def sample_table(metadata_obj: MetaData, db_engine: Engine): + """This fixture creates a sample table and cleans it up after the test is complete.""" + from databricks.sqlalchemy._parse import GET_COLUMNS_TYPE_MAP + + table_name = "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")) + + args: List[Column] = [ + Column(colname, coltype) for colname, coltype in GET_COLUMNS_TYPE_MAP.items() + ] + + SampleTable = Table(table_name, metadata_obj, *args) + + metadata_obj.create_all(db_engine) + + yield table_name + + metadata_obj.drop_all(db_engine) + + +def test_get_columns(db_engine, sample_table: str): + """Created after PECO-1297 and Github Issue #295 to verify that get_columsn behaves like it should for all known SQLAlchemy types""" + + inspector = Inspector.from_engine(db_engine) + + # this raises an exception if `parse_column_info_from_tgetcolumnsresponse` fails a lookup + columns = inspector.get_columns(sample_table) + + assert True + + +class TestCommentReflection: + @pytest.fixture(scope="class") + def engine(self, connection_details: dict): + HOST = connection_details["host"] + HTTP_PATH = connection_details["http_path"] + ACCESS_TOKEN = connection_details["access_token"] + CATALOG = connection_details["catalog"] + SCHEMA = connection_details["schema"] + + connection_string = f"databricks://token:{ACCESS_TOKEN}@{HOST}?http_path={HTTP_PATH}&catalog={CATALOG}&schema={SCHEMA}" + connect_args = {"_user_agent_entry": USER_AGENT_TOKEN} + + engine = create_engine(connection_string, connect_args=connect_args) + return engine + + @pytest.fixture + def inspector(self, engine: Engine) -> Inspector: + return Inspector.from_engine(engine) + + @pytest.fixture(scope="class") + def table(self, engine): + md = MetaData() + tbl = Table( + "foo", + md, + Column("bar", String, comment="column comment"), + comment="table comment", + ) + md.create_all(bind=engine) + + yield tbl + + md.drop_all(bind=engine) + + def test_table_comment_reflection(self, inspector: Inspector, table: Table): + comment = inspector.get_table_comment(table.name) + assert comment == {"text": "table comment"} + + def test_column_comment(self, inspector: Inspector, table: Table): + result = inspector.get_columns(table.name)[0].get("comment") + assert result == "column comment" diff --git a/src/databricks/sqlalchemy/test_local/test_ddl.py b/src/databricks/sqlalchemy/test_local/test_ddl.py new file mode 100644 index 00000000..f596dffa --- /dev/null +++ b/src/databricks/sqlalchemy/test_local/test_ddl.py @@ -0,0 +1,96 @@ +import pytest +from sqlalchemy import Column, MetaData, String, Table, create_engine +from sqlalchemy.schema import ( + CreateTable, + DropColumnComment, + DropTableComment, + SetColumnComment, + SetTableComment, +) + + +class DDLTestBase: + engine = create_engine( + "databricks://token:****@****?http_path=****&catalog=****&schema=****" + ) + + def compile(self, stmt): + return str(stmt.compile(bind=self.engine)) + + +class TestColumnCommentDDL(DDLTestBase): + @pytest.fixture + def metadata(self) -> MetaData: + """Assemble a metadata object with one table containing one column.""" + metadata = MetaData() + + column = Column("foo", String, comment="bar") + table = Table("foobar", metadata, column) + + return metadata + + @pytest.fixture + def table(self, metadata) -> Table: + return metadata.tables.get("foobar") + + @pytest.fixture + def column(self, table) -> Column: + return table.columns[0] + + def test_create_table_with_column_comment(self, table): + stmt = CreateTable(table) + output = self.compile(stmt) + + # output is a CREATE TABLE statement + assert "foo STRING COMMENT 'bar'" in output + + def test_alter_table_add_column_comment(self, column): + stmt = SetColumnComment(column) + output = self.compile(stmt) + assert output == "ALTER TABLE foobar ALTER COLUMN foo COMMENT 'bar'" + + def test_alter_table_drop_column_comment(self, column): + stmt = DropColumnComment(column) + output = self.compile(stmt) + assert output == "ALTER TABLE foobar ALTER COLUMN foo COMMENT ''" + + +class TestTableCommentDDL(DDLTestBase): + @pytest.fixture + def metadata(self) -> MetaData: + """Assemble a metadata object with one table containing one column.""" + metadata = MetaData() + + col1 = Column("foo", String) + col2 = Column("foo", String) + tbl_w_comment = Table("martin", metadata, col1, comment="foobar") + tbl_wo_comment = Table("prs", metadata, col2) + + return metadata + + @pytest.fixture + def table_with_comment(self, metadata) -> Table: + return metadata.tables.get("martin") + + @pytest.fixture + def table_without_comment(self, metadata) -> Table: + return metadata.tables.get("prs") + + def test_create_table_with_comment(self, table_with_comment): + stmt = CreateTable(table_with_comment) + output = self.compile(stmt) + assert "USING DELTA" in output + assert "COMMENT 'foobar'" in output + + def test_alter_table_add_comment(self, table_without_comment: Table): + table_without_comment.comment = "wireless mechanical keyboard" + stmt = SetTableComment(table_without_comment) + output = self.compile(stmt) + + assert output == "COMMENT ON TABLE prs IS 'wireless mechanical keyboard'" + + def test_alter_table_drop_comment(self, table_with_comment): + """The syntax for COMMENT ON is here: https://docs.databricks.com/en/sql/language-manual/sql-ref-syntax-ddl-comment.html""" + stmt = DropTableComment(table_with_comment) + output = self.compile(stmt) + assert output == "COMMENT ON TABLE martin IS NULL" diff --git a/src/databricks/sqlalchemy/test_local/test_parsing.py b/src/databricks/sqlalchemy/test_local/test_parsing.py new file mode 100644 index 00000000..c8ab443d --- /dev/null +++ b/src/databricks/sqlalchemy/test_local/test_parsing.py @@ -0,0 +1,160 @@ +import pytest +from databricks.sqlalchemy._parse import ( + extract_identifiers_from_string, + extract_identifier_groups_from_string, + extract_three_level_identifier_from_constraint_string, + build_fk_dict, + build_pk_dict, + match_dte_rows_by_value, + get_comment_from_dte_output, + DatabricksSqlAlchemyParseException, +) + + +# These are outputs from DESCRIBE TABLE EXTENDED +@pytest.mark.parametrize( + "input, expected", + [ + ("PRIMARY KEY (`pk1`, `pk2`)", ["pk1", "pk2"]), + ("PRIMARY KEY (`a`, `b`, `c`)", ["a", "b", "c"]), + ("PRIMARY KEY (`name`, `id`, `attr`)", ["name", "id", "attr"]), + ], +) +def test_extract_identifiers(input, expected): + assert ( + extract_identifiers_from_string(input) == expected + ), "Failed to extract identifiers from string" + + +@pytest.mark.parametrize( + "input, expected", + [ + ( + "FOREIGN KEY (`pname`, `pid`, `pattr`) REFERENCES `main`.`pysql_sqlalchemy`.`tb1` (`name`, `id`, `attr`)", + [ + "(`pname`, `pid`, `pattr`)", + "(`name`, `id`, `attr`)", + ], + ) + ], +) +def test_extract_identifer_batches(input, expected): + assert ( + extract_identifier_groups_from_string(input) == expected + ), "Failed to extract identifier groups from string" + + +def test_extract_3l_namespace_from_constraint_string(): + input = "FOREIGN KEY (`parent_user_id`) REFERENCES `main`.`pysql_dialect_compliance`.`users` (`user_id`)" + expected = { + "catalog": "main", + "schema": "pysql_dialect_compliance", + "table": "users", + } + + assert ( + extract_three_level_identifier_from_constraint_string(input) == expected + ), "Failed to extract 3L namespace from constraint string" + + +def test_extract_3l_namespace_from_bad_constraint_string(): + input = "FOREIGN KEY (`parent_user_id`) REFERENCES `pysql_dialect_compliance`.`users` (`user_id`)" + + with pytest.raises(DatabricksSqlAlchemyParseException): + extract_three_level_identifier_from_constraint_string(input) + + +@pytest.mark.parametrize("tschema", [None, "some_schema"]) +def test_build_fk_dict(tschema): + fk_constraint_string = "FOREIGN KEY (`parent_user_id`) REFERENCES `main`.`some_schema`.`users` (`user_id`)" + + result = build_fk_dict("some_fk_name", fk_constraint_string, schema_name=tschema) + + assert result == { + "name": "some_fk_name", + "constrained_columns": ["parent_user_id"], + "referred_schema": tschema, + "referred_table": "users", + "referred_columns": ["user_id"], + } + + +def test_build_pk_dict(): + pk_constraint_string = "PRIMARY KEY (`id`, `name`, `email_address`)" + pk_name = "pk1" + + result = build_pk_dict(pk_name, pk_constraint_string) + + assert result == { + "constrained_columns": ["id", "name", "email_address"], + "name": "pk1", + } + + +# This is a real example of the output from DESCRIBE TABLE EXTENDED as of 15 October 2023 +RAW_SAMPLE_DTE_OUTPUT = [ + ["id", "int"], + ["name", "string"], + ["", ""], + ["# Detailed Table Information", ""], + ["Catalog", "main"], + ["Database", "pysql_sqlalchemy"], + ["Table", "exampleexampleexample"], + ["Created Time", "Sun Oct 15 21:12:54 UTC 2023"], + ["Last Access", "UNKNOWN"], + ["Created By", "Spark "], + ["Type", "MANAGED"], + ["Location", "s3://us-west-2-****-/19a85dee-****/tables/ccb7***"], + ["Provider", "delta"], + ["Comment", "some comment"], + ["Owner", "some.user@example.com"], + ["Is_managed_location", "true"], + ["Predictive Optimization", "ENABLE (inherited from CATALOG main)"], + [ + "Table Properties", + "[delta.checkpoint.writeStatsAsJson=false,delta.checkpoint.writeStatsAsStruct=true,delta.minReaderVersion=1,delta.minWriterVersion=2]", + ], + ["", ""], + ["# Constraints", ""], + ["exampleexampleexample_pk", "PRIMARY KEY (`id`)"], + [ + "exampleexampleexample_fk", + "FOREIGN KEY (`parent_user_id`) REFERENCES `main`.`pysql_dialect_compliance`.`users` (`user_id`)", + ], +] + +FMT_SAMPLE_DT_OUTPUT = [ + {"col_name": i[0], "data_type": i[1]} for i in RAW_SAMPLE_DTE_OUTPUT +] + + +@pytest.mark.parametrize( + "match, output", + [ + ( + "PRIMARY KEY", + [ + { + "col_name": "exampleexampleexample_pk", + "data_type": "PRIMARY KEY (`id`)", + } + ], + ), + ( + "FOREIGN KEY", + [ + { + "col_name": "exampleexampleexample_fk", + "data_type": "FOREIGN KEY (`parent_user_id`) REFERENCES `main`.`pysql_dialect_compliance`.`users` (`user_id`)", + } + ], + ), + ], +) +def test_filter_dict_by_value(match, output): + result = match_dte_rows_by_value(FMT_SAMPLE_DT_OUTPUT, match) + assert result == output + + +def test_get_comment_from_dte_output(): + assert get_comment_from_dte_output(FMT_SAMPLE_DT_OUTPUT) == "some comment" diff --git a/src/databricks/sqlalchemy/test_local/test_types.py b/src/databricks/sqlalchemy/test_local/test_types.py new file mode 100644 index 00000000..b91217ed --- /dev/null +++ b/src/databricks/sqlalchemy/test_local/test_types.py @@ -0,0 +1,161 @@ +import enum + +import pytest +import sqlalchemy + +from databricks.sqlalchemy.base import DatabricksDialect +from databricks.sqlalchemy._types import TINYINT, TIMESTAMP, TIMESTAMP_NTZ + + +class DatabricksDataType(enum.Enum): + """https://docs.databricks.com/en/sql/language-manual/sql-ref-datatypes.html""" + + BIGINT = enum.auto() + BINARY = enum.auto() + BOOLEAN = enum.auto() + DATE = enum.auto() + DECIMAL = enum.auto() + DOUBLE = enum.auto() + FLOAT = enum.auto() + INT = enum.auto() + INTERVAL = enum.auto() + VOID = enum.auto() + SMALLINT = enum.auto() + STRING = enum.auto() + TIMESTAMP = enum.auto() + TIMESTAMP_NTZ = enum.auto() + TINYINT = enum.auto() + ARRAY = enum.auto() + MAP = enum.auto() + STRUCT = enum.auto() + + +# Defines the way that SQLAlchemy CamelCase types are compiled into Databricks SQL types. +# Note: I wish I could define this within the TestCamelCaseTypesCompilation class, but pytest doesn't like that. +camel_case_type_map = { + sqlalchemy.types.BigInteger: DatabricksDataType.BIGINT, + sqlalchemy.types.LargeBinary: DatabricksDataType.BINARY, + sqlalchemy.types.Boolean: DatabricksDataType.BOOLEAN, + sqlalchemy.types.Date: DatabricksDataType.DATE, + sqlalchemy.types.DateTime: DatabricksDataType.TIMESTAMP_NTZ, + sqlalchemy.types.Double: DatabricksDataType.DOUBLE, + sqlalchemy.types.Enum: DatabricksDataType.STRING, + sqlalchemy.types.Float: DatabricksDataType.FLOAT, + sqlalchemy.types.Integer: DatabricksDataType.INT, + sqlalchemy.types.Interval: DatabricksDataType.TIMESTAMP_NTZ, + sqlalchemy.types.Numeric: DatabricksDataType.DECIMAL, + sqlalchemy.types.PickleType: DatabricksDataType.BINARY, + sqlalchemy.types.SmallInteger: DatabricksDataType.SMALLINT, + sqlalchemy.types.String: DatabricksDataType.STRING, + sqlalchemy.types.Text: DatabricksDataType.STRING, + sqlalchemy.types.Time: DatabricksDataType.STRING, + sqlalchemy.types.Unicode: DatabricksDataType.STRING, + sqlalchemy.types.UnicodeText: DatabricksDataType.STRING, + sqlalchemy.types.Uuid: DatabricksDataType.STRING, +} + + +def dict_as_tuple_list(d: dict): + """Return a list of [(key, value), ...] from a dictionary.""" + return [(key, value) for key, value in d.items()] + + +class CompilationTestBase: + dialect = DatabricksDialect() + + def _assert_compiled_value( + self, type_: sqlalchemy.types.TypeEngine, expected: DatabricksDataType + ): + """Assert that when type_ is compiled for the databricks dialect, it renders the DatabricksDataType name. + + This method initialises the type_ with no arguments. + """ + compiled_result = type_().compile(dialect=self.dialect) # type: ignore + assert compiled_result == expected.name + + def _assert_compiled_value_explicit( + self, type_: sqlalchemy.types.TypeEngine, expected: str + ): + """Assert that when type_ is compiled for the databricks dialect, it renders the expected string. + + This method expects an initialised type_ so that we can test how a TypeEngine created with arguments + is compiled. + """ + compiled_result = type_.compile(dialect=self.dialect) + assert compiled_result == expected + + +class TestCamelCaseTypesCompilation(CompilationTestBase): + """Per the sqlalchemy documentation[^1] here, the camel case members of sqlalchemy.types are + are expected to work across all dialects. These tests verify that the types compile into valid + Databricks SQL type strings. For example, the sqlalchemy.types.Integer() should compile as "INT". + + Truly custom types like STRUCT (notice the uppercase) are not expected to work across all dialects. + We test these separately. + + Note that these tests have to do with type **name** compiliation. Which is separate from actually + mapping values between Python and Databricks. + + Note: SchemaType and MatchType are not tested because it's not used in table definitions + + [1]: https://docs.sqlalchemy.org/en/20/core/type_basics.html#generic-camelcase-types + """ + + @pytest.mark.parametrize("type_, expected", dict_as_tuple_list(camel_case_type_map)) + def test_bare_camel_case_types_compile(self, type_, expected): + self._assert_compiled_value(type_, expected) + + def test_numeric_renders_as_decimal_with_precision(self): + self._assert_compiled_value_explicit( + sqlalchemy.types.Numeric(10), "DECIMAL(10)" + ) + + def test_numeric_renders_as_decimal_with_precision_and_scale(self): + self._assert_compiled_value_explicit( + sqlalchemy.types.Numeric(10, 2), "DECIMAL(10, 2)" + ) + + +uppercase_type_map = { + sqlalchemy.types.ARRAY: DatabricksDataType.ARRAY, + sqlalchemy.types.BIGINT: DatabricksDataType.BIGINT, + sqlalchemy.types.BINARY: DatabricksDataType.BINARY, + sqlalchemy.types.BOOLEAN: DatabricksDataType.BOOLEAN, + sqlalchemy.types.DATE: DatabricksDataType.DATE, + sqlalchemy.types.DECIMAL: DatabricksDataType.DECIMAL, + sqlalchemy.types.DOUBLE: DatabricksDataType.DOUBLE, + sqlalchemy.types.FLOAT: DatabricksDataType.FLOAT, + sqlalchemy.types.INT: DatabricksDataType.INT, + sqlalchemy.types.SMALLINT: DatabricksDataType.SMALLINT, + sqlalchemy.types.TIMESTAMP: DatabricksDataType.TIMESTAMP, + TINYINT: DatabricksDataType.TINYINT, + TIMESTAMP: DatabricksDataType.TIMESTAMP, + TIMESTAMP_NTZ: DatabricksDataType.TIMESTAMP_NTZ, +} + + +class TestUppercaseTypesCompilation(CompilationTestBase): + """Per the sqlalchemy documentation[^1], uppercase types are considered to be specific to some + database backends. These tests verify that the types compile into valid Databricks SQL type strings. + + [1]: https://docs.sqlalchemy.org/en/20/core/type_basics.html#backend-specific-uppercase-datatypes + """ + + @pytest.mark.parametrize("type_, expected", dict_as_tuple_list(uppercase_type_map)) + def test_bare_uppercase_types_compile(self, type_, expected): + if isinstance(type_, type(sqlalchemy.types.ARRAY)): + # ARRAY cannot be initialised without passing an item definition so we test separately + # I preserve it in the uppercase_type_map for clarity + assert True + else: + self._assert_compiled_value(type_, expected) + + def test_array_string_renders_as_array_of_string(self): + """SQLAlchemy's ARRAY type requires an item definition. And their docs indicate that they've only tested + it with Postgres since that's the only first-class dialect with support for ARRAY. + + https://docs.sqlalchemy.org/en/20/core/type_basics.html#sqlalchemy.types.ARRAY + """ + self._assert_compiled_value_explicit( + sqlalchemy.types.ARRAY(sqlalchemy.types.String), "ARRAY" + ) diff --git a/test.env.example b/test.env.example new file mode 100644 index 00000000..f99abc9d --- /dev/null +++ b/test.env.example @@ -0,0 +1,11 @@ +# Authentication details for running e2e tests +DATABRICKS_SERVER_HOSTNAME= +DATABRICKS_HTTP_PATH= +DATABRICKS_TOKEN= + +# Only required to run the PySQLStagingIngestionTestSuite +DATABRICKS_USER= + +# Only required to run SQLAlchemy tests +DATABRICKS_CATALOG= +DATABRICKS_SCHEMA= diff --git a/tests/e2e/__init__.py b/tests/e2e/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/tests/e2e/common/core_tests.py b/tests/e2e/common/core_tests.py index cd325e8d..e89289ef 100644 --- a/tests/e2e/common/core_tests.py +++ b/tests/e2e/common/core_tests.py @@ -3,14 +3,17 @@ from collections import namedtuple TypeFailure = namedtuple( - "TypeFailure", "query,columnType,resultType,resultValue," - "actualValue,actualType,description,conf") + "TypeFailure", + "query,columnType,resultType,resultValue," "actualValue,actualType,description,conf", +) ResultFailure = namedtuple( - "ResultFailure", "query,columnType,resultType,resultValue," - "actualValue,actualType,description,conf") + "ResultFailure", + "query,columnType,resultType,resultValue," "actualValue,actualType,description,conf", +) ExecFailure = namedtuple( - "ExecFailure", "query,columnType,resultType,resultValue," - "actualValue,actualType,description,conf,error") + "ExecFailure", + "query,columnType,resultType,resultValue," "actualValue,actualType,description,conf,error", +) class SmokeTestMixin: @@ -18,8 +21,8 @@ def test_smoke_test(self): with self.cursor() as cursor: cursor.execute("select 0") rows = cursor.fetchall() - self.assertEqual(len(rows), 1) - self.assertEqual(rows[0][0], 0) + assert len(rows) == 1 + assert rows[0][0] == 0 class CoreTestMixin: @@ -32,69 +35,109 @@ class CoreTestMixin: # A list of (subquery, column_type, python_type, expected_result) # To be executed as "SELECT {} FROM RANGE(...)" and "SELECT {}" range_queries = [ - ("TRUE", 'boolean', bool, True), - ("cast(1 AS TINYINT)", 'byte', int, 1), - ("cast(1000 AS SMALLINT)", 'short', int, 1000), - ("cast(100000 AS INTEGER)", 'integer', int, 100000), - ("cast(10000000000000 AS BIGINT)", 'long', int, 10000000000000), - ("cast(100.001 AS DECIMAL(6, 3))", 'decimal', decimal.Decimal, 100.001), - ("date '2020-02-20'", 'date', datetime.date, datetime.date(2020, 2, 20)), - ("unhex('f000')", 'binary', bytes, b'\xf0\x00'), # pyodbc internal mismatch - ("'foo'", 'string', str, 'foo'), + ("TRUE", "boolean", bool, True), + ("cast(1 AS TINYINT)", "byte", int, 1), + ("cast(1000 AS SMALLINT)", "short", int, 1000), + ("cast(100000 AS INTEGER)", "integer", int, 100000), + ("cast(10000000000000 AS BIGINT)", "long", int, 10000000000000), + ("cast(100.001 AS DECIMAL(6, 3))", "decimal", decimal.Decimal, 100.001), + ("date '2020-02-20'", "date", datetime.date, datetime.date(2020, 2, 20)), + ("unhex('f000')", "binary", bytes, b"\xf0\x00"), # pyodbc internal mismatch + ("'foo'", "string", str, "foo"), # SPARK-32130: 6.x: "4 weeks 2 days" vs 7.x: "30 days" # ("interval 30 days", str, str, "interval 4 weeks 2 days"), # ("interval 3 days", str, str, "interval 3 days"), - ("CAST(NULL AS DOUBLE)", 'double', type(None), None), + ("CAST(NULL AS DOUBLE)", "double", type(None), None), ] # Full queries, only the first column of the first row is checked - queries = [("NULL UNION (SELECT 1) order by 1", 'integer', type(None), None)] + queries = [("NULL UNION (SELECT 1) order by 1", "integer", type(None), None)] def run_tests_on_queries(self, default_conf): failures = [] - for (query, columnType, rowValueType, answer) in self.range_queries: + for query, columnType, rowValueType, answer in self.range_queries: with self.cursor(default_conf) as cursor: failures.extend( - self.run_query(cursor, query, columnType, rowValueType, answer, default_conf)) + self.run_query(cursor, query, columnType, rowValueType, answer, default_conf) + ) failures.extend( - self.run_range_query(cursor, query, columnType, rowValueType, answer, - default_conf)) + self.run_range_query( + cursor, query, columnType, rowValueType, answer, default_conf + ) + ) - for (query, columnType, rowValueType, answer) in self.queries: + for query, columnType, rowValueType, answer in self.queries: with self.cursor(default_conf) as cursor: failures.extend( - self.run_query(cursor, query, columnType, rowValueType, answer, default_conf)) + self.run_query(cursor, query, columnType, rowValueType, answer, default_conf) + ) if failures: - self.fail("Failed testing result set with Arrow. " - "Failed queries: {}".format("\n\n".join([str(f) for f in failures]))) + self.fail( + "Failed testing result set with Arrow. " + "Failed queries: {}".format("\n\n".join([str(f) for f in failures])) + ) def run_query(self, cursor, query, columnType, rowValueType, answer, conf): full_query = "SELECT {}".format(query) expected_column_types = self.expected_column_types(columnType) try: cursor.execute(full_query) - (result, ) = cursor.fetchone() + (result,) = cursor.fetchone() if not all(cursor.description[0][1] == type for type in expected_column_types): return [ - TypeFailure(full_query, expected_column_types, rowValueType, answer, result, - type(result), cursor.description, conf) + TypeFailure( + full_query, + expected_column_types, + rowValueType, + answer, + result, + type(result), + cursor.description, + conf, + ) ] if self.validate_row_value_type and type(result) is not rowValueType: return [ - TypeFailure(full_query, expected_column_types, rowValueType, answer, result, - type(result), cursor.description, conf) + TypeFailure( + full_query, + expected_column_types, + rowValueType, + answer, + result, + type(result), + cursor.description, + conf, + ) ] if self.validate_result and str(answer) != str(result): return [ - ResultFailure(full_query, query, expected_column_types, rowValueType, answer, - result, type(result), cursor.description, conf) + ResultFailure( + full_query, + query, + expected_column_types, + rowValueType, + answer, + result, + type(result), + cursor.description, + conf, + ) ] return [] except Exception as e: return [ - ExecFailure(full_query, columnType, rowValueType, None, None, None, - cursor.description, conf, e) + ExecFailure( + full_query, + columnType, + rowValueType, + None, + None, + None, + cursor.description, + conf, + e, + ) ] def run_range_query(self, cursor, query, columnType, rowValueType, expected, conf): @@ -109,23 +152,55 @@ def run_range_query(self, cursor, query, columnType, rowValueType, expected, con for index, (result, id) in enumerate(rows): if not all(cursor.description[0][1] == type for type in expected_column_types): return [ - TypeFailure(full_query, expected_column_types, rowValueType, expected, - result, type(result), cursor.description, conf) + TypeFailure( + full_query, + expected_column_types, + rowValueType, + expected, + result, + type(result), + cursor.description, + conf, + ) ] - if self.validate_row_value_type and type(result) \ - is not rowValueType: + if self.validate_row_value_type and type(result) is not rowValueType: return [ - TypeFailure(full_query, expected_column_types, rowValueType, expected, - result, type(result), cursor.description, conf) + TypeFailure( + full_query, + expected_column_types, + rowValueType, + expected, + result, + type(result), + cursor.description, + conf, + ) ] if self.validate_result and str(expected) != str(result): return [ - ResultFailure(full_query, expected_column_types, rowValueType, expected, - result, type(result), cursor.description, conf) + ResultFailure( + full_query, + expected_column_types, + rowValueType, + expected, + result, + type(result), + cursor.description, + conf, + ) ] return [] except Exception as e: return [ - ExecFailure(full_query, columnType, rowValueType, None, None, None, - cursor.description, conf, e) + ExecFailure( + full_query, + columnType, + rowValueType, + None, + None, + None, + cursor.description, + conf, + e, + ) ] diff --git a/tests/e2e/common/decimal_tests.py b/tests/e2e/common/decimal_tests.py index 8051d2a1..5005cdf1 100644 --- a/tests/e2e/common/decimal_tests.py +++ b/tests/e2e/common/decimal_tests.py @@ -1,6 +1,7 @@ from decimal import Decimal import pyarrow +import pytest class DecimalTestsMixin: @@ -9,7 +10,7 @@ class DecimalTestsMixin: ("1000000.0000 AS DECIMAL(11, 4)", Decimal("1000000.0000"), pyarrow.decimal128(11, 4)), ("-10.2343 AS DECIMAL(10, 6)", Decimal("-10.234300"), pyarrow.decimal128(10, 6)), # TODO(SC-90767): Re-enable this test after we have a way of passing `ansi_mode` = False - #("-13872347.2343 AS DECIMAL(10, 10)", None, pyarrow.decimal128(10, 10)), + # ("-13872347.2343 AS DECIMAL(10, 10)", None, pyarrow.decimal128(10, 10)), ("NULL AS DECIMAL(1, 1)", None, pyarrow.decimal128(1, 1)), ("1 AS DECIMAL(1, 0)", Decimal("1"), pyarrow.decimal128(1, 0)), ("0.00000 AS DECIMAL(5, 3)", Decimal("0.000"), pyarrow.decimal128(5, 3)), @@ -17,32 +18,36 @@ class DecimalTestsMixin: ] multi_decimals_and_expected_results = [ - (["1 AS DECIMAL(6, 3)", "100.001 AS DECIMAL(6, 3)", "NULL AS DECIMAL(6, 3)"], - [Decimal("1.00"), Decimal("100.001"), None], pyarrow.decimal128(6, 3)), - (["1 AS DECIMAL(6, 3)", "2 AS DECIMAL(5, 2)"], [Decimal('1.000'), - Decimal('2.000')], pyarrow.decimal128(6, - 3)), + ( + ["1 AS DECIMAL(6, 3)", "100.001 AS DECIMAL(6, 3)", "NULL AS DECIMAL(6, 3)"], + [Decimal("1.00"), Decimal("100.001"), None], + pyarrow.decimal128(6, 3), + ), + ( + ["1 AS DECIMAL(6, 3)", "2 AS DECIMAL(5, 2)"], + [Decimal("1.000"), Decimal("2.000")], + pyarrow.decimal128(6, 3), + ), ] - def test_decimals(self): + @pytest.mark.parametrize("decimal, expected_value, expected_type", decimal_and_expected_results) + def test_decimals(self, decimal, expected_value, expected_type): with self.cursor({}) as cursor: - for (decimal, expected_value, expected_type) in self.decimal_and_expected_results: - query = "SELECT CAST ({})".format(decimal) - with self.subTest(query=query): - cursor.execute(query) - table = cursor.fetchmany_arrow(1) - self.assertEqual(table.field(0).type, expected_type) - self.assertEqual(table.to_pydict().popitem()[1][0], expected_value) + query = "SELECT CAST ({})".format(decimal) + cursor.execute(query) + table = cursor.fetchmany_arrow(1) + assert table.field(0).type == expected_type + assert table.to_pydict().popitem()[1][0] == expected_value - def test_multi_decimals(self): + @pytest.mark.parametrize( + "decimals, expected_values, expected_type", multi_decimals_and_expected_results + ) + def test_multi_decimals(self, decimals, expected_values, expected_type): with self.cursor({}) as cursor: - for (decimals, expected_values, - expected_type) in self.multi_decimals_and_expected_results: - union_str = " UNION ".join(["(SELECT CAST ({}))".format(dec) for dec in decimals]) - query = "SELECT * FROM ({}) ORDER BY 1 NULLS LAST".format(union_str) + union_str = " UNION ".join(["(SELECT CAST ({}))".format(dec) for dec in decimals]) + query = "SELECT * FROM ({}) ORDER BY 1 NULLS LAST".format(union_str) - with self.subTest(query=query): - cursor.execute(query) - table = cursor.fetchall_arrow() - self.assertEqual(table.field(0).type, expected_type) - self.assertEqual(table.to_pydict().popitem()[1], expected_values) + cursor.execute(query) + table = cursor.fetchall_arrow() + assert table.field(0).type == expected_type + assert table.to_pydict().popitem()[1] == expected_values diff --git a/tests/e2e/common/large_queries_mixin.py b/tests/e2e/common/large_queries_mixin.py index 3e1e45bc..9ebc3f01 100644 --- a/tests/e2e/common/large_queries_mixin.py +++ b/tests/e2e/common/large_queries_mixin.py @@ -35,8 +35,10 @@ def fetch_rows(self, cursor, row_count, fetchmany_size): num_fetches = max(math.ceil(n / 10000), 1) latency_ms = int((time.time() - start_time) * 1000 / num_fetches), 1 - print('Fetched {} rows with an avg latency of {} per fetch, '.format(n, latency_ms) + - 'assuming 10K fetch size.') + print( + "Fetched {} rows with an avg latency of {} per fetch, ".format(n, latency_ms) + + "assuming 10K fetch size." + ) def test_query_with_large_wide_result_set(self): resultSize = 300 * 1000 * 1000 # 300 MB @@ -50,14 +52,15 @@ def test_query_with_large_wide_result_set(self): self.arraysize = 1000 with self.cursor() as cursor: for lz4_compression in [False, True]: - cursor.connection.lz4_compression=lz4_compression + cursor.connection.lz4_compression = lz4_compression uuids = ", ".join(["uuid() uuid{}".format(i) for i in range(cols)]) - cursor.execute("SELECT id, {uuids} FROM RANGE({rows})".format(uuids=uuids, rows=rows)) - self.assertEqual(lz4_compression, cursor.active_result_set.lz4_compressed) + cursor.execute( + "SELECT id, {uuids} FROM RANGE({rows})".format(uuids=uuids, rows=rows) + ) + assert lz4_compression == cursor.active_result_set.lz4_compressed for row_id, row in enumerate(self.fetch_rows(cursor, rows, fetchmany_size)): - self.assertEqual(row[0], row_id) # Verify no rows are dropped in the middle. - self.assertEqual(len(row[1]), 36) - + assert row[0] == row_id # Verify no rows are dropped in the middle. + assert len(row[1]) == 36 def test_query_with_large_narrow_result_set(self): resultSize = 300 * 1000 * 1000 # 300 MB @@ -71,10 +74,10 @@ def test_query_with_large_narrow_result_set(self): with self.cursor() as cursor: cursor.execute("SELECT * FROM RANGE({rows})".format(rows=rows)) for row_id, row in enumerate(self.fetch_rows(cursor, rows, fetchmany_size)): - self.assertEqual(row[0], row_id) + assert row[0] == row_id def test_long_running_query(self): - """ Incrementally increase query size until it takes at least 5 minutes, + """Incrementally increase query size until it takes at least 5 minutes, and asserts that the query completes successfully. """ minutes = 60 @@ -85,20 +88,24 @@ def test_long_running_query(self): scale_factor = 1 with self.cursor() as cursor: while duration < min_duration: - self.assertLess(scale_factor, 512, msg="Detected infinite loop") + assert scale_factor < 512, "Detected infinite loop" start = time.time() - cursor.execute("""SELECT count(*) + cursor.execute( + """SELECT count(*) FROM RANGE({scale}) x JOIN RANGE({scale0}) y - ON from_unixtime(x.id * y.id, "yyyy-MM-dd") LIKE "%not%a%date%" - """.format(scale=scale_factor * scale0, scale0=scale0)) + ON from_unixtime(x.id * y.id, "yyyy-MM-dd") LIKE "%not%a%date%" + """.format( + scale=scale_factor * scale0, scale0=scale0 + ) + ) - n, = cursor.fetchone() - self.assertEqual(n, 0) + (n,) = cursor.fetchone() + assert n == 0 duration = time.time() - start current_fraction = duration / min_duration - print('Took {} s with scale factor={}'.format(duration, scale_factor)) + print("Took {} s with scale factor={}".format(duration, scale_factor)) # Extrapolate linearly to reach 5 min and add 50% padding to push over the limit scale_factor = math.ceil(1.5 * scale_factor / current_fraction) diff --git a/tests/e2e/common/predicates.py b/tests/e2e/common/predicates.py index 3450087f..88b14961 100644 --- a/tests/e2e/common/predicates.py +++ b/tests/e2e/common/predicates.py @@ -29,10 +29,10 @@ def test_some_pyhive_v1_stuff(): def is_endpoint_test(cli_args=None): - + # Currently only supporting tests against DBSQL Endpoints # So we don't read `is_endpoint_test` from the CLI args - return True + return True def compare_dbr_versions(cli_args, compare, major_version, minor_version): diff --git a/tests/e2e/common/retry_test_mixins.py b/tests/e2e/common/retry_test_mixins.py old mode 100644 new mode 100755 index a088ba1e..106a8fb5 --- a/tests/e2e/common/retry_test_mixins.py +++ b/tests/e2e/common/retry_test_mixins.py @@ -1,3 +1,21 @@ +from contextlib import contextmanager +import time +from typing import Optional, List +from unittest.mock import MagicMock, PropertyMock, patch + +import pytest +from urllib3.exceptions import MaxRetryError + +from databricks.sql.auth.retry import DatabricksRetryPolicy +from databricks.sql.exc import ( + MaxRetryDurationError, + NonRecoverableNetworkError, + RequestError, + SessionAlreadyClosedError, + UnsafeToRetryError, +) + + class Client429ResponseMixin: def test_client_should_retry_automatically_when_getting_429(self): with self.cursor() as cursor: @@ -8,15 +26,17 @@ def test_client_should_retry_automatically_when_getting_429(self): self.assertEqual(rows[0][0], 1) def test_client_should_not_retry_429_if_RateLimitRetry_is_0(self): - with self.assertRaises(self.error_type) as cm: + with pytest.raises(self.error_type) as cm: with self.cursor(self.conf_to_disable_rate_limit_retries) as cursor: for _ in range(10): cursor.execute("SELECT 1") rows = cursor.fetchall() self.assertEqual(len(rows), 1) self.assertEqual(rows[0][0], 1) - expected = "Maximum rate of 1 requests per SECOND has been exceeded. " \ - "Please reduce the rate of requests and try again after 1 seconds." + expected = ( + "Maximum rate of 1 requests per SECOND has been exceeded. " + "Please reduce the rate of requests and try again after 1 seconds." + ) exception_str = str(cm.exception) # FIXME (Ali Smesseim, 7-Jul-2020): ODBC driver does not always return the @@ -32,7 +52,385 @@ def test_wait_cluster_startup(self): cursor.fetchall() def _test_retry_disabled_with_message(self, error_msg_substring, exception_type): - with self.assertRaises(exception_type) as cm: + with pytest.raises(exception_type) as cm: with self.connection(self.conf_to_disable_temporarily_unavailable_retries): pass - self.assertIn(error_msg_substring, str(cm.exception)) + assert error_msg_substring in str(cm.exception) + + +@contextmanager +def mocked_server_response(status: int = 200, headers: dict = {}, redirect_location: Optional[str] = None): + """Context manager for patching urllib3 responses""" + + # When mocking mocking a BaseHTTPResponse for urllib3 the mock must include + # 1. A status code + # 2. A headers dict + # 3. mock.get_redirect_location() return falsy by default + + # `msg` is included for testing when urllib3~=1.0.0 is installed + mock_response = MagicMock(headers=headers, msg=headers, status=status) + mock_response.get_redirect_location.return_value = ( + False if redirect_location is None else redirect_location + ) + + with patch("urllib3.connectionpool.HTTPSConnectionPool._get_conn") as getconn_mock: + getconn_mock.return_value.getresponse.return_value = mock_response + try: + yield getconn_mock + finally: + pass + + +@contextmanager +def mock_sequential_server_responses(responses: List[dict]): + """Same as the mocked_server_response context manager but it will yield + the provided responses in the order received + + `responses` should be a list of dictionaries containing these members: + - status: int + - headers: dict + - redirect_location: str + """ + + mock_responses = [] + + # Each resp should have these members: + + for resp in responses: + _mock = MagicMock(headers=resp["headers"], msg=resp["headers"], status=resp["status"]) + _mock.get_redirect_location.return_value = ( + False if resp["redirect_location"] is None else resp["redirect_location"] + ) + mock_responses.append(_mock) + + with patch("urllib3.connectionpool.HTTPSConnectionPool._get_conn") as getconn_mock: + getconn_mock.return_value.getresponse.side_effect = mock_responses + try: + yield getconn_mock + finally: + pass + + +class PySQLRetryTestsMixin: + """Home for retry tests where we patch urllib to return different codes and monitor that it tries to retry""" + + # For testing purposes + _retry_policy = { + "_retry_delay_min": 0.1, + "_retry_delay_max": 5, + "_retry_stop_after_attempts_count": 5, + "_retry_stop_after_attempts_duration": 10, + "_retry_delay_default": 0.5, + } + + def test_retry_urllib3_settings_are_honored(self): + """Databricks overrides some of urllib3's configuration. This tests confirms that what configuration + we DON'T override is preserved in urllib3's internals + """ + + urllib3_config = {"connect": 10, "read": 11, "redirect": 12} + rp = DatabricksRetryPolicy( + delay_min=0.1, + delay_max=10.0, + stop_after_attempts_count=10, + stop_after_attempts_duration=10.0, + delay_default=1.0, + force_dangerous_codes=[], + urllib3_kwargs=urllib3_config, + ) + + assert rp.connect == 10 + assert rp.read == 11 + assert rp.redirect == 12 + + def test_oserror_retries(self): + """If a network error occurs during make_request, the request is retried according to policy""" + with patch( + "urllib3.connectionpool.HTTPSConnectionPool._validate_conn", + ) as mock_validate_conn: + mock_validate_conn.side_effect = OSError("Some arbitrary network error") + with pytest.raises(MaxRetryError) as cm: + with self.connection(extra_params=self._retry_policy) as conn: + pass + + assert mock_validate_conn.call_count == 6 + + def test_retry_max_count_not_exceeded(self): + """GIVEN the max_attempts_count is 5 + WHEN the server sends nothing but 429 responses + THEN the connector issues six request (original plus five retries) + before raising an exception + """ + with mocked_server_response(status=404) as mock_obj: + with pytest.raises(MaxRetryError) as cm: + with self.connection(extra_params=self._retry_policy) as conn: + pass + assert mock_obj.return_value.getresponse.call_count == 6 + + def test_retry_exponential_backoff(self): + """GIVEN the retry policy is configured for reasonable exponential backoff + WHEN the server sends nothing but 429 responses with retry-afters + THEN the connector will use those retry-afters as a floor + """ + retry_policy = self._retry_policy.copy() + retry_policy["_retry_delay_min"] = 1 + + time_start = time.time() + with mocked_server_response(status=429, headers={"Retry-After": "3"}) as mock_obj: + with pytest.raises(RequestError) as cm: + with self.connection(extra_params=retry_policy) as conn: + pass + + duration = time.time() - time_start + assert isinstance(cm.value.args[1], MaxRetryDurationError) + + # With setting delay_min to 1, the expected retry delays should be: + # 3, 3, 4 + # The first 2 retries are allowed, the 3rd retry puts the total duration over the limit + # of 10 seconds + assert mock_obj.return_value.getresponse.call_count == 3 + assert duration > 6 + + # Should be less than 7, but this is a safe margin for CI/CD slowness + assert duration < 10 + + def test_retry_max_duration_not_exceeded(self): + """GIVEN the max attempt duration of 10 seconds + WHEN the server sends a Retry-After header of 60 seconds + THEN the connector raises a MaxRetryDurationError + """ + with mocked_server_response(status=429, headers={"Retry-After": "60"}): + with pytest.raises(RequestError) as cm: + with self.connection(extra_params=self._retry_policy) as conn: + pass + assert isinstance(cm.value.args[1], MaxRetryDurationError) + + def test_retry_abort_non_recoverable_error(self): + """GIVEN the server returns a code 501 + WHEN the connector receives this response + THEN nothing is retried and an exception is raised + """ + + # Code 501 is a Not Implemented error + with mocked_server_response(status=501): + with pytest.raises(RequestError) as cm: + with self.connection(extra_params=self._retry_policy) as conn: + pass + assert isinstance(cm.value.args[1], NonRecoverableNetworkError) + + def test_retry_abort_unsafe_execute_statement_retry_condition(self): + """GIVEN the server sends a code other than 429 or 503 + WHEN the connector sent an ExecuteStatement command + THEN nothing is retried because it's idempotent + """ + with self.connection(extra_params=self._retry_policy) as conn: + with conn.cursor() as cursor: + # Code 502 is a Bad Gateway, which we commonly see in production under heavy load + with mocked_server_response(status=502): + with pytest.raises(RequestError) as cm: + cursor.execute("Not a real query") + assert isinstance(cm.value.args[1], UnsafeToRetryError) + + def test_retry_dangerous_codes(self): + """GIVEN the server sends a dangerous code and the user forced this to be retryable + WHEN the connector sent an ExecuteStatement command + THEN the command is retried + """ + + # These http codes are not retried by default + # For some applications, idempotency is not important so we give users a way to force retries anyway + DANGEROUS_CODES = [502, 504, 400] + + additional_settings = { + "_retry_dangerous_codes": DANGEROUS_CODES, + "_retry_stop_after_attempts_count": 1, + } + + # Prove that these codes are not retried by default + with self.connection(extra_params={**self._retry_policy}) as conn: + with conn.cursor() as cursor: + for dangerous_code in DANGEROUS_CODES: + with mocked_server_response(status=dangerous_code): + with pytest.raises(RequestError) as cm: + cursor.execute("Not a real query") + assert isinstance(cm.value.args[1], UnsafeToRetryError) + + # Prove that these codes are retried if forced by the user + with self.connection(extra_params={**self._retry_policy, **additional_settings}) as conn: + with conn.cursor() as cursor: + for dangerous_code in DANGEROUS_CODES: + with mocked_server_response(status=dangerous_code): + with pytest.raises(MaxRetryError) as cm: + cursor.execute("Not a real query") + + def test_retry_safe_execute_statement_retry_condition(self): + """GIVEN the server sends either code 429 or 503 + WHEN the connector sent an ExecuteStatement command + THEN the request is retried because these are idempotent + """ + + responses = [ + {"status": 429, "headers": {"Retry-After": "1"}, "redirect_location": None}, + {"status": 503, "headers": {}, "redirect_location": None}, + ] + + with self.connection( + extra_params={**self._retry_policy, "_retry_stop_after_attempts_count": 1} + ) as conn: + with conn.cursor() as cursor: + # Code 502 is a Bad Gateway, which we commonly see in production under heavy load + with mock_sequential_server_responses(responses) as mock_obj: + with pytest.raises(MaxRetryError): + cursor.execute("This query never reaches the server") + assert mock_obj.return_value.getresponse.call_count == 2 + + def test_retry_abort_close_session_on_404(self, caplog): + """GIVEN the connector sends a CloseSession command + WHEN server sends a 404 (which is normally retried) + THEN nothing is retried because 404 means the session already closed + """ + + # First response is a Bad Gateway -> Result is the command actually goes through + # Second response is a 404 because the session is no longer found + responses = [ + {"status": 502, "headers": {"Retry-After": "1"}, "redirect_location": None}, + {"status": 404, "headers": {}, "redirect_location": None}, + ] + + with self.connection(extra_params={**self._retry_policy}) as conn: + with mock_sequential_server_responses(responses): + conn.close() + assert "Session was closed by a prior request" in caplog.text + + def test_retry_abort_close_operation_on_404(self, caplog): + """GIVEN the connector sends a CancelOperation command + WHEN server sends a 404 (which is normally retried) + THEN nothing is retried because 404 means the operation was already canceled + """ + + # First response is a Bad Gateway -> Result is the command actually goes through + # Second response is a 404 because the session is no longer found + responses = [ + {"status": 502, "headers": {"Retry-After": "1"}, "redirect_location": None}, + {"status": 404, "headers": {}, "redirect_location": None}, + ] + + with self.connection(extra_params={**self._retry_policy}) as conn: + with conn.cursor() as curs: + with patch( + "databricks.sql.utils.ExecuteResponse.has_been_closed_server_side", + new_callable=PropertyMock, + return_value=False, + ): + # This call guarantees we have an open cursor at the server + curs.execute("SELECT 1") + with mock_sequential_server_responses(responses): + curs.close() + assert "Operation was canceled by a prior request" in caplog.text + + def test_retry_max_redirects_raises_too_many_redirects_exception(self): + """GIVEN the connector is configured with a custom max_redirects + WHEN the DatabricksRetryPolicy is created + THEN the connector raises a MaxRedirectsError if that number is exceeded + """ + + max_redirects, expected_call_count = 1, 2 + + # Code 302 is a redirect + with mocked_server_response(status=302, redirect_location="/foo.bar") as mock_obj: + with pytest.raises(MaxRetryError) as cm: + with self.connection( + extra_params={ + **self._retry_policy, + "_retry_max_redirects": max_redirects, + } + ): + pass + assert "too many redirects" == str(cm.value.reason) + # Total call count should be 2 (original + 1 retry) + assert mock_obj.return_value.getresponse.call_count == expected_call_count + + def test_retry_max_redirects_unset_doesnt_redirect_forever(self): + """GIVEN the connector is configured without a custom max_redirects + WHEN the DatabricksRetryPolicy is used + THEN the connector raises a MaxRedirectsError if that number is exceeded + + This test effectively guarantees that regardless of _retry_max_redirects, + _stop_after_attempts_count is enforced. + """ + # Code 302 is a redirect + with mocked_server_response(status=302, redirect_location="/foo.bar/") as mock_obj: + with pytest.raises(MaxRetryError) as cm: + with self.connection( + extra_params={ + **self._retry_policy, + } + ): + pass + + # Total call count should be 6 (original + _retry_stop_after_attempts_count) + assert mock_obj.return_value.getresponse.call_count == 6 + + def test_retry_max_redirects_is_bounded_by_stop_after_attempts_count(self): + # If I add another 503 or 302 here the test will fail with a MaxRetryError + responses = [ + {"status": 302, "headers": {}, "redirect_location": "/foo.bar"}, + {"status": 500, "headers": {}, "redirect_location": None}, + ] + + additional_settings = { + "_retry_max_redirects": 1, + "_retry_stop_after_attempts_count": 2, + } + + with pytest.raises(RequestError) as cm: + with mock_sequential_server_responses(responses): + with self.connection(extra_params={**self._retry_policy, **additional_settings}): + pass + + # The error should be the result of the 500, not because of too many requests. + assert "too many redirects" not in str(cm.value.message) + assert "Error during request to server" in str(cm.value.message) + + def test_retry_max_redirects_exceeds_max_attempts_count_warns_user(self, caplog): + with self.connection( + extra_params={ + **self._retry_policy, + **{ + "_retry_max_redirects": 100, + "_retry_stop_after_attempts_count": 1, + }, + } + ): + assert "it will have no affect!" in caplog.text + + def test_retry_legacy_behavior_warns_user(self, caplog): + with self.connection(extra_params={**self._retry_policy, "_enable_v3_retries": False}): + assert "Legacy retry behavior is enabled for this connection." in caplog.text + + + def test_403_not_retried(self): + """GIVEN the server returns a code 403 + WHEN the connector receives this response + THEN nothing is retried and an exception is raised + """ + + # Code 403 is a Forbidden error + with mocked_server_response(status=403): + with pytest.raises(RequestError) as cm: + with self.connection(extra_params=self._retry_policy) as conn: + pass + assert isinstance(cm.value.args[1], NonRecoverableNetworkError) + + def test_401_not_retried(self): + """GIVEN the server returns a code 401 + WHEN the connector receives this response + THEN nothing is retried and an exception is raised + """ + + # Code 401 is an Unauthorized error + with mocked_server_response(status=401): + with pytest.raises(RequestError) as cm: + with self.connection(extra_params=self._retry_policy): + pass + assert isinstance(cm.value.args[1], NonRecoverableNetworkError) diff --git a/tests/e2e/common/staging_ingestion_tests.py b/tests/e2e/common/staging_ingestion_tests.py new file mode 100644 index 00000000..d8d0429f --- /dev/null +++ b/tests/e2e/common/staging_ingestion_tests.py @@ -0,0 +1,304 @@ +import os +import tempfile + +import pytest +import databricks.sql as sql +from databricks.sql import Error + + +@pytest.fixture(scope="module", autouse=True) +def check_staging_ingestion_user(ingestion_user): + """This fixture verifies that a staging ingestion user email address + is present in the environment and raises an exception if not. The fixture + only evaluates when the test _isn't skipped_. + """ + + if ingestion_user is None: + raise ValueError( + "To run this test you must designate a `DATABRICKS_USER` environment variable. This will be the user associated with the personal access token." + ) + + +class PySQLStagingIngestionTestSuiteMixin: + """Simple namespace for ingestion tests. These should be run against DBR >12.x + + In addition to connection credentials (host, path, token) this suite requires an env var + named staging_ingestion_user""" + + def test_staging_ingestion_life_cycle(self, ingestion_user): + """PUT a file into the staging location + GET the file from the staging location + REMOVE the file from the staging location + Try to GET the file again expecting to raise an exception + """ + + # PUT should succeed + + fh, temp_path = tempfile.mkstemp() + + original_text = "hello world!".encode("utf-8") + + with open(fh, "wb") as fp: + fp.write(original_text) + + with self.connection(extra_params={"staging_allowed_local_path": temp_path}) as conn: + + cursor = conn.cursor() + query = f"PUT '{temp_path}' INTO 'stage://tmp/{ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" + cursor.execute(query) + + # GET should succeed + + new_fh, new_temp_path = tempfile.mkstemp() + + with self.connection(extra_params={"staging_allowed_local_path": new_temp_path}) as conn: + cursor = conn.cursor() + query = f"GET 'stage://tmp/{ingestion_user}/tmp/11/15/file1.csv' TO '{new_temp_path}'" + cursor.execute(query) + + with open(new_fh, "rb") as fp: + fetched_text = fp.read() + + assert fetched_text == original_text + + # REMOVE should succeed + + remove_query = f"REMOVE 'stage://tmp/{ingestion_user}/tmp/11/15/file1.csv'" + + with self.connection(extra_params={"staging_allowed_local_path": "/"}) as conn: + cursor = conn.cursor() + cursor.execute(remove_query) + + # GET after REMOVE should fail + + with pytest.raises(Error, match="Staging operation over HTTP was unsuccessful: 404"): + cursor = conn.cursor() + query = ( + f"GET 'stage://tmp/{ingestion_user}/tmp/11/15/file1.csv' TO '{new_temp_path}'" + ) + cursor.execute(query) + + os.remove(temp_path) + os.remove(new_temp_path) + + def test_staging_ingestion_put_fails_without_staging_allowed_local_path(self, ingestion_user): + """PUT operations are not supported unless the connection was built with + a parameter called staging_allowed_local_path + """ + + fh, temp_path = tempfile.mkstemp() + + original_text = "hello world!".encode("utf-8") + + with open(fh, "wb") as fp: + fp.write(original_text) + + with pytest.raises(Error, match="You must provide at least one staging_allowed_local_path"): + with self.connection() as conn: + cursor = conn.cursor() + query = f"PUT '{temp_path}' INTO 'stage://tmp/{ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" + cursor.execute(query) + + def test_staging_ingestion_put_fails_if_localFile_not_in_staging_allowed_local_path( + self, ingestion_user + ): + + fh, temp_path = tempfile.mkstemp() + + original_text = "hello world!".encode("utf-8") + + with open(fh, "wb") as fp: + fp.write(original_text) + + base_path, filename = os.path.split(temp_path) + + # Add junk to base_path + base_path = os.path.join(base_path, "temp") + + with pytest.raises( + Error, + match="Local file operations are restricted to paths within the configured staging_allowed_local_path", + ): + with self.connection(extra_params={"staging_allowed_local_path": base_path}) as conn: + cursor = conn.cursor() + query = f"PUT '{temp_path}' INTO 'stage://tmp/{ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" + cursor.execute(query) + + def test_staging_ingestion_put_fails_if_file_exists_and_overwrite_not_set(self, ingestion_user): + """PUT a file into the staging location twice. First command should succeed. Second should fail.""" + + fh, temp_path = tempfile.mkstemp() + + original_text = "hello world!".encode("utf-8") + + with open(fh, "wb") as fp: + fp.write(original_text) + + def perform_put(): + with self.connection(extra_params={"staging_allowed_local_path": temp_path}) as conn: + cursor = conn.cursor() + query = f"PUT '{temp_path}' INTO 'stage://tmp/{ingestion_user}/tmp/12/15/file1.csv'" + cursor.execute(query) + + def perform_remove(): + try: + remove_query = f"REMOVE 'stage://tmp/{ingestion_user}/tmp/12/15/file1.csv'" + + with self.connection(extra_params={"staging_allowed_local_path": "/"}) as conn: + cursor = conn.cursor() + cursor.execute(remove_query) + except Exception: + pass + + # Make sure file does not exist + perform_remove() + + # Put the file + perform_put() + + # Try to put it again + with pytest.raises( + sql.exc.ServerOperationError, match="FILE_IN_STAGING_PATH_ALREADY_EXISTS" + ): + perform_put() + + # Clean up after ourselves + perform_remove() + + def test_staging_ingestion_fails_to_modify_another_staging_user(self): + """The server should only allow modification of the staging_ingestion_user's files""" + + some_other_user = "mary.poppins@databricks.com" + + fh, temp_path = tempfile.mkstemp() + + original_text = "hello world!".encode("utf-8") + + with open(fh, "wb") as fp: + fp.write(original_text) + + def perform_put(): + with self.connection(extra_params={"staging_allowed_local_path": temp_path}) as conn: + cursor = conn.cursor() + query = f"PUT '{temp_path}' INTO 'stage://tmp/{some_other_user}/tmp/12/15/file1.csv' OVERWRITE" + cursor.execute(query) + + def perform_remove(): + remove_query = f"REMOVE 'stage://tmp/{some_other_user}/tmp/12/15/file1.csv'" + + with self.connection(extra_params={"staging_allowed_local_path": "/"}) as conn: + cursor = conn.cursor() + cursor.execute(remove_query) + + def perform_get(): + with self.connection(extra_params={"staging_allowed_local_path": temp_path}) as conn: + cursor = conn.cursor() + query = f"GET 'stage://tmp/{some_other_user}/tmp/11/15/file1.csv' TO '{temp_path}'" + cursor.execute(query) + + # PUT should fail with permissions error + with pytest.raises(sql.exc.ServerOperationError, match="PERMISSION_DENIED"): + perform_put() + + # REMOVE should fail with permissions error + with pytest.raises(sql.exc.ServerOperationError, match="PERMISSION_DENIED"): + perform_remove() + + # GET should fail with permissions error + with pytest.raises(sql.exc.ServerOperationError, match="PERMISSION_DENIED"): + perform_get() + + def test_staging_ingestion_put_fails_if_absolute_localFile_not_in_staging_allowed_local_path( + self, ingestion_user + ): + """ + This test confirms that staging_allowed_local_path and target_file are resolved into absolute paths. + """ + + # If these two paths are not resolved absolutely, they appear to share a common path of /var/www/html + # after resolution their common path is only /var/www which should raise an exception + # Because the common path must always be equal to staging_allowed_local_path + staging_allowed_local_path = "/var/www/html" + target_file = "/var/www/html/../html1/not_allowed.html" + + with pytest.raises( + Error, + match="Local file operations are restricted to paths within the configured staging_allowed_local_path", + ): + with self.connection( + extra_params={"staging_allowed_local_path": staging_allowed_local_path} + ) as conn: + cursor = conn.cursor() + query = f"PUT '{target_file}' INTO 'stage://tmp/{ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" + cursor.execute(query) + + def test_staging_ingestion_empty_local_path_fails_to_parse_at_server(self, ingestion_user): + staging_allowed_local_path = "/var/www/html" + target_file = "" + + with pytest.raises(Error, match="EMPTY_LOCAL_FILE_IN_STAGING_ACCESS_QUERY"): + with self.connection( + extra_params={"staging_allowed_local_path": staging_allowed_local_path} + ) as conn: + cursor = conn.cursor() + query = f"PUT '{target_file}' INTO 'stage://tmp/{ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" + cursor.execute(query) + + def test_staging_ingestion_invalid_staging_path_fails_at_server(self, ingestion_user): + staging_allowed_local_path = "/var/www/html" + target_file = "index.html" + + with pytest.raises(Error, match="INVALID_STAGING_PATH_IN_STAGING_ACCESS_QUERY"): + with self.connection( + extra_params={"staging_allowed_local_path": staging_allowed_local_path} + ) as conn: + cursor = conn.cursor() + query = f"PUT '{target_file}' INTO 'stageRANDOMSTRINGOFCHARACTERS://tmp/{ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" + cursor.execute(query) + + def test_staging_ingestion_supports_multiple_staging_allowed_local_path_values( + self, ingestion_user + ): + """staging_allowed_local_path may be either a path-like object or a list of path-like objects. + + This test confirms that two configured base paths: + 1 - doesn't raise an exception + 2 - allows uploads from both paths + 3 - doesn't allow uploads from a third path + """ + + def generate_file_and_path_and_queries(): + """ + 1. Makes a temp file with some contents. + 2. Write a query to PUT it into a staging location + 3. Write a query to REMOVE it from that location (for cleanup) + """ + fh, temp_path = tempfile.mkstemp() + with open(fh, "wb") as fp: + original_text = "hello world!".encode("utf-8") + fp.write(original_text) + put_query = f"PUT '{temp_path}' INTO 'stage://tmp/{ingestion_user}/tmp/11/15/{id(temp_path)}.csv' OVERWRITE" + remove_query = f"REMOVE 'stage://tmp/{ingestion_user}/tmp/11/15/{id(temp_path)}.csv'" + return fh, temp_path, put_query, remove_query + + fh1, temp_path1, put_query1, remove_query1 = generate_file_and_path_and_queries() + fh2, temp_path2, put_query2, remove_query2 = generate_file_and_path_and_queries() + fh3, temp_path3, put_query3, remove_query3 = generate_file_and_path_and_queries() + + with self.connection( + extra_params={"staging_allowed_local_path": [temp_path1, temp_path2]} + ) as conn: + cursor = conn.cursor() + + cursor.execute(put_query1) + cursor.execute(put_query2) + + with pytest.raises( + Error, + match="Local file operations are restricted to paths within the configured staging_allowed_local_path", + ): + cursor.execute(put_query3) + + # Then clean up the files we made + cursor.execute(remove_query1) + cursor.execute(remove_query2) diff --git a/tests/e2e/common/timestamp_tests.py b/tests/e2e/common/timestamp_tests.py index 38b14e9e..f25aed7e 100644 --- a/tests/e2e/common/timestamp_tests.py +++ b/tests/e2e/common/timestamp_tests.py @@ -1,29 +1,31 @@ import datetime +import pytest + from .predicates import compare_dbr_versions, is_thrift_v5_plus, pysql_has_version class TimestampTestsMixin: - timestamp_and_expected_results = [ - ('2021-09-30 11:27:35.123+04:00', datetime.datetime(2021, 9, 30, 7, 27, 35, 123000)), - ('2021-09-30 11:27:35+04:00', datetime.datetime(2021, 9, 30, 7, 27, 35)), - ('2021-09-30 11:27:35.123', datetime.datetime(2021, 9, 30, 11, 27, 35, 123000)), - ('2021-09-30 11:27:35', datetime.datetime(2021, 9, 30, 11, 27, 35)), - ('2021-09-30 11:27', datetime.datetime(2021, 9, 30, 11, 27)), - ('2021-09-30 11', datetime.datetime(2021, 9, 30, 11)), - ('2021-09-30', datetime.datetime(2021, 9, 30)), - ('2021-09', datetime.datetime(2021, 9, 1)), - ('2021', datetime.datetime(2021, 1, 1)), - ('9999-12-31T15:59:59', datetime.datetime(9999, 12, 31, 15, 59, 59)), - ('9999-99-31T15:59:59', None), + date_and_expected_results = [ + ("2021-09-30", datetime.date(2021, 9, 30)), + ("2021-09", datetime.date(2021, 9, 1)), + ("2021", datetime.date(2021, 1, 1)), + ("9999-12-31", datetime.date(9999, 12, 31)), + ("9999-99-31", None), ] - date_and_expected_results = [ - ('2021-09-30', datetime.date(2021, 9, 30)), - ('2021-09', datetime.date(2021, 9, 1)), - ('2021', datetime.date(2021, 1, 1)), - ('9999-12-31', datetime.date(9999, 12, 31)), - ('9999-99-31', None), + timestamp_and_expected_results = [ + ("2021-09-30 11:27:35.123+04:00", datetime.datetime(2021, 9, 30, 7, 27, 35, 123000)), + ("2021-09-30 11:27:35+04:00", datetime.datetime(2021, 9, 30, 7, 27, 35)), + ("2021-09-30 11:27:35.123", datetime.datetime(2021, 9, 30, 11, 27, 35, 123000)), + ("2021-09-30 11:27:35", datetime.datetime(2021, 9, 30, 11, 27, 35)), + ("2021-09-30 11:27", datetime.datetime(2021, 9, 30, 11, 27)), + ("2021-09-30 11", datetime.datetime(2021, 9, 30, 11)), + ("2021-09-30", datetime.datetime(2021, 9, 30)), + ("2021-09", datetime.datetime(2021, 9, 1)), + ("2021", datetime.datetime(2021, 1, 1)), + ("9999-12-31T15:59:59", datetime.datetime(9999, 12, 31, 15, 59, 59)), + ("9999-99-31T15:59:59", None), ] def should_add_timezone(self): @@ -31,7 +33,7 @@ def should_add_timezone(self): def maybe_add_timezone_to_timestamp(self, ts): """If we're using DBR >= 10.2, then we expect back aware timestamps, so add timezone to `ts` - Otherwise we have naive timestamps, so no change is needed + Otherwise we have naive timestamps, so no change is needed """ if ts and self.should_add_timezone(): return ts.replace(tzinfo=datetime.timezone.utc) @@ -39,19 +41,21 @@ def maybe_add_timezone_to_timestamp(self, ts): return ts def assertTimestampsEqual(self, result, expected): - self.assertEqual(result, self.maybe_add_timezone_to_timestamp(expected)) + assert result == self.maybe_add_timezone_to_timestamp(expected) def multi_query(self, n_rows=10): row_sql = "SELECT " + ", ".join( - ["TIMESTAMP('{}')".format(ts) for (ts, _) in self.timestamp_and_expected_results]) + ["TIMESTAMP('{}')".format(ts) for (ts, _) in self.timestamp_and_expected_results] + ) query = " UNION ALL ".join([row_sql for _ in range(n_rows)]) - expected_matrix = [[dt for (_, dt) in self.timestamp_and_expected_results] - for _ in range(n_rows)] + expected_matrix = [ + [dt for (_, dt) in self.timestamp_and_expected_results] for _ in range(n_rows) + ] return query, expected_matrix def test_timestamps(self): with self.cursor({"session_configuration": {"ansi_mode": False}}) as cursor: - for (timestamp, expected) in self.timestamp_and_expected_results: + for timestamp, expected in self.timestamp_and_expected_results: cursor.execute("SELECT TIMESTAMP('{timestamp}')".format(timestamp=timestamp)) result = cursor.fetchone()[0] self.assertTimestampsEqual(result, expected) @@ -62,13 +66,14 @@ def test_multi_timestamps(self): cursor.execute(query) result = cursor.fetchall() # We list-ify the rows because PyHive will return a tuple for a row - self.assertEqual([list(r) for r in result], - [[self.maybe_add_timezone_to_timestamp(ts) for ts in r] - for r in expected]) + assert [list(r) for r in result] == [ + [self.maybe_add_timezone_to_timestamp(ts) for ts in r] for r in expected + ] - def test_dates(self): + @pytest.mark.parametrize("date, expected", date_and_expected_results) + def test_dates(self, date, expected): with self.cursor({"session_configuration": {"ansi_mode": False}}) as cursor: - for (date, expected) in self.date_and_expected_results: + for date, expected in self.date_and_expected_results: cursor.execute("SELECT DATE('{date}')".format(date=date)) result = cursor.fetchone()[0] - self.assertEqual(result, expected) + assert result == expected diff --git a/tests/e2e/common/uc_volume_tests.py b/tests/e2e/common/uc_volume_tests.py new file mode 100644 index 00000000..21e43036 --- /dev/null +++ b/tests/e2e/common/uc_volume_tests.py @@ -0,0 +1,258 @@ +import os +import tempfile + +import pytest +import databricks.sql as sql +from databricks.sql import Error + + +@pytest.fixture(scope="module", autouse=True) +def check_catalog_and_schema(catalog, schema): + """This fixture verifies that a catalog and schema are present in the environment. + The fixture only evaluates when the test _isn't skipped_. + """ + + if catalog is None or schema is None: + raise ValueError( + f"UC Volume tests require values for the `catalog` and `schema` environment variables. Found catalog {_catalog} schema {_schema}" + ) + + +class PySQLUCVolumeTestSuiteMixin: + """Simple namespace for UC Volume tests. + + In addition to connection credentials (host, path, token) this suite requires env vars + named catalog and schema""" + + def test_uc_volume_life_cycle(self, catalog, schema): + """PUT a file into the UC Volume + GET the file from the UC Volume + REMOVE the file from the UC Volume + Try to GET the file again expecting to raise an exception + """ + + # PUT should succeed + + fh, temp_path = tempfile.mkstemp() + + original_text = "hello world!".encode("utf-8") + + with open(fh, "wb") as fp: + fp.write(original_text) + + with self.connection(extra_params={"staging_allowed_local_path": temp_path}) as conn: + + cursor = conn.cursor() + query = ( + f"PUT '{temp_path}' INTO '/Volumes/{catalog}/{schema}/e2etests/file1.csv' OVERWRITE" + ) + cursor.execute(query) + + # GET should succeed + + new_fh, new_temp_path = tempfile.mkstemp() + + with self.connection(extra_params={"staging_allowed_local_path": new_temp_path}) as conn: + cursor = conn.cursor() + query = f"GET '/Volumes/{catalog}/{schema}/e2etests/file1.csv' TO '{new_temp_path}'" + cursor.execute(query) + + with open(new_fh, "rb") as fp: + fetched_text = fp.read() + + assert fetched_text == original_text + + # REMOVE should succeed + + remove_query = f"REMOVE '/Volumes/{catalog}/{schema}/e2etests/file1.csv'" + + with self.connection(extra_params={"staging_allowed_local_path": "/"}) as conn: + cursor = conn.cursor() + cursor.execute(remove_query) + + # GET after REMOVE should fail + + with pytest.raises(Error, match="Staging operation over HTTP was unsuccessful: 404"): + cursor = conn.cursor() + query = f"GET '/Volumes/{catalog}/{schema}/e2etests/file1.csv' TO '{new_temp_path}'" + cursor.execute(query) + + os.remove(temp_path) + os.remove(new_temp_path) + + def test_uc_volume_put_fails_without_staging_allowed_local_path(self, catalog, schema): + """PUT operations are not supported unless the connection was built with + a parameter called staging_allowed_local_path + """ + + fh, temp_path = tempfile.mkstemp() + + original_text = "hello world!".encode("utf-8") + + with open(fh, "wb") as fp: + fp.write(original_text) + + with pytest.raises(Error, match="You must provide at least one staging_allowed_local_path"): + with self.connection() as conn: + cursor = conn.cursor() + query = f"PUT '{temp_path}' INTO '/Volumes/{catalog}/{schema}/e2etests/file1.csv' OVERWRITE" + cursor.execute(query) + + def test_uc_volume_put_fails_if_localFile_not_in_staging_allowed_local_path( + self, catalog, schema + ): + + fh, temp_path = tempfile.mkstemp() + + original_text = "hello world!".encode("utf-8") + + with open(fh, "wb") as fp: + fp.write(original_text) + + base_path, filename = os.path.split(temp_path) + + # Add junk to base_path + base_path = os.path.join(base_path, "temp") + + with pytest.raises( + Error, + match="Local file operations are restricted to paths within the configured staging_allowed_local_path", + ): + with self.connection(extra_params={"staging_allowed_local_path": base_path}) as conn: + cursor = conn.cursor() + query = f"PUT '{temp_path}' INTO '/Volumes/{catalog}/{schema}/e2etests/file1.csv' OVERWRITE" + cursor.execute(query) + + def test_uc_volume_put_fails_if_file_exists_and_overwrite_not_set(self, catalog, schema): + """PUT a file into the staging location twice. First command should succeed. Second should fail.""" + + fh, temp_path = tempfile.mkstemp() + + original_text = "hello world!".encode("utf-8") + + with open(fh, "wb") as fp: + fp.write(original_text) + + def perform_put(): + with self.connection(extra_params={"staging_allowed_local_path": temp_path}) as conn: + cursor = conn.cursor() + query = f"PUT '{temp_path}' INTO '/Volumes/{catalog}/{schema}/e2etests/file1.csv'" + cursor.execute(query) + + def perform_remove(): + try: + remove_query = f"REMOVE '/Volumes/{catalog}/{schema}/e2etests/file1.csv'" + + with self.connection(extra_params={"staging_allowed_local_path": "/"}) as conn: + cursor = conn.cursor() + cursor.execute(remove_query) + except Exception: + pass + + # Make sure file does not exist + perform_remove() + + # Put the file + perform_put() + + # Try to put it again + with pytest.raises( + sql.exc.ServerOperationError, match="FILE_IN_STAGING_PATH_ALREADY_EXISTS" + ): + perform_put() + + # Clean up after ourselves + perform_remove() + + def test_uc_volume_put_fails_if_absolute_localFile_not_in_staging_allowed_local_path( + self, catalog, schema + ): + """ + This test confirms that staging_allowed_local_path and target_file are resolved into absolute paths. + """ + + # If these two paths are not resolved absolutely, they appear to share a common path of /var/www/html + # after resolution their common path is only /var/www which should raise an exception + # Because the common path must always be equal to staging_allowed_local_path + staging_allowed_local_path = "/var/www/html" + target_file = "/var/www/html/../html1/not_allowed.html" + + with pytest.raises( + Error, + match="Local file operations are restricted to paths within the configured staging_allowed_local_path", + ): + with self.connection( + extra_params={"staging_allowed_local_path": staging_allowed_local_path} + ) as conn: + cursor = conn.cursor() + query = f"PUT '{target_file}' INTO '/Volumes/{catalog}/{schema}/e2etests/file1.csv' OVERWRITE" + cursor.execute(query) + + def test_uc_volume_empty_local_path_fails_to_parse_at_server(self, catalog, schema): + staging_allowed_local_path = "/var/www/html" + target_file = "" + + with pytest.raises(Error, match="EMPTY_LOCAL_FILE_IN_STAGING_ACCESS_QUERY"): + with self.connection( + extra_params={"staging_allowed_local_path": staging_allowed_local_path} + ) as conn: + cursor = conn.cursor() + query = f"PUT '{target_file}' INTO '/Volumes/{catalog}/{schema}/e2etests/file1.csv' OVERWRITE" + cursor.execute(query) + + def test_uc_volume_invalid_volume_path_fails_at_server(self, catalog, schema): + staging_allowed_local_path = "/var/www/html" + target_file = "index.html" + + with pytest.raises(Error, match="NOT_FOUND: Catalog"): + with self.connection( + extra_params={"staging_allowed_local_path": staging_allowed_local_path} + ) as conn: + cursor = conn.cursor() + query = f"PUT '{target_file}' INTO '/Volumes/RANDOMSTRINGOFCHARACTERS/{catalog}/{schema}/e2etests/file1.csv' OVERWRITE" + cursor.execute(query) + + def test_uc_volume_supports_multiple_staging_allowed_local_path_values(self, catalog, schema): + """staging_allowed_local_path may be either a path-like object or a list of path-like objects. + + This test confirms that two configured base paths: + 1 - doesn't raise an exception + 2 - allows uploads from both paths + 3 - doesn't allow uploads from a third path + """ + + def generate_file_and_path_and_queries(): + """ + 1. Makes a temp file with some contents. + 2. Write a query to PUT it into a staging location + 3. Write a query to REMOVE it from that location (for cleanup) + """ + fh, temp_path = tempfile.mkstemp() + with open(fh, "wb") as fp: + original_text = "hello world!".encode("utf-8") + fp.write(original_text) + put_query = f"PUT '{temp_path}' INTO '/Volumes/{catalog}/{schema}/e2etests/{id(temp_path)}.csv' OVERWRITE" + remove_query = f"REMOVE '/Volumes/{catalog}/{schema}/e2etests/{id(temp_path)}.csv'" + return fh, temp_path, put_query, remove_query + + fh1, temp_path1, put_query1, remove_query1 = generate_file_and_path_and_queries() + fh2, temp_path2, put_query2, remove_query2 = generate_file_and_path_and_queries() + fh3, temp_path3, put_query3, remove_query3 = generate_file_and_path_and_queries() + + with self.connection( + extra_params={"staging_allowed_local_path": [temp_path1, temp_path2]} + ) as conn: + cursor = conn.cursor() + + cursor.execute(put_query1) + cursor.execute(put_query2) + + with pytest.raises( + Error, + match="Local file operations are restricted to paths within the configured staging_allowed_local_path", + ): + cursor.execute(put_query3) + + # Then clean up the files we made + cursor.execute(remove_query1) + cursor.execute(remove_query2) diff --git a/tests/e2e/driver_tests.py b/tests/e2e/driver_tests.py deleted file mode 100644 index 1c09d70e..00000000 --- a/tests/e2e/driver_tests.py +++ /dev/null @@ -1,917 +0,0 @@ -from contextlib import contextmanager -from collections import OrderedDict -import datetime -import io -import logging -import os -import sys -import tempfile -import threading -import time -from unittest import loader, skipIf, skipUnless, TestCase -from uuid import uuid4 - -import numpy as np -import pyarrow -import pytz -import thrift -import pytest - -import databricks.sql as sql -from databricks.sql import STRING, BINARY, NUMBER, DATETIME, DATE, DatabaseError, Error, OperationalError -from tests.e2e.common.predicates import pysql_has_version, pysql_supports_arrow, compare_dbr_versions, is_thrift_v5_plus -from tests.e2e.common.core_tests import CoreTestMixin, SmokeTestMixin -from tests.e2e.common.large_queries_mixin import LargeQueriesMixin -from tests.e2e.common.timestamp_tests import TimestampTestsMixin -from tests.e2e.common.decimal_tests import DecimalTestsMixin -from tests.e2e.common.retry_test_mixins import Client429ResponseMixin, Client503ResponseMixin - -log = logging.getLogger(__name__) - -# manually decorate DecimalTestsMixin to need arrow support -for name in loader.getTestCaseNames(DecimalTestsMixin, 'test_'): - fn = getattr(DecimalTestsMixin, name) - decorated = skipUnless(pysql_supports_arrow(), 'Decimal tests need arrow support')(fn) - setattr(DecimalTestsMixin, name, decorated) - -get_args_from_env = True - - -class PySQLTestCase(TestCase): - error_type = Error - conf_to_disable_rate_limit_retries = {"_retry_stop_after_attempts_count": 1} - conf_to_disable_temporarily_unavailable_retries = {"_retry_stop_after_attempts_count": 1} - - def __init__(self, method_name): - super().__init__(method_name) - # If running in local mode, just use environment variables for params. - self.arguments = os.environ if get_args_from_env else {} - self.arraysize = 1000 - - def connection_params(self, arguments): - params = { - "server_hostname": arguments["host"], - "http_path": arguments["http_path"], - **self.auth_params(arguments) - } - - return params - - def auth_params(self, arguments): - return { - "_username": arguments.get("rest_username"), - "_password": arguments.get("rest_password"), - "access_token": arguments.get("access_token") - } - - @contextmanager - def connection(self, extra_params=()): - connection_params = dict(self.connection_params(self.arguments), **dict(extra_params)) - - log.info("Connecting with args: {}".format(connection_params)) - conn = sql.connect(**connection_params) - - try: - yield conn - finally: - conn.close() - - @contextmanager - def cursor(self, extra_params=()): - with self.connection(extra_params) as conn: - cursor = conn.cursor(arraysize=self.arraysize) - try: - yield cursor - finally: - cursor.close() - - def assertEqualRowValues(self, actual, expected): - self.assertEqual(len(actual) if actual else 0, len(expected) if expected else 0) - for act, exp in zip(actual, expected): - self.assertSequenceEqual(act, exp) - - -class PySQLLargeQueriesSuite(PySQLTestCase, LargeQueriesMixin): - def get_some_rows(self, cursor, fetchmany_size): - row = cursor.fetchone() - if row: - return [row] - else: - return None - - -# Exclude Retry tests because they require specific setups, and LargeQueries too slow for core -# tests -class PySQLCoreTestSuite(SmokeTestMixin, CoreTestMixin, DecimalTestsMixin, TimestampTestsMixin, - PySQLTestCase): - validate_row_value_type = True - validate_result = True - - # An output column in description evaluates to equal to multiple types - # - type code returned by the client as string. - # - also potentially a PEP-249 object like NUMBER, DATETIME etc. - def expected_column_types(self, type_): - type_mappings = { - 'boolean': ['boolean', NUMBER], - 'byte': ['tinyint', NUMBER], - 'short': ['smallint', NUMBER], - 'integer': ['int', NUMBER], - 'long': ['bigint', NUMBER], - 'decimal': ['decimal', NUMBER], - 'timestamp': ['timestamp', DATETIME], - 'date': ['date', DATE], - 'binary': ['binary', BINARY], - 'string': ['string', STRING], - 'array': ['array'], - 'struct': ['struct'], - 'map': ['map'], - 'double': ['double', NUMBER], - 'null': ['null'] - } - return type_mappings[type_] - - def test_queries(self): - if not self._should_have_native_complex_types(): - array_type = str - array_val = "[1,2,3]" - struct_type = str - struct_val = "{\"a\":1,\"b\":2}" - map_type = str - map_val = "{1:2,3:4}" - else: - array_type = np.ndarray - array_val = np.array([1, 2, 3]) - struct_type = dict - struct_val = {"a": 1, "b": 2} - map_type = list - map_val = [(1, 2), (3, 4)] - - null_type = "null" if float(sql.__version__[0:2]) < 2.0 else "string" - self.range_queries = CoreTestMixin.range_queries + [ - ("NULL", null_type, type(None), None), - ("array(1, 2, 3)", 'array', array_type, array_val), - ("struct(1 as a, 2 as b)", 'struct', struct_type, struct_val), - ("map(1, 2, 3, 4)", 'map', map_type, map_val), - ] - - self.run_tests_on_queries({}) - - @skipIf(pysql_has_version('<', '2'), 'requires pysql v2') - def test_incorrect_query_throws_exception(self): - with self.cursor({}) as cursor: - # Syntax errors should contain the invalid SQL - with self.assertRaises(DatabaseError) as cm: - cursor.execute("^ FOO BAR") - self.assertIn("FOO BAR", str(cm.exception)) - - # Database error should contain the missing database - with self.assertRaises(DatabaseError) as cm: - cursor.execute("USE foo234823498ydfsiusdhf") - self.assertIn("foo234823498ydfsiusdhf", str(cm.exception)) - - # SQL with Extraneous input should send back the extraneous input - with self.assertRaises(DatabaseError) as cm: - cursor.execute("CREATE TABLE IF NOT EXISTS TABLE table_234234234") - self.assertIn("table_234234234", str(cm.exception)) - - def test_create_table_will_return_empty_result_set(self): - with self.cursor({}) as cursor: - table_name = 'table_{uuid}'.format(uuid=str(uuid4()).replace('-', '_')) - try: - cursor.execute( - "CREATE TABLE IF NOT EXISTS {} AS (SELECT 1 AS col_1, '2' AS col_2)".format( - table_name)) - self.assertEqual(cursor.fetchall(), []) - finally: - cursor.execute("DROP TABLE IF EXISTS {}".format(table_name)) - - def test_get_tables(self): - with self.cursor({}) as cursor: - table_name = 'table_{uuid}'.format(uuid=str(uuid4()).replace('-', '_')) - table_names = [table_name + '_1', table_name + '_2'] - - try: - for table in table_names: - cursor.execute( - "CREATE TABLE IF NOT EXISTS {} AS (SELECT 1 AS col_1, '2' AS col_2)".format( - table)) - cursor.tables(schema_name="defa%") - tables = cursor.fetchall() - tables_desc = cursor.description - - for table in table_names: - # Test only schema name and table name. - # From other columns, what is supported depends on DBR version. - self.assertIn(['default', table], [list(table[1:3]) for table in tables]) - self.assertEqual( - tables_desc, - [('TABLE_CAT', 'string', None, None, None, None, None), - ('TABLE_SCHEM', 'string', None, None, None, None, None), - ('TABLE_NAME', 'string', None, None, None, None, None), - ('TABLE_TYPE', 'string', None, None, None, None, None), - ('REMARKS', 'string', None, None, None, None, None), - ('TYPE_CAT', 'string', None, None, None, None, None), - ('TYPE_SCHEM', 'string', None, None, None, None, None), - ('TYPE_NAME', 'string', None, None, None, None, None), - ('SELF_REFERENCING_COL_NAME', 'string', None, None, None, None, None), - ('REF_GENERATION', 'string', None, None, None, None, None)]) - finally: - for table in table_names: - cursor.execute('DROP TABLE IF EXISTS {}'.format(table)) - - def test_get_columns(self): - with self.cursor({}) as cursor: - table_name = 'table_{uuid}'.format(uuid=str(uuid4()).replace('-', '_')) - table_names = [table_name + '_1', table_name + '_2'] - - try: - for table in table_names: - cursor.execute("CREATE TABLE IF NOT EXISTS {} AS (SELECT " - "1 AS col_1, " - "'2' AS col_2, " - "named_struct('name', 'alice', 'age', 28) as col_3, " - "map('items', 45, 'cost', 228) as col_4, " - "array('item1', 'item2', 'item3') as col_5)".format(table)) - - cursor.columns(schema_name="defa%", table_name=table_name + '%') - cols = cursor.fetchall() - cols_desc = cursor.description - - # Catalogue name not consistent across DBR versions, so we skip that - cleaned_response = [list(col[1:6]) for col in cols] - # We also replace ` as DBR changes how it represents struct names - for col in cleaned_response: - col[4] = col[4].replace("`", "") - - self.assertEqual(cleaned_response, [ - ['default', table_name + '_1', 'col_1', 4, 'INT'], - ['default', table_name + '_1', 'col_2', 12, 'STRING'], - ['default', table_name + '_1', 'col_3', 2002, 'STRUCT'], - ['default', table_name + '_1', 'col_4', 2000, 'MAP'], - ['default', table_name + '_1', 'col_5', 2003, 'ARRAY'], - ['default', table_name + '_2', 'col_1', 4, 'INT'], - ['default', table_name + '_2', 'col_2', 12, 'STRING'], - ['default', table_name + '_2', 'col_3', 2002, 'STRUCT'], - ['default', table_name + '_2', 'col_4', 2000, 'MAP'], - [ - 'default', - table_name + '_2', - 'col_5', - 2003, - 'ARRAY', - ] - ]) - - self.assertEqual(cols_desc, - [('TABLE_CAT', 'string', None, None, None, None, None), - ('TABLE_SCHEM', 'string', None, None, None, None, None), - ('TABLE_NAME', 'string', None, None, None, None, None), - ('COLUMN_NAME', 'string', None, None, None, None, None), - ('DATA_TYPE', 'int', None, None, None, None, None), - ('TYPE_NAME', 'string', None, None, None, None, None), - ('COLUMN_SIZE', 'int', None, None, None, None, None), - ('BUFFER_LENGTH', 'tinyint', None, None, None, None, None), - ('DECIMAL_DIGITS', 'int', None, None, None, None, None), - ('NUM_PREC_RADIX', 'int', None, None, None, None, None), - ('NULLABLE', 'int', None, None, None, None, None), - ('REMARKS', 'string', None, None, None, None, None), - ('COLUMN_DEF', 'string', None, None, None, None, None), - ('SQL_DATA_TYPE', 'int', None, None, None, None, None), - ('SQL_DATETIME_SUB', 'int', None, None, None, None, None), - ('CHAR_OCTET_LENGTH', 'int', None, None, None, None, None), - ('ORDINAL_POSITION', 'int', None, None, None, None, None), - ('IS_NULLABLE', 'string', None, None, None, None, None), - ('SCOPE_CATALOG', 'string', None, None, None, None, None), - ('SCOPE_SCHEMA', 'string', None, None, None, None, None), - ('SCOPE_TABLE', 'string', None, None, None, None, None), - ('SOURCE_DATA_TYPE', 'smallint', None, None, None, None, None), - ('IS_AUTO_INCREMENT', 'string', None, None, None, None, None)]) - finally: - for table in table_names: - cursor.execute('DROP TABLE IF EXISTS {}'.format(table)) - - def test_escape_single_quotes(self): - with self.cursor({}) as cursor: - table_name = 'table_{uuid}'.format(uuid=str(uuid4()).replace('-', '_')) - # Test escape syntax directly - cursor.execute("CREATE TABLE IF NOT EXISTS {} AS (SELECT 'you\\'re' AS col_1)".format(table_name)) - cursor.execute("SELECT * FROM {} WHERE col_1 LIKE 'you\\'re'".format(table_name)) - rows = cursor.fetchall() - assert rows[0]["col_1"] == "you're" - - # Test escape syntax in parameter - cursor.execute("SELECT * FROM {} WHERE {}.col_1 LIKE %(var)s".format(table_name, table_name), parameters={"var": "you're"}) - rows = cursor.fetchall() - assert rows[0]["col_1"] == "you're" - - def test_get_schemas(self): - with self.cursor({}) as cursor: - database_name = 'db_{uuid}'.format(uuid=str(uuid4()).replace('-', '_')) - try: - cursor.execute('CREATE DATABASE IF NOT EXISTS {}'.format(database_name)) - cursor.schemas() - schemas = cursor.fetchall() - schemas_desc = cursor.description - # Catalogue name not consistent across DBR versions, so we skip that - self.assertIn(database_name, [schema[0] for schema in schemas]) - self.assertEqual(schemas_desc, - [('TABLE_SCHEM', 'string', None, None, None, None, None), - ('TABLE_CATALOG', 'string', None, None, None, None, None)]) - finally: - cursor.execute('DROP DATABASE IF EXISTS {}'.format(database_name)) - - def test_get_catalogs(self): - with self.cursor({}) as cursor: - cursor.catalogs() - cursor.fetchall() - catalogs_desc = cursor.description - self.assertEqual(catalogs_desc, [('TABLE_CAT', 'string', None, None, None, None, None)]) - - @skipUnless(pysql_supports_arrow(), 'arrow test need arrow support') - def test_get_arrow(self): - # These tests are quite light weight as the arrow fetch methods are used internally - # by everything else - with self.cursor({}) as cursor: - cursor.execute("SELECT * FROM range(10)") - table_1 = cursor.fetchmany_arrow(1).to_pydict() - self.assertEqual(table_1, OrderedDict([("id", [0])])) - - table_2 = cursor.fetchall_arrow().to_pydict() - self.assertEqual(table_2, OrderedDict([("id", [1, 2, 3, 4, 5, 6, 7, 8, 9])])) - - def test_unicode(self): - unicode_str = "数据砖" - with self.cursor({}) as cursor: - cursor.execute("SELECT '{}'".format(unicode_str)) - results = cursor.fetchall() - self.assertTrue(len(results) == 1 and len(results[0]) == 1) - self.assertEqual(results[0][0], unicode_str) - - def test_cancel_during_execute(self): - with self.cursor({}) as cursor: - - def execute_really_long_query(): - cursor.execute("SELECT SUM(A.id - B.id) " + - "FROM range(1000000000) A CROSS JOIN range(100000000) B " + - "GROUP BY (A.id - B.id)") - - exec_thread = threading.Thread(target=execute_really_long_query) - - exec_thread.start() - # Make sure the query has started before cancelling - time.sleep(15) - cursor.cancel() - exec_thread.join(5) - self.assertFalse(exec_thread.is_alive()) - - # Fetching results should throw an exception - with self.assertRaises((Error, thrift.Thrift.TException)): - cursor.fetchall() - with self.assertRaises((Error, thrift.Thrift.TException)): - cursor.fetchone() - with self.assertRaises((Error, thrift.Thrift.TException)): - cursor.fetchmany(10) - - # We should be able to execute a new command on the cursor - cursor.execute("SELECT * FROM range(3)") - self.assertEqual(len(cursor.fetchall()), 3) - - @skipIf(pysql_has_version('<', '2'), 'requires pysql v2') - def test_can_execute_command_after_failure(self): - with self.cursor({}) as cursor: - with self.assertRaises(DatabaseError): - cursor.execute("this is a sytnax error") - - cursor.execute("SELECT 1;") - - res = cursor.fetchall() - self.assertEqualRowValues(res, [[1]]) - - @skipIf(pysql_has_version('<', '2'), 'requires pysql v2') - def test_can_execute_command_after_success(self): - with self.cursor({}) as cursor: - cursor.execute("SELECT 1;") - cursor.execute("SELECT 2;") - - res = cursor.fetchall() - self.assertEqualRowValues(res, [[2]]) - - def generate_multi_row_query(self): - query = "SELECT * FROM range(3);" - return query - - @skipIf(pysql_has_version('<', '2'), 'requires pysql v2') - def test_fetchone(self): - with self.cursor({}) as cursor: - query = self.generate_multi_row_query() - cursor.execute(query) - - self.assertSequenceEqual(cursor.fetchone(), [0]) - self.assertSequenceEqual(cursor.fetchone(), [1]) - self.assertSequenceEqual(cursor.fetchone(), [2]) - - self.assertEqual(cursor.fetchone(), None) - - @skipIf(pysql_has_version('<', '2'), 'requires pysql v2') - def test_fetchall(self): - with self.cursor({}) as cursor: - query = self.generate_multi_row_query() - cursor.execute(query) - - self.assertEqualRowValues(cursor.fetchall(), [[0], [1], [2]]) - - self.assertEqual(cursor.fetchone(), None) - - @skipIf(pysql_has_version('<', '2'), 'requires pysql v2') - def test_fetchmany_when_stride_fits(self): - with self.cursor({}) as cursor: - query = "SELECT * FROM range(4)" - cursor.execute(query) - - self.assertEqualRowValues(cursor.fetchmany(2), [[0], [1]]) - self.assertEqualRowValues(cursor.fetchmany(2), [[2], [3]]) - - @skipIf(pysql_has_version('<', '2'), 'requires pysql v2') - def test_fetchmany_in_excess(self): - with self.cursor({}) as cursor: - query = "SELECT * FROM range(4)" - cursor.execute(query) - - self.assertEqualRowValues(cursor.fetchmany(3), [[0], [1], [2]]) - self.assertEqualRowValues(cursor.fetchmany(3), [[3]]) - - @skipIf(pysql_has_version('<', '2'), 'requires pysql v2') - def test_iterator_api(self): - with self.cursor({}) as cursor: - query = "SELECT * FROM range(4)" - cursor.execute(query) - - expected_results = [[0], [1], [2], [3]] - for (i, row) in enumerate(cursor): - self.assertSequenceEqual(row, expected_results[i]) - - def test_temp_view_fetch(self): - with self.cursor({}) as cursor: - query = "create temporary view f as select * from range(10)" - cursor.execute(query) - # TODO assert on a result - # once what is being returned has stabilised - - @skipIf(pysql_has_version('<', '2'), 'requires pysql v2') - def test_socket_timeout(self): - # We we expect to see a BlockingIO error when the socket is opened - # in non-blocking mode, since no poll is done before the read - with self.assertRaises(OperationalError) as cm: - with self.cursor({"_socket_timeout": 0}): - pass - - self.assertIsInstance(cm.exception.args[1], io.BlockingIOError) - - def test_ssp_passthrough(self): - for enable_ansi in (True, False): - with self.cursor({"session_configuration": {"ansi_mode": enable_ansi}}) as cursor: - cursor.execute("SET ansi_mode") - self.assertEqual(list(cursor.fetchone()), ["ansi_mode", str(enable_ansi)]) - - @skipUnless(pysql_supports_arrow(), 'arrow test needs arrow support') - def test_timestamps_arrow(self): - with self.cursor({"session_configuration": {"ansi_mode": False}}) as cursor: - for (timestamp, expected) in self.timestamp_and_expected_results: - cursor.execute("SELECT TIMESTAMP('{timestamp}')".format(timestamp=timestamp)) - arrow_table = cursor.fetchmany_arrow(1) - if self.should_add_timezone(): - ts_type = pyarrow.timestamp("us", tz="Etc/UTC") - else: - ts_type = pyarrow.timestamp("us") - self.assertEqual(arrow_table.field(0).type, ts_type) - result_value = arrow_table.column(0).combine_chunks()[0].value - # To work consistently across different local timezones, we specify the timezone - # of the expected result to - # be UTC (what it should be by default on the server) - aware_timestamp = expected and expected.replace(tzinfo=datetime.timezone.utc) - self.assertEqual(result_value, aware_timestamp and - aware_timestamp.timestamp() * 1000000, - "timestamp {} did not match {}".format(timestamp, expected)) - - @skipUnless(pysql_supports_arrow(), 'arrow test needs arrow support') - def test_multi_timestamps_arrow(self): - with self.cursor({"session_configuration": {"ansi_mode": False}}) as cursor: - query, expected = self.multi_query() - expected = [[self.maybe_add_timezone_to_timestamp(ts) for ts in row] - for row in expected] - cursor.execute(query) - table = cursor.fetchall_arrow() - # Transpose columnar result to list of rows - list_of_cols = [c.to_pylist() for c in table] - result = [[col[row_index] for col in list_of_cols] - for row_index in range(table.num_rows)] - self.assertEqual(result, expected) - - @skipUnless(pysql_supports_arrow(), 'arrow test needs arrow support') - def test_timezone_with_timestamp(self): - if self.should_add_timezone(): - with self.cursor() as cursor: - cursor.execute("SET TIME ZONE 'Europe/Amsterdam'") - cursor.execute("select CAST('2022-03-02 12:54:56' as TIMESTAMP)") - amsterdam = pytz.timezone("Europe/Amsterdam") - expected = amsterdam.localize(datetime.datetime(2022, 3, 2, 12, 54, 56)) - result = cursor.fetchone()[0] - self.assertEqual(result, expected) - - cursor.execute("select CAST('2022-03-02 12:54:56' as TIMESTAMP)") - arrow_result_table = cursor.fetchmany_arrow(1) - arrow_result_value = arrow_result_table.column(0).combine_chunks()[0].value - ts_type = pyarrow.timestamp("us", tz="Europe/Amsterdam") - - self.assertEqual(arrow_result_table.field(0).type, ts_type) - self.assertEqual(arrow_result_value, expected.timestamp() * 1000000) - - @skipUnless(pysql_supports_arrow(), 'arrow test needs arrow support') - def test_can_flip_compression(self): - with self.cursor() as cursor: - cursor.execute("SELECT array(1,2,3,4)") - cursor.fetchall() - lz4_compressed = cursor.active_result_set.lz4_compressed - #The endpoint should support compression - self.assertEqual(lz4_compressed, True) - cursor.connection.lz4_compression=False - cursor.execute("SELECT array(1,2,3,4)") - cursor.fetchall() - lz4_compressed = cursor.active_result_set.lz4_compressed - self.assertEqual(lz4_compressed, False) - - def _should_have_native_complex_types(self): - return pysql_has_version(">=", 2) and is_thrift_v5_plus(self.arguments) - - @skipUnless(pysql_supports_arrow(), 'arrow test needs arrow support') - def test_arrays_are_not_returned_as_strings_arrow(self): - if self._should_have_native_complex_types(): - with self.cursor() as cursor: - cursor.execute("SELECT array(1,2,3,4)") - arrow_df = cursor.fetchall_arrow() - - list_type = arrow_df.field(0).type - self.assertTrue(pyarrow.types.is_list(list_type)) - self.assertTrue(pyarrow.types.is_integer(list_type.value_type)) - - @skipUnless(pysql_supports_arrow(), 'arrow test needs arrow support') - def test_structs_are_not_returned_as_strings_arrow(self): - if self._should_have_native_complex_types(): - with self.cursor() as cursor: - cursor.execute("SELECT named_struct('foo', 42, 'bar', 'baz')") - arrow_df = cursor.fetchall_arrow() - - struct_type = arrow_df.field(0).type - self.assertTrue(pyarrow.types.is_struct(struct_type)) - - @skipUnless(pysql_supports_arrow(), 'arrow test needs arrow support') - def test_decimal_not_returned_as_strings_arrow(self): - if self._should_have_native_complex_types(): - with self.cursor() as cursor: - cursor.execute("SELECT 5E3BD") - arrow_df = cursor.fetchall_arrow() - - decimal_type = arrow_df.field(0).type - self.assertTrue(pyarrow.types.is_decimal(decimal_type)) - - def test_close_connection_closes_cursors(self): - - from databricks.sql.thrift_api.TCLIService import ttypes - - with self.connection() as conn: - cursor = conn.cursor() - cursor.execute('SELECT id, id `id2`, id `id3` FROM RANGE(1000000) order by RANDOM()') - ars = cursor.active_result_set - - # We must manually run this check because thrift_backend always forces `has_been_closed_server_side` to True - - # Cursor op state should be open before connection is closed - status_request = ttypes.TGetOperationStatusReq(operationHandle=ars.command_id, getProgressUpdate=False) - op_status_at_server = ars.thrift_backend._client.GetOperationStatus(status_request) - assert op_status_at_server.operationState != ttypes.TOperationState.CLOSED_STATE - - conn.close() - - # When connection closes, any cursor operations should no longer exist at the server - with self.assertRaises(thrift.Thrift.TApplicationException) as cm: - op_status_at_server = ars.thrift_backend._client.GetOperationStatus(status_request) - if hasattr(cm, "exception"): - assert "RESOURCE_DOES_NOT_EXIST" in cm.exception.message - - - - -# use a RetrySuite to encapsulate these tests which we'll typically want to run together; however keep -# the 429/503 subsuites separate since they execute under different circumstances. -class PySQLRetryTestSuite: - class HTTP429Suite(Client429ResponseMixin, PySQLTestCase): - pass # Mixin covers all - - class HTTP503Suite(Client503ResponseMixin, PySQLTestCase): - # 503Response suite gets custom error here vs PyODBC - def test_retry_disabled(self): - self._test_retry_disabled_with_message("TEMPORARILY_UNAVAILABLE", OperationalError) - - -class PySQLUnityCatalogTestSuite(PySQLTestCase): - """Simple namespace tests that should be run against a unity-catalog-enabled cluster""" - - @skipIf(pysql_has_version('<', '2'), 'requires pysql v2') - def test_initial_namespace(self): - table_name = 'table_{uuid}'.format(uuid=str(uuid4()).replace('-', '_')) - with self.cursor() as cursor: - cursor.execute("USE CATALOG {}".format(self.arguments["catA"])) - cursor.execute("CREATE TABLE table_{} (col1 int)".format(table_name)) - with self.connection({ - "catalog": self.arguments["catA"], - "schema": table_name - }) as connection: - cursor = connection.cursor() - cursor.execute("select current_catalog()") - self.assertEqual(cursor.fetchone()[0], self.arguments["catA"]) - cursor.execute("select current_database()") - self.assertEqual(cursor.fetchone()[0], table_name) - -class PySQLStagingIngestionTestSuite(PySQLTestCase): - """Simple namespace for ingestion tests. These should be run against DBR >12.x - - In addition to connection credentials (host, path, token) this suite requires an env var - named staging_ingestion_user""" - - staging_ingestion_user = os.getenv("staging_ingestion_user") - - if staging_ingestion_user is None: - raise ValueError( - "To run these tests you must designate a `staging_ingestion_user` environment variable. This will the user associated with the personal access token." - ) - - def test_staging_ingestion_life_cycle(self): - """PUT a file into the staging location - GET the file from the staging location - REMOVE the file from the staging location - Try to GET the file again expecting to raise an exception - """ - - # PUT should succeed - - fh, temp_path = tempfile.mkstemp() - - original_text = "hello world!".encode("utf-8") - - with open(fh, "wb") as fp: - fp.write(original_text) - - with self.connection(extra_params={"staging_allowed_local_path": temp_path}) as conn: - - cursor = conn.cursor() - query = f"PUT '{temp_path}' INTO 'stage://tmp/{self.staging_ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" - cursor.execute(query) - - # GET should succeed - - new_fh, new_temp_path = tempfile.mkstemp() - - with self.connection(extra_params={"staging_allowed_local_path": new_temp_path}) as conn: - cursor = conn.cursor() - query = f"GET 'stage://tmp/{self.staging_ingestion_user}/tmp/11/15/file1.csv' TO '{new_temp_path}'" - cursor.execute(query) - - with open(new_fh, "rb") as fp: - fetched_text = fp.read() - - assert fetched_text == original_text - - # REMOVE should succeed - - remove_query = ( - f"REMOVE 'stage://tmp/{self.staging_ingestion_user}/tmp/11/15/file1.csv'" - ) - - with self.connection(extra_params={"staging_allowed_local_path": "/"}) as conn: - cursor = conn.cursor() - cursor.execute(remove_query) - - # GET after REMOVE should fail - - with pytest.raises(Error): - cursor = conn.cursor() - query = f"GET 'stage://tmp/{self.staging_ingestion_user}/tmp/11/15/file1.csv' TO '{new_temp_path}'" - cursor.execute(query) - - os.remove(temp_path) - os.remove(new_temp_path) - - - def test_staging_ingestion_put_fails_without_staging_allowed_local_path(self): - """PUT operations are not supported unless the connection was built with - a parameter called staging_allowed_local_path - """ - - fh, temp_path = tempfile.mkstemp() - - original_text = "hello world!".encode("utf-8") - - with open(fh, "wb") as fp: - fp.write(original_text) - - with pytest.raises(Error): - with self.connection() as conn: - cursor = conn.cursor() - query = f"PUT '{temp_path}' INTO 'stage://tmp/{self.staging_ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" - cursor.execute(query) - - def test_staging_ingestion_put_fails_if_localFile_not_in_staging_allowed_local_path(self): - - - fh, temp_path = tempfile.mkstemp() - - original_text = "hello world!".encode("utf-8") - - with open(fh, "wb") as fp: - fp.write(original_text) - - base_path, filename = os.path.split(temp_path) - - # Add junk to base_path - base_path = os.path.join(base_path, "temp") - - with pytest.raises(Error): - with self.connection(extra_params={"staging_allowed_local_path": base_path}) as conn: - cursor = conn.cursor() - query = f"PUT '{temp_path}' INTO 'stage://tmp/{self.staging_ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" - cursor.execute(query) - - def test_staging_ingestion_put_fails_if_file_exists_and_overwrite_not_set(self): - """PUT a file into the staging location twice. First command should succeed. Second should fail. - """ - - fh, temp_path = tempfile.mkstemp() - - original_text = "hello world!".encode("utf-8") - - with open(fh, "wb") as fp: - fp.write(original_text) - - def perform_put(): - with self.connection(extra_params={"staging_allowed_local_path": temp_path}) as conn: - cursor = conn.cursor() - query = f"PUT '{temp_path}' INTO 'stage://tmp/{self.staging_ingestion_user}/tmp/12/15/file1.csv'" - cursor.execute(query) - - def perform_remove(): - remove_query = ( - f"REMOVE 'stage://tmp/{self.staging_ingestion_user}/tmp/12/15/file1.csv'" - ) - - with self.connection(extra_params={"staging_allowed_local_path": "/"}) as conn: - cursor = conn.cursor() - cursor.execute(remove_query) - - - # Make sure file does not exist - perform_remove() - - # Put the file - perform_put() - - # Try to put it again - with pytest.raises(sql.exc.ServerOperationError, match="FILE_IN_STAGING_PATH_ALREADY_EXISTS"): - perform_put() - - # Clean up after ourselves - perform_remove() - - def test_staging_ingestion_fails_to_modify_another_staging_user(self): - """The server should only allow modification of the staging_ingestion_user's files - """ - - some_other_user = "mary.poppins@databricks.com" - - fh, temp_path = tempfile.mkstemp() - - original_text = "hello world!".encode("utf-8") - - with open(fh, "wb") as fp: - fp.write(original_text) - - def perform_put(): - with self.connection(extra_params={"staging_allowed_local_path": temp_path}) as conn: - cursor = conn.cursor() - query = f"PUT '{temp_path}' INTO 'stage://tmp/{some_other_user}/tmp/12/15/file1.csv' OVERWRITE" - cursor.execute(query) - - def perform_remove(): - remove_query = ( - f"REMOVE 'stage://tmp/{some_other_user}/tmp/12/15/file1.csv'" - ) - - with self.connection(extra_params={"staging_allowed_local_path": "/"}) as conn: - cursor = conn.cursor() - cursor.execute(remove_query) - - def perform_get(): - with self.connection(extra_params={"staging_allowed_local_path": temp_path}) as conn: - cursor = conn.cursor() - query = f"GET 'stage://tmp/{some_other_user}/tmp/11/15/file1.csv' TO '{temp_path}'" - cursor.execute(query) - - # PUT should fail with permissions error - with pytest.raises(sql.exc.ServerOperationError, match="PERMISSION_DENIED"): - perform_put() - - # REMOVE should fail with permissions error - with pytest.raises(sql.exc.ServerOperationError, match="PERMISSION_DENIED"): - perform_remove() - - # GET should fail with permissions error - with pytest.raises(sql.exc.ServerOperationError, match="PERMISSION_DENIED"): - perform_get() - - def test_staging_ingestion_put_fails_if_absolute_localFile_not_in_staging_allowed_local_path(self): - """ - This test confirms that staging_allowed_local_path and target_file are resolved into absolute paths. - """ - - # If these two paths are not resolved absolutely, they appear to share a common path of /var/www/html - # after resolution their common path is only /var/www which should raise an exception - # Because the common path must always be equal to staging_allowed_local_path - staging_allowed_local_path = "/var/www/html" - target_file = "/var/www/html/../html1/not_allowed.html" - - with pytest.raises(Error): - with self.connection(extra_params={"staging_allowed_local_path": staging_allowed_local_path}) as conn: - cursor = conn.cursor() - query = f"PUT '{target_file}' INTO 'stage://tmp/{self.staging_ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" - cursor.execute(query) - - def test_staging_ingestion_empty_local_path_fails_to_parse_at_server(self): - staging_allowed_local_path = "/var/www/html" - target_file = "" - - with pytest.raises(Error, match="EMPTY_LOCAL_FILE_IN_STAGING_ACCESS_QUERY"): - with self.connection(extra_params={"staging_allowed_local_path": staging_allowed_local_path}) as conn: - cursor = conn.cursor() - query = f"PUT '{target_file}' INTO 'stage://tmp/{self.staging_ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" - cursor.execute(query) - - def test_staging_ingestion_invalid_staging_path_fails_at_server(self): - staging_allowed_local_path = "/var/www/html" - target_file = "index.html" - - with pytest.raises(Error, match="INVALID_STAGING_PATH_IN_STAGING_ACCESS_QUERY"): - with self.connection(extra_params={"staging_allowed_local_path": staging_allowed_local_path}) as conn: - cursor = conn.cursor() - query = f"PUT '{target_file}' INTO 'stageRANDOMSTRINGOFCHARACTERS://tmp/{self.staging_ingestion_user}/tmp/11/15/file1.csv' OVERWRITE" - cursor.execute(query) - - def test_staging_ingestion_supports_multiple_staging_allowed_local_path_values(self): - """staging_allowed_local_path may be either a path-like object or a list of path-like objects. - - This test confirms that two configured base paths: - 1 - doesn't raise an exception - 2 - allows uploads from both paths - 3 - doesn't allow uploads from a third path - """ - - def generate_file_and_path_and_queries(): - """ - 1. Makes a temp file with some contents. - 2. Write a query to PUT it into a staging location - 3. Write a query to REMOVE it from that location (for cleanup) - """ - fh, temp_path = tempfile.mkstemp() - with open(fh, "wb") as fp: - original_text = "hello world!".encode("utf-8") - fp.write(original_text) - put_query = f"PUT '{temp_path}' INTO 'stage://tmp/{self.staging_ingestion_user}/tmp/11/15/{id(temp_path)}.csv' OVERWRITE" - remove_query = f"REMOVE 'stage://tmp/{self.staging_ingestion_user}/tmp/11/15/{id(temp_path)}.csv'" - return fh, temp_path, put_query, remove_query - - fh1, temp_path1, put_query1, remove_query1 = generate_file_and_path_and_queries() - fh2, temp_path2, put_query2, remove_query2 = generate_file_and_path_and_queries() - fh3, temp_path3, put_query3, remove_query3 = generate_file_and_path_and_queries() - - with self.connection(extra_params={"staging_allowed_local_path": [temp_path1, temp_path2]}) as conn: - cursor = conn.cursor() - - cursor.execute(put_query1) - cursor.execute(put_query2) - - with pytest.raises(Error, match="Local file operations are restricted to paths within the configured staging_allowed_local_path"): - cursor.execute(put_query3) - - # Then clean up the files we made - cursor.execute(remove_query1) - cursor.execute(remove_query2) - - -def main(cli_args): - global get_args_from_env - get_args_from_env = True - print(f"Running tests with version: {sql.__version__}") - logging.getLogger("databricks.sql").setLevel(logging.INFO) - unittest.main(module=__file__, argv=sys.argv[0:1] + cli_args) - - -if __name__ == "__main__": - main(sys.argv[1:]) \ No newline at end of file diff --git a/tests/e2e/sqlalchemy/test_basic.py b/tests/e2e/sqlalchemy/test_basic.py deleted file mode 100644 index 4f4df91b..00000000 --- a/tests/e2e/sqlalchemy/test_basic.py +++ /dev/null @@ -1,254 +0,0 @@ -import os, datetime, decimal -import pytest -from unittest import skipIf -from sqlalchemy import create_engine, select, insert, Column, MetaData, Table -from sqlalchemy.orm import declarative_base, Session -from sqlalchemy.types import SMALLINT, Integer, BOOLEAN, String, DECIMAL, Date - - -USER_AGENT_TOKEN = "PySQL e2e Tests" - - -@pytest.fixture -def db_engine(): - - HOST = os.environ.get("host") - HTTP_PATH = os.environ.get("http_path") - ACCESS_TOKEN = os.environ.get("access_token") - CATALOG = os.environ.get("catalog") - SCHEMA = os.environ.get("schema") - - connect_args = {"_user_agent_entry": USER_AGENT_TOKEN} - - engine = create_engine( - f"databricks://token:{ACCESS_TOKEN}@{HOST}?http_path={HTTP_PATH}&catalog={CATALOG}&schema={SCHEMA}", - connect_args=connect_args, - ) - return engine - - -@pytest.fixture() -def base(db_engine): - return declarative_base(bind=db_engine) - - -@pytest.fixture() -def session(db_engine): - return Session(bind=db_engine) - - -@pytest.fixture() -def metadata_obj(db_engine): - return MetaData(bind=db_engine) - - -def test_can_connect(db_engine): - simple_query = "SELECT 1" - result = db_engine.execute(simple_query).fetchall() - assert len(result) == 1 - - -def test_connect_args(db_engine): - """Verify that extra connect args passed to sqlalchemy.create_engine are passed to DBAPI - - This will most commonly happen when partners supply a user agent entry - """ - - conn = db_engine.connect() - connection_headers = conn.connection.thrift_backend._transport._headers - user_agent = connection_headers["User-Agent"] - - expected = f"(sqlalchemy + {USER_AGENT_TOKEN})" - assert expected in user_agent - - -def test_pandas_upload(db_engine, metadata_obj): - - import pandas as pd - - SCHEMA = os.environ.get("schema") - try: - df = pd.read_excel("tests/sqlalchemy/demo_data/MOCK_DATA.xlsx") - df.to_sql( - "mock_data", - db_engine, - schema=SCHEMA, - index=False, - method="multi", - if_exists="replace", - ) - - df_after = pd.read_sql_table("mock_data", db_engine, schema=SCHEMA) - assert len(df) == len(df_after) - except Exception as e: - raise e - finally: - db_engine.execute("DROP TABLE mock_data") - - -def test_create_table_not_null(db_engine, metadata_obj): - - table_name = "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")) - - SampleTable = Table( - table_name, - metadata_obj, - Column("name", String(255)), - Column("episodes", Integer), - Column("some_bool", BOOLEAN, nullable=False), - ) - - metadata_obj.create_all() - - columns = db_engine.dialect.get_columns( - connection=db_engine.connect(), table_name=table_name - ) - - name_column_description = columns[0] - some_bool_column_description = columns[2] - - assert name_column_description.get("nullable") is True - assert some_bool_column_description.get("nullable") is False - - metadata_obj.drop_all() - - -def test_bulk_insert_with_core(db_engine, metadata_obj, session): - - import random - - num_to_insert = random.choice(range(10_000, 20_000)) - - table_name = "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")) - - names = ["Bim", "Miki", "Sarah", "Ira"] - - SampleTable = Table( - table_name, metadata_obj, Column("name", String(255)), Column("number", Integer) - ) - - rows = [ - {"name": names[i % 3], "number": random.choice(range(10000))} - for i in range(num_to_insert) - ] - - metadata_obj.create_all() - db_engine.execute(insert(SampleTable).values(rows)) - - rows = db_engine.execute(select(SampleTable)).fetchall() - - assert len(rows) == num_to_insert - - -def test_create_insert_drop_table_core(base, db_engine, metadata_obj: MetaData): - """ """ - - SampleTable = Table( - "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")), - metadata_obj, - Column("name", String(255)), - Column("episodes", Integer), - Column("some_bool", BOOLEAN), - Column("dollars", DECIMAL(10, 2)), - ) - - metadata_obj.create_all() - - insert_stmt = insert(SampleTable).values( - name="Bim Adewunmi", episodes=6, some_bool=True, dollars=decimal.Decimal(125) - ) - - with db_engine.connect() as conn: - conn.execute(insert_stmt) - - select_stmt = select(SampleTable) - resp = db_engine.execute(select_stmt) - - result = resp.fetchall() - - assert len(result) == 1 - - metadata_obj.drop_all() - - -# ORM tests are made following this tutorial -# https://docs.sqlalchemy.org/en/14/orm/quickstart.html - - -@skipIf(False, "Unity catalog must be supported") -def test_create_insert_drop_table_orm(base, session: Session): - """ORM classes built on the declarative base class must have a primary key. - This is restricted to Unity Catalog. - """ - - class SampleObject(base): - - __tablename__ = "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")) - - name = Column(String(255), primary_key=True) - episodes = Column(Integer) - some_bool = Column(BOOLEAN) - - base.metadata.create_all() - - sample_object_1 = SampleObject(name="Bim Adewunmi", episodes=6, some_bool=True) - sample_object_2 = SampleObject(name="Miki Meek", episodes=12, some_bool=False) - session.add(sample_object_1) - session.add(sample_object_2) - session.commit() - - stmt = select(SampleObject).where( - SampleObject.name.in_(["Bim Adewunmi", "Miki Meek"]) - ) - - output = [i for i in session.scalars(stmt)] - assert len(output) == 2 - - base.metadata.drop_all() - - -def test_dialect_type_mappings(base, db_engine, metadata_obj: MetaData): - """Confirms that we get back the same time we declared in a model and inserted using Core""" - - SampleTable = Table( - "PySQLTest_{}".format(datetime.datetime.utcnow().strftime("%s")), - metadata_obj, - Column("string_example", String(255)), - Column("integer_example", Integer), - Column("boolean_example", BOOLEAN), - Column("decimal_example", DECIMAL(10, 2)), - Column("date_example", Date), - ) - - string_example = "" - integer_example = 100 - boolean_example = True - decimal_example = decimal.Decimal(125) - date_example = datetime.date(2013, 1, 1) - - metadata_obj.create_all() - - insert_stmt = insert(SampleTable).values( - string_example=string_example, - integer_example=integer_example, - boolean_example=boolean_example, - decimal_example=decimal_example, - date_example=date_example, - ) - - with db_engine.connect() as conn: - conn.execute(insert_stmt) - - select_stmt = select(SampleTable) - resp = db_engine.execute(select_stmt) - - result = resp.fetchall() - this_row = result[0] - - assert this_row["string_example"] == string_example - assert this_row["integer_example"] == integer_example - assert this_row["boolean_example"] == boolean_example - assert this_row["decimal_example"] == decimal_example - assert this_row["date_example"] == date_example - - metadata_obj.drop_all() diff --git a/tests/e2e/test_complex_types.py b/tests/e2e/test_complex_types.py new file mode 100644 index 00000000..0a7f514a --- /dev/null +++ b/tests/e2e/test_complex_types.py @@ -0,0 +1,61 @@ +import pytest +from numpy import ndarray + +from tests.e2e.test_driver import PySQLPytestTestCase + + +class TestComplexTypes(PySQLPytestTestCase): + @pytest.fixture(scope="class") + def table_fixture(self, connection_details): + self.arguments = connection_details.copy() + """A pytest fixture that creates a table with a complex type, inserts a record, yields, and then drops the table""" + + with self.cursor() as cursor: + # Create the table + cursor.execute( + """ + CREATE TABLE IF NOT EXISTS pysql_test_complex_types_table ( + array_col ARRAY, + map_col MAP, + struct_col STRUCT + ) + """ + ) + # Insert a record + cursor.execute( + """ + INSERT INTO pysql_test_complex_types_table + VALUES ( + ARRAY('a', 'b', 'c'), + MAP('a', 1, 'b', 2, 'c', 3), + NAMED_STRUCT('field1', 'a', 'field2', 1) + ) + """ + ) + yield + # Clean up the table after the test + cursor.execute("DROP TABLE IF EXISTS pysql_test_complex_types_table") + + @pytest.mark.parametrize( + "field,expected_type", + [("array_col", ndarray), ("map_col", list), ("struct_col", dict)], + ) + def test_read_complex_types_as_arrow(self, field, expected_type, table_fixture): + """Confirms the return types of a complex type field when reading as arrow""" + + with self.cursor() as cursor: + result = cursor.execute( + "SELECT * FROM pysql_test_complex_types_table LIMIT 1" + ).fetchone() + + assert isinstance(result[field], expected_type) + + @pytest.mark.parametrize("field", [("array_col"), ("map_col"), ("struct_col")]) + def test_read_complex_types_as_string(self, field, table_fixture): + """Confirms the return type of a complex type that is returned as a string""" + with self.cursor(extra_params={"_use_arrow_native_complex_types": False}) as cursor: + result = cursor.execute( + "SELECT * FROM pysql_test_complex_types_table LIMIT 1" + ).fetchone() + + assert isinstance(result[field], str) diff --git a/tests/e2e/test_driver.py b/tests/e2e/test_driver.py new file mode 100644 index 00000000..c23e4f79 --- /dev/null +++ b/tests/e2e/test_driver.py @@ -0,0 +1,759 @@ +import itertools +from contextlib import contextmanager +from collections import OrderedDict +import datetime +import io +import logging +import os +import sys +import threading +import time +from unittest import loader, skipIf, skipUnless, TestCase +from uuid import uuid4 + +import numpy as np +import pyarrow +import pytz +import thrift +import pytest +from urllib3.connectionpool import ReadTimeoutError + +import databricks.sql as sql +from databricks.sql import ( + STRING, + BINARY, + NUMBER, + DATETIME, + DATE, + DatabaseError, + Error, + OperationalError, + RequestError, +) +from tests.e2e.common.predicates import ( + pysql_has_version, + pysql_supports_arrow, + compare_dbr_versions, + is_thrift_v5_plus, +) +from tests.e2e.common.core_tests import CoreTestMixin, SmokeTestMixin +from tests.e2e.common.large_queries_mixin import LargeQueriesMixin +from tests.e2e.common.timestamp_tests import TimestampTestsMixin +from tests.e2e.common.decimal_tests import DecimalTestsMixin +from tests.e2e.common.retry_test_mixins import Client429ResponseMixin, Client503ResponseMixin +from tests.e2e.common.staging_ingestion_tests import PySQLStagingIngestionTestSuiteMixin +from tests.e2e.common.retry_test_mixins import PySQLRetryTestsMixin + +from tests.e2e.common.uc_volume_tests import PySQLUCVolumeTestSuiteMixin + +from databricks.sql.exc import SessionAlreadyClosedError + +log = logging.getLogger(__name__) + +unsafe_logger = logging.getLogger("databricks.sql.unsafe") +unsafe_logger.setLevel(logging.DEBUG) +unsafe_logger.addHandler(logging.FileHandler("./tests-unsafe.log")) + +# manually decorate DecimalTestsMixin to need arrow support +for name in loader.getTestCaseNames(DecimalTestsMixin, "test_"): + fn = getattr(DecimalTestsMixin, name) + decorated = skipUnless(pysql_supports_arrow(), "Decimal tests need arrow support")(fn) + setattr(DecimalTestsMixin, name, decorated) + + +class PySQLPytestTestCase: + """A mirror of PySQLTest case that doesn't inherit from unittest.TestCase + so that we can use pytest.mark.parameterize + """ + + error_type = Error + conf_to_disable_rate_limit_retries = {"_retry_stop_after_attempts_count": 1} + conf_to_disable_temporarily_unavailable_retries = {"_retry_stop_after_attempts_count": 1} + arraysize = 1000 + buffer_size_bytes = 104857600 + + @pytest.fixture(autouse=True) + def get_details(self, connection_details): + self.arguments = connection_details.copy() + + def connection_params(self): + params = { + "server_hostname": self.arguments["host"], + "http_path": self.arguments["http_path"], + **self.auth_params(), + } + + return params + + def auth_params(self): + return { + "access_token": self.arguments.get("access_token"), + } + + @contextmanager + def connection(self, extra_params=()): + connection_params = dict(self.connection_params(), **dict(extra_params)) + + log.info("Connecting with args: {}".format(connection_params)) + conn = sql.connect(**connection_params) + + try: + yield conn + finally: + conn.close() + + @contextmanager + def cursor(self, extra_params=()): + with self.connection(extra_params) as conn: + cursor = conn.cursor(arraysize=self.arraysize, buffer_size_bytes=self.buffer_size_bytes) + try: + yield cursor + finally: + cursor.close() + + def assertEqualRowValues(self, actual, expected): + len_actual = len(actual) if actual else 0 + len_expected = len(expected) if expected else 0 + assert len_actual == len_expected + for act, exp in zip(actual, expected): + assert len(act) == len(exp) + for i in range(len(act)): + assert act[i] == exp[i] + + +class TestPySQLLargeQueriesSuite(PySQLPytestTestCase, LargeQueriesMixin): + def get_some_rows(self, cursor, fetchmany_size): + row = cursor.fetchone() + if row: + return [row] + else: + return None + + @skipUnless(pysql_supports_arrow(), "needs arrow support") + @pytest.mark.skip("This test requires a previously uploaded data set") + def test_cloud_fetch(self): + # This test can take several minutes to run + limits = [100000, 300000] + threads = [10, 25] + self.arraysize = 100000 + # This test requires a large table with many rows to properly initiate cloud fetch. + # e2-dogfood host > hive_metastore catalog > main schema has such a table called store_sales. + # If this table is deleted or this test is run on a different host, a different table may need to be used. + base_query = "SELECT * FROM store_sales WHERE ss_sold_date_sk = 2452234 " + for num_limit, num_threads, lz4_compression in itertools.product( + limits, threads, [True, False] + ): + with self.subTest( + num_limit=num_limit, num_threads=num_threads, lz4_compression=lz4_compression + ): + cf_result, noop_result = None, None + query = base_query + "LIMIT " + str(num_limit) + with self.cursor( + { + "use_cloud_fetch": True, + "max_download_threads": num_threads, + "catalog": "hive_metastore", + }, + ) as cursor: + cursor.execute(query) + cf_result = cursor.fetchall() + with self.cursor({"catalog": "hive_metastore"}) as cursor: + cursor.execute(query) + noop_result = cursor.fetchall() + assert len(cf_result) == len(noop_result) + for i in range(len(cf_result)): + assert cf_result[i] == noop_result[i] + + +# Exclude Retry tests because they require specific setups, and LargeQueries too slow for core +# tests +class TestPySQLCoreSuite( + SmokeTestMixin, + CoreTestMixin, + DecimalTestsMixin, + TimestampTestsMixin, + PySQLPytestTestCase, + PySQLStagingIngestionTestSuiteMixin, + PySQLRetryTestsMixin, + PySQLUCVolumeTestSuiteMixin, +): + validate_row_value_type = True + validate_result = True + + # An output column in description evaluates to equal to multiple types + # - type code returned by the client as string. + # - also potentially a PEP-249 object like NUMBER, DATETIME etc. + def expected_column_types(self, type_): + type_mappings = { + "boolean": ["boolean", NUMBER], + "byte": ["tinyint", NUMBER], + "short": ["smallint", NUMBER], + "integer": ["int", NUMBER], + "long": ["bigint", NUMBER], + "decimal": ["decimal", NUMBER], + "timestamp": ["timestamp", DATETIME], + "date": ["date", DATE], + "binary": ["binary", BINARY], + "string": ["string", STRING], + "array": ["array"], + "struct": ["struct"], + "map": ["map"], + "double": ["double", NUMBER], + "null": ["null"], + } + return type_mappings[type_] + + def test_queries(self): + if not self._should_have_native_complex_types(): + array_type = str + array_val = "[1,2,3]" + struct_type = str + struct_val = '{"a":1,"b":2}' + map_type = str + map_val = "{1:2,3:4}" + else: + array_type = np.ndarray + array_val = np.array([1, 2, 3]) + struct_type = dict + struct_val = {"a": 1, "b": 2} + map_type = list + map_val = [(1, 2), (3, 4)] + + null_type = "null" if float(sql.__version__[0:2]) < 2.0 else "string" + self.range_queries = CoreTestMixin.range_queries + [ + ("NULL", null_type, type(None), None), + ("array(1, 2, 3)", "array", array_type, array_val), + ("struct(1 as a, 2 as b)", "struct", struct_type, struct_val), + ("map(1, 2, 3, 4)", "map", map_type, map_val), + ] + + self.run_tests_on_queries({}) + + @skipIf(pysql_has_version("<", "2"), "requires pysql v2") + def test_incorrect_query_throws_exception(self): + with self.cursor({}) as cursor: + # Syntax errors should contain the invalid SQL + with pytest.raises(DatabaseError) as cm: + cursor.execute("^ FOO BAR") + assert "FOO BAR" in str(cm.value) + + # Database error should contain the missing database + with pytest.raises(DatabaseError) as cm: + cursor.execute("USE foo234823498ydfsiusdhf") + assert "foo234823498ydfsiusdhf" in str(cm.value) + + # SQL with Extraneous input should send back the extraneous input + with pytest.raises(DatabaseError) as cm: + cursor.execute("CREATE TABLE IF NOT EXISTS TABLE table_234234234") + assert "table_234234234" in str(cm.value) + + def test_create_table_will_return_empty_result_set(self): + with self.cursor({}) as cursor: + table_name = "table_{uuid}".format(uuid=str(uuid4()).replace("-", "_")) + try: + cursor.execute( + "CREATE TABLE IF NOT EXISTS {} AS (SELECT 1 AS col_1, '2' AS col_2)".format( + table_name + ) + ) + assert cursor.fetchall() == [] + finally: + cursor.execute("DROP TABLE IF EXISTS {}".format(table_name)) + + def test_get_tables(self): + with self.cursor({}) as cursor: + table_name = "table_{uuid}".format(uuid=str(uuid4()).replace("-", "_")) + table_names = [table_name + "_1", table_name + "_2"] + + try: + for table in table_names: + cursor.execute( + "CREATE TABLE IF NOT EXISTS {} AS (SELECT 1 AS col_1, '2' AS col_2)".format( + table + ) + ) + cursor.tables(schema_name="defa%") + tables = cursor.fetchall() + tables_desc = cursor.description + + for table in table_names: + # Test only schema name and table name. + # From other columns, what is supported depends on DBR version. + assert ["default", table] in [list(table[1:3]) for table in tables] + expected = [ + ("TABLE_CAT", "string", None, None, None, None, None), + ("TABLE_SCHEM", "string", None, None, None, None, None), + ("TABLE_NAME", "string", None, None, None, None, None), + ("TABLE_TYPE", "string", None, None, None, None, None), + ("REMARKS", "string", None, None, None, None, None), + ("TYPE_CAT", "string", None, None, None, None, None), + ("TYPE_SCHEM", "string", None, None, None, None, None), + ("TYPE_NAME", "string", None, None, None, None, None), + ("SELF_REFERENCING_COL_NAME", "string", None, None, None, None, None), + ("REF_GENERATION", "string", None, None, None, None, None), + ] + assert tables_desc == expected + + finally: + for table in table_names: + cursor.execute("DROP TABLE IF EXISTS {}".format(table)) + + def test_get_columns(self): + with self.cursor({}) as cursor: + table_name = "table_{uuid}".format(uuid=str(uuid4()).replace("-", "_")) + table_names = [table_name + "_1", table_name + "_2"] + + try: + for table in table_names: + cursor.execute( + "CREATE TABLE IF NOT EXISTS {} AS (SELECT " + "1 AS col_1, " + "'2' AS col_2, " + "named_struct('name', 'alice', 'age', 28) as col_3, " + "map('items', 45, 'cost', 228) as col_4, " + "array('item1', 'item2', 'item3') as col_5)".format(table) + ) + + cursor.columns(schema_name="defa%", table_name=table_name + "%") + cols = cursor.fetchall() + cols_desc = cursor.description + + # Catalogue name not consistent across DBR versions, so we skip that + cleaned_response = [list(col[1:6]) for col in cols] + # We also replace ` as DBR changes how it represents struct names + for col in cleaned_response: + col[4] = col[4].replace("`", "") + + expected = [ + ["default", table_name + "_1", "col_1", 4, "INT"], + ["default", table_name + "_1", "col_2", 12, "STRING"], + [ + "default", + table_name + "_1", + "col_3", + 2002, + "STRUCT", + ], + ["default", table_name + "_1", "col_4", 2000, "MAP"], + ["default", table_name + "_1", "col_5", 2003, "ARRAY"], + ["default", table_name + "_2", "col_1", 4, "INT"], + ["default", table_name + "_2", "col_2", 12, "STRING"], + [ + "default", + table_name + "_2", + "col_3", + 2002, + "STRUCT", + ], + ["default", table_name + "_2", "col_4", 2000, "MAP"], + [ + "default", + table_name + "_2", + "col_5", + 2003, + "ARRAY", + ], + ] + assert cleaned_response == expected + expected = [ + ("TABLE_CAT", "string", None, None, None, None, None), + ("TABLE_SCHEM", "string", None, None, None, None, None), + ("TABLE_NAME", "string", None, None, None, None, None), + ("COLUMN_NAME", "string", None, None, None, None, None), + ("DATA_TYPE", "int", None, None, None, None, None), + ("TYPE_NAME", "string", None, None, None, None, None), + ("COLUMN_SIZE", "int", None, None, None, None, None), + ("BUFFER_LENGTH", "tinyint", None, None, None, None, None), + ("DECIMAL_DIGITS", "int", None, None, None, None, None), + ("NUM_PREC_RADIX", "int", None, None, None, None, None), + ("NULLABLE", "int", None, None, None, None, None), + ("REMARKS", "string", None, None, None, None, None), + ("COLUMN_DEF", "string", None, None, None, None, None), + ("SQL_DATA_TYPE", "int", None, None, None, None, None), + ("SQL_DATETIME_SUB", "int", None, None, None, None, None), + ("CHAR_OCTET_LENGTH", "int", None, None, None, None, None), + ("ORDINAL_POSITION", "int", None, None, None, None, None), + ("IS_NULLABLE", "string", None, None, None, None, None), + ("SCOPE_CATALOG", "string", None, None, None, None, None), + ("SCOPE_SCHEMA", "string", None, None, None, None, None), + ("SCOPE_TABLE", "string", None, None, None, None, None), + ("SOURCE_DATA_TYPE", "smallint", None, None, None, None, None), + ("IS_AUTO_INCREMENT", "string", None, None, None, None, None), + ] + assert cols_desc == expected + finally: + for table in table_names: + cursor.execute("DROP TABLE IF EXISTS {}".format(table)) + + def test_escape_single_quotes(self): + with self.cursor({}) as cursor: + table_name = "table_{uuid}".format(uuid=str(uuid4()).replace("-", "_")) + # Test escape syntax directly + cursor.execute( + "CREATE TABLE IF NOT EXISTS {} AS (SELECT 'you\\'re' AS col_1)".format(table_name) + ) + cursor.execute("SELECT * FROM {} WHERE col_1 LIKE 'you\\'re'".format(table_name)) + rows = cursor.fetchall() + assert rows[0]["col_1"] == "you're" + + # Test escape syntax in parameter + cursor.execute( + "SELECT * FROM {} WHERE {}.col_1 LIKE %(var)s".format(table_name, table_name), + parameters={"var": "you're"}, + ) + rows = cursor.fetchall() + assert rows[0]["col_1"] == "you're" + + def test_get_schemas(self): + with self.cursor({}) as cursor: + database_name = "db_{uuid}".format(uuid=str(uuid4()).replace("-", "_")) + try: + cursor.execute("CREATE DATABASE IF NOT EXISTS {}".format(database_name)) + cursor.schemas() + schemas = cursor.fetchall() + schemas_desc = cursor.description + # Catalogue name not consistent across DBR versions, so we skip that + assert database_name in [schema[0] for schema in schemas] + assert schemas_desc == [ + ("TABLE_SCHEM", "string", None, None, None, None, None), + ("TABLE_CATALOG", "string", None, None, None, None, None), + ] + + finally: + cursor.execute("DROP DATABASE IF EXISTS {}".format(database_name)) + + def test_get_catalogs(self): + with self.cursor({}) as cursor: + cursor.catalogs() + cursor.fetchall() + catalogs_desc = cursor.description + assert catalogs_desc == [("TABLE_CAT", "string", None, None, None, None, None)] + + @skipUnless(pysql_supports_arrow(), "arrow test need arrow support") + def test_get_arrow(self): + # These tests are quite light weight as the arrow fetch methods are used internally + # by everything else + with self.cursor({}) as cursor: + cursor.execute("SELECT * FROM range(10)") + table_1 = cursor.fetchmany_arrow(1).to_pydict() + assert table_1 == OrderedDict([("id", [0])]) + + table_2 = cursor.fetchall_arrow().to_pydict() + assert table_2 == OrderedDict([("id", [1, 2, 3, 4, 5, 6, 7, 8, 9])]) + + def test_unicode(self): + unicode_str = "数据砖" + with self.cursor({}) as cursor: + cursor.execute("SELECT '{}'".format(unicode_str)) + results = cursor.fetchall() + assert len(results) == 1 and len(results[0]) == 1 + assert results[0][0] == unicode_str + + def test_cancel_during_execute(self): + with self.cursor({}) as cursor: + + def execute_really_long_query(): + cursor.execute( + "SELECT SUM(A.id - B.id) " + + "FROM range(1000000000) A CROSS JOIN range(100000000) B " + + "GROUP BY (A.id - B.id)" + ) + + exec_thread = threading.Thread(target=execute_really_long_query) + + exec_thread.start() + # Make sure the query has started before cancelling + time.sleep(15) + cursor.cancel() + exec_thread.join(5) + assert not exec_thread.is_alive() + + # Fetching results should throw an exception + with pytest.raises((Error, thrift.Thrift.TException)): + cursor.fetchall() + with pytest.raises((Error, thrift.Thrift.TException)): + cursor.fetchone() + with pytest.raises((Error, thrift.Thrift.TException)): + cursor.fetchmany(10) + + # We should be able to execute a new command on the cursor + cursor.execute("SELECT * FROM range(3)") + assert len(cursor.fetchall()) == 3 + + @skipIf(pysql_has_version("<", "2"), "requires pysql v2") + def test_can_execute_command_after_failure(self): + with self.cursor({}) as cursor: + with pytest.raises(DatabaseError): + cursor.execute("this is a sytnax error") + + cursor.execute("SELECT 1;") + + res = cursor.fetchall() + self.assertEqualRowValues(res, [[1]]) + + @skipIf(pysql_has_version("<", "2"), "requires pysql v2") + def test_can_execute_command_after_success(self): + with self.cursor({}) as cursor: + cursor.execute("SELECT 1;") + cursor.execute("SELECT 2;") + + res = cursor.fetchall() + self.assertEqualRowValues(res, [[2]]) + + def generate_multi_row_query(self): + query = "SELECT * FROM range(3);" + return query + + @skipIf(pysql_has_version("<", "2"), "requires pysql v2") + def test_fetchone(self): + with self.cursor({}) as cursor: + query = self.generate_multi_row_query() + cursor.execute(query) + + assert cursor.fetchone()[0] == 0 + assert cursor.fetchone()[0] == 1 + assert cursor.fetchone()[0] == 2 + + assert cursor.fetchone() == None + + @skipIf(pysql_has_version("<", "2"), "requires pysql v2") + def test_fetchall(self): + with self.cursor({}) as cursor: + query = self.generate_multi_row_query() + cursor.execute(query) + + self.assertEqualRowValues(cursor.fetchall(), [[0], [1], [2]]) + + assert cursor.fetchone() == None + + @skipIf(pysql_has_version("<", "2"), "requires pysql v2") + def test_fetchmany_when_stride_fits(self): + with self.cursor({}) as cursor: + query = "SELECT * FROM range(4)" + cursor.execute(query) + + self.assertEqualRowValues(cursor.fetchmany(2), [[0], [1]]) + self.assertEqualRowValues(cursor.fetchmany(2), [[2], [3]]) + + @skipIf(pysql_has_version("<", "2"), "requires pysql v2") + def test_fetchmany_in_excess(self): + with self.cursor({}) as cursor: + query = "SELECT * FROM range(4)" + cursor.execute(query) + + self.assertEqualRowValues(cursor.fetchmany(3), [[0], [1], [2]]) + self.assertEqualRowValues(cursor.fetchmany(3), [[3]]) + + @skipIf(pysql_has_version("<", "2"), "requires pysql v2") + def test_iterator_api(self): + with self.cursor({}) as cursor: + query = "SELECT * FROM range(4)" + cursor.execute(query) + + expected_results = [[0], [1], [2], [3]] + for i, row in enumerate(cursor): + for j in range(len(row)): + assert row[j] == expected_results[i][j] + + def test_temp_view_fetch(self): + with self.cursor({}) as cursor: + query = "create temporary view f as select * from range(10)" + cursor.execute(query) + # TODO assert on a result + # once what is being returned has stabilised + + @skipIf(pysql_has_version("<", "2"), "requires pysql v2") + @skipIf( + True, "Unclear the purpose of this test since urllib3 does not complain when timeout == 0" + ) + def test_socket_timeout(self): + # We expect to see a BlockingIO error when the socket is opened + # in non-blocking mode, since no poll is done before the read + with pytest.raises(OperationalError) as cm: + with self.cursor({"_socket_timeout": 0}): + pass + + self.assertIsInstance(cm.exception.args[1], io.BlockingIOError) + + @skipIf(pysql_has_version("<", "2"), "requires pysql v2") + @skipIf(pysql_has_version(">", "2.8"), "This test has been broken for a while") + def test_socket_timeout_user_defined(self): + # We expect to see a TimeoutError when the socket timeout is only + # 1 sec for a query that takes longer than that to process + with pytest.raises(ReadTimeoutError) as cm: + with self.cursor({"_socket_timeout": 1}) as cursor: + query = "select * from range(1000000000)" + cursor.execute(query) + + def test_ssp_passthrough(self): + for enable_ansi in (True, False): + with self.cursor({"session_configuration": {"ansi_mode": enable_ansi}}) as cursor: + cursor.execute("SET ansi_mode") + assert list(cursor.fetchone()) == ["ansi_mode", str(enable_ansi)] + + @skipUnless(pysql_supports_arrow(), "arrow test needs arrow support") + def test_timestamps_arrow(self): + with self.cursor({"session_configuration": {"ansi_mode": False}}) as cursor: + for timestamp, expected in self.timestamp_and_expected_results: + cursor.execute("SELECT TIMESTAMP('{timestamp}')".format(timestamp=timestamp)) + arrow_table = cursor.fetchmany_arrow(1) + if self.should_add_timezone(): + ts_type = pyarrow.timestamp("us", tz="Etc/UTC") + else: + ts_type = pyarrow.timestamp("us") + assert arrow_table.field(0).type == ts_type + result_value = arrow_table.column(0).combine_chunks()[0].value + # To work consistently across different local timezones, we specify the timezone + # of the expected result to + # be UTC (what it should be by default on the server) + aware_timestamp = expected and expected.replace(tzinfo=datetime.timezone.utc) + assert result_value == ( + aware_timestamp and aware_timestamp.timestamp() * 1000000 + ), "timestamp {} did not match {}".format(timestamp, expected) + + @skipUnless(pysql_supports_arrow(), "arrow test needs arrow support") + def test_multi_timestamps_arrow(self): + with self.cursor({"session_configuration": {"ansi_mode": False}}) as cursor: + query, expected = self.multi_query() + expected = [ + [self.maybe_add_timezone_to_timestamp(ts) for ts in row] for row in expected + ] + cursor.execute(query) + table = cursor.fetchall_arrow() + # Transpose columnar result to list of rows + list_of_cols = [c.to_pylist() for c in table] + result = [ + [col[row_index] for col in list_of_cols] for row_index in range(table.num_rows) + ] + assert result == expected + + @skipUnless(pysql_supports_arrow(), "arrow test needs arrow support") + def test_timezone_with_timestamp(self): + if self.should_add_timezone(): + with self.cursor() as cursor: + cursor.execute("SET TIME ZONE 'Europe/Amsterdam'") + cursor.execute("select CAST('2022-03-02 12:54:56' as TIMESTAMP)") + amsterdam = pytz.timezone("Europe/Amsterdam") + expected = amsterdam.localize(datetime.datetime(2022, 3, 2, 12, 54, 56)) + result = cursor.fetchone()[0] + assert result == expected + + cursor.execute("select CAST('2022-03-02 12:54:56' as TIMESTAMP)") + arrow_result_table = cursor.fetchmany_arrow(1) + arrow_result_value = arrow_result_table.column(0).combine_chunks()[0].value + ts_type = pyarrow.timestamp("us", tz="Europe/Amsterdam") + + assert arrow_result_table.field(0).type == ts_type + assert arrow_result_value == expected.timestamp() * 1000000 + + @skipUnless(pysql_supports_arrow(), "arrow test needs arrow support") + def test_can_flip_compression(self): + with self.cursor() as cursor: + cursor.execute("SELECT array(1,2,3,4)") + cursor.fetchall() + lz4_compressed = cursor.active_result_set.lz4_compressed + # The endpoint should support compression + assert lz4_compressed + cursor.connection.lz4_compression = False + cursor.execute("SELECT array(1,2,3,4)") + cursor.fetchall() + lz4_compressed = cursor.active_result_set.lz4_compressed + assert not lz4_compressed + + def _should_have_native_complex_types(self): + return pysql_has_version(">=", 2) and is_thrift_v5_plus(self.arguments) + + @skipUnless(pysql_supports_arrow(), "arrow test needs arrow support") + def test_arrays_are_not_returned_as_strings_arrow(self): + if self._should_have_native_complex_types(): + with self.cursor() as cursor: + cursor.execute("SELECT array(1,2,3,4)") + arrow_df = cursor.fetchall_arrow() + + list_type = arrow_df.field(0).type + assert pyarrow.types.is_list(list_type) + assert pyarrow.types.is_integer(list_type.value_type) + + @skipUnless(pysql_supports_arrow(), "arrow test needs arrow support") + def test_structs_are_not_returned_as_strings_arrow(self): + if self._should_have_native_complex_types(): + with self.cursor() as cursor: + cursor.execute("SELECT named_struct('foo', 42, 'bar', 'baz')") + arrow_df = cursor.fetchall_arrow() + + struct_type = arrow_df.field(0).type + assert pyarrow.types.is_struct(struct_type) + + @skipUnless(pysql_supports_arrow(), "arrow test needs arrow support") + def test_decimal_not_returned_as_strings_arrow(self): + if self._should_have_native_complex_types(): + with self.cursor() as cursor: + cursor.execute("SELECT 5E3BD") + arrow_df = cursor.fetchall_arrow() + + decimal_type = arrow_df.field(0).type + assert pyarrow.types.is_decimal(decimal_type) + + def test_close_connection_closes_cursors(self): + + from databricks.sql.thrift_api.TCLIService import ttypes + + with self.connection() as conn: + cursor = conn.cursor() + cursor.execute("SELECT id, id `id2`, id `id3` FROM RANGE(1000000) order by RANDOM()") + ars = cursor.active_result_set + + # We must manually run this check because thrift_backend always forces `has_been_closed_server_side` to True + + # Cursor op state should be open before connection is closed + status_request = ttypes.TGetOperationStatusReq( + operationHandle=ars.command_id, getProgressUpdate=False + ) + op_status_at_server = ars.thrift_backend._client.GetOperationStatus(status_request) + assert op_status_at_server.operationState != ttypes.TOperationState.CLOSED_STATE + + conn.close() + + # When connection closes, any cursor operations should no longer exist at the server + with pytest.raises(SessionAlreadyClosedError) as cm: + op_status_at_server = ars.thrift_backend._client.GetOperationStatus(status_request) + + def test_closing_a_closed_connection_doesnt_fail(self, caplog): + caplog.set_level(logging.DEBUG) + # Second .close() call is when this context manager exits + with self.connection() as conn: + # First .close() call is explicit here + conn.close() + + assert "Session appears to have been closed already" in caplog.text + + +# use a RetrySuite to encapsulate these tests which we'll typically want to run together; however keep +# the 429/503 subsuites separate since they execute under different circumstances. +class TestPySQLRetrySuite: + class HTTP429Suite(Client429ResponseMixin, PySQLPytestTestCase): + pass # Mixin covers all + + class HTTP503Suite(Client503ResponseMixin, PySQLPytestTestCase): + # 503Response suite gets custom error here vs PyODBC + def test_retry_disabled(self): + self._test_retry_disabled_with_message("TEMPORARILY_UNAVAILABLE", OperationalError) + + +class TestPySQLUnityCatalogSuite(PySQLPytestTestCase): + """Simple namespace tests that should be run against a unity-catalog-enabled cluster""" + + @skipIf(pysql_has_version("<", "2"), "requires pysql v2") + def test_initial_namespace(self): + table_name = "table_{uuid}".format(uuid=str(uuid4()).replace("-", "_")) + with self.cursor() as cursor: + cursor.execute("USE CATALOG {}".format(self.arguments["catalog"])) + cursor.execute("CREATE TABLE table_{} (col1 int)".format(table_name)) + with self.connection( + {"catalog": self.arguments["catalog"], "schema": table_name} + ) as connection: + cursor = connection.cursor() + cursor.execute("select current_catalog()") + assert cursor.fetchone()[0] == self.arguments["catalog"] + cursor.execute("select current_database()") + assert cursor.fetchone()[0] == table_name diff --git a/tests/e2e/test_parameterized_queries.py b/tests/e2e/test_parameterized_queries.py new file mode 100644 index 00000000..47dfc38c --- /dev/null +++ b/tests/e2e/test_parameterized_queries.py @@ -0,0 +1,453 @@ +import datetime +from contextlib import contextmanager +from decimal import Decimal +from enum import Enum +from typing import Dict, List, Type, Union +from unittest.mock import patch + +import pytest +import pytz + +from databricks.sql.parameters.native import ( + BigIntegerParameter, + BooleanParameter, + DateParameter, + DbsqlParameterBase, + DecimalParameter, + DoubleParameter, + FloatParameter, + IntegerParameter, + ParameterApproach, + ParameterStructure, + SmallIntParameter, + StringParameter, + TDbsqlParameter, + TimestampNTZParameter, + TimestampParameter, + TinyIntParameter, + VoidParameter, +) +from tests.e2e.test_driver import PySQLPytestTestCase + + +class ParamStyle(Enum): + NAMED = 1 + PYFORMAT = 2 + NONE = 3 + + +class Primitive(Enum): + """These are the inferrable types. This Enum is used for parametrized tests.""" + + NONE = None + BOOL = True + INT = 50 + BIGINT = 2147483648 + STRING = "Hello" + DECIMAL = Decimal("1234.56") + DATE = datetime.date(2023, 9, 6) + TIMESTAMP = datetime.datetime(2023, 9, 6, 3, 14, 27, 843, tzinfo=pytz.UTC) + DOUBLE = 3.14 + FLOAT = 3.15 + SMALLINT = 51 + + +class PrimitiveExtra(Enum): + """These are not inferrable types. This Enum is used for parametrized tests.""" + + TIMESTAMP_NTZ = datetime.datetime(2023, 9, 6, 3, 14, 27, 843) + TINYINT = 20 + + +# We don't test inline approach with named paramstyle because it's never supported +# We don't test inline approach with positional parameters because it's never supported +# Paramstyle doesn't apply when ParameterStructure.POSITIONAL because question marks are used. +approach_paramstyle_combinations = [ + (ParameterApproach.INLINE, ParamStyle.PYFORMAT, ParameterStructure.NAMED), + (ParameterApproach.NATIVE, ParamStyle.NONE, ParameterStructure.POSITIONAL), + (ParameterApproach.NATIVE, ParamStyle.PYFORMAT, ParameterStructure.NAMED), + (ParameterApproach.NATIVE, ParamStyle.NONE, ParameterStructure.POSITIONAL), + (ParameterApproach.NATIVE, ParamStyle.NAMED, ParameterStructure.NAMED), +] + + +class TestParameterizedQueries(PySQLPytestTestCase): + """Namespace for tests of this connector's parameterisation behaviour. + + databricks-sql-connector can approach parameterisation in two ways: + + NATIVE: the connector will use server-side bound parameters implemented by DBR 14.1 and above. + INLINE: the connector will render parameter values as strings and interpolate them into the query. + + Prior to connector version 3.0.0, the connector would always use the INLINE approach. This approach + is still the default but this will be changed in a subsequent release. + + The INLINE and NATIVE approaches use different query syntax, which these tests verify. + + There is not 1-to-1 feature parity between these approaches. Where possible, we run the same test + for @both_approaches. + """ + + NAMED_PARAMSTYLE_QUERY = "SELECT :p AS col" + PYFORMAT_PARAMSTYLE_QUERY = "SELECT %(p)s AS col" + POSITIONAL_PARAMSTYLE_QUERY = "SELECT ? AS col" + + inline_type_map = { + Primitive.INT: "int_col", + Primitive.BIGINT: "bigint_col", + Primitive.SMALLINT: "small_int_col", + Primitive.FLOAT: "float_col", + Primitive.DOUBLE: "double_col", + Primitive.DECIMAL: "decimal_col", + Primitive.STRING: "string_col", + Primitive.BOOL: "boolean_col", + Primitive.DATE: "date_col", + Primitive.TIMESTAMP: "timestamp_col", + Primitive.NONE: "null_col", + } + + def _get_inline_table_column(self, value): + return self.inline_type_map[Primitive(value)] + + @pytest.fixture(scope="class") + def inline_table(self, connection_details): + self.arguments = connection_details.copy() + """This table is necessary to verify that a parameter sent with INLINE + approach can actually write to its analogous data type. + + For example, a Python Decimal(), when rendered inline, should be able + to read/write into a DECIMAL column in Databricks + + Note that this fixture doesn't clean itself up. So the table will remain + in the schema for use by subsequent test runs. + """ + + query = """ + CREATE TABLE IF NOT EXISTS pysql_e2e_inline_param_test_table ( + null_col INT, + int_col INT, + bigint_col BIGINT, + small_int_col SMALLINT, + float_col FLOAT, + double_col DOUBLE, + decimal_col DECIMAL(10, 2), + string_col STRING, + boolean_col BOOLEAN, + date_col DATE, + timestamp_col TIMESTAMP + ) USING DELTA + """ + + with self.connection() as conn: + with conn.cursor() as cursor: + cursor.execute(query) + + @contextmanager + def patch_server_supports_native_params(self, supports_native_params: bool = True): + """Applies a patch so we can test the connector's behaviour under different SPARK_CLI_SERVICE_PROTOCOL_VERSION conditions.""" + + with patch( + "databricks.sql.client.Connection.server_parameterized_queries_enabled", + return_value=supports_native_params, + ) as mock_parameterized_queries_enabled: + try: + yield mock_parameterized_queries_enabled + finally: + pass + + def _inline_roundtrip(self, params: dict, paramstyle: ParamStyle): + """This INSERT, SELECT, DELETE dance is necessary because simply selecting + ``` + "SELECT %(param)s" + ``` + in INLINE mode would always return a str and the nature of the test is to + confirm that types are maintained. + + :paramstyle: + This is a no-op but is included to make the test-code easier to read. + """ + target_column = self._get_inline_table_column(params.get("p")) + INSERT_QUERY = ( + f"INSERT INTO pysql_e2e_inline_param_test_table (`{target_column}`) VALUES (%(p)s)" + ) + SELECT_QUERY = ( + f"SELECT {target_column} `col` FROM pysql_e2e_inline_param_test_table LIMIT 1" + ) + DELETE_QUERY = "DELETE FROM pysql_e2e_inline_param_test_table" + + with self.connection(extra_params={"use_inline_params": True}) as conn: + with conn.cursor() as cursor: + cursor.execute(INSERT_QUERY, parameters=params) + with conn.cursor() as cursor: + to_return = cursor.execute(SELECT_QUERY).fetchone() + with conn.cursor() as cursor: + cursor.execute(DELETE_QUERY) + + return to_return + + def _native_roundtrip( + self, + parameters: Union[Dict, List[Dict]], + paramstyle: ParamStyle, + parameter_structure: ParameterStructure, + ): + if parameter_structure == ParameterStructure.POSITIONAL: + _query = self.POSITIONAL_PARAMSTYLE_QUERY + elif paramstyle == ParamStyle.NAMED: + _query = self.NAMED_PARAMSTYLE_QUERY + elif paramstyle == ParamStyle.PYFORMAT: + _query = self.PYFORMAT_PARAMSTYLE_QUERY + with self.connection(extra_params={"use_inline_params": False}) as conn: + with conn.cursor() as cursor: + cursor.execute(_query, parameters=parameters) + return cursor.fetchone() + + def _get_one_result( + self, + params, + approach: ParameterApproach = ParameterApproach.NONE, + paramstyle: ParamStyle = ParamStyle.NONE, + parameter_structure: ParameterStructure = ParameterStructure.NONE, + ): + """When approach is INLINE then we use %(param)s paramstyle and a connection with use_inline_params=True + When approach is NATIVE then we use :param paramstyle and a connection with use_inline_params=False + """ + + if approach == ParameterApproach.INLINE: + # inline mode always uses ParamStyle.PYFORMAT + # inline mode doesn't support positional parameters + return self._inline_roundtrip(params, paramstyle=ParamStyle.PYFORMAT) + elif approach == ParameterApproach.NATIVE: + # native mode can use either ParamStyle.NAMED or ParamStyle.PYFORMAT + # native mode can use either ParameterStructure.NAMED or ParameterStructure.POSITIONAL + return self._native_roundtrip( + params, paramstyle=paramstyle, parameter_structure=parameter_structure + ) + + def _quantize(self, input: Union[float, int], place_value=2) -> Decimal: + return Decimal(str(input)).quantize(Decimal("0." + "0" * place_value)) + + def _eq(self, actual, expected: Primitive): + """This is a helper function to make the test code more readable. + + If primitive is Primitive.DOUBLE than an extra quantize step is performed before + making the assertion. + """ + if expected in (Primitive.DOUBLE, Primitive.FLOAT): + return self._quantize(actual) == self._quantize(expected.value) + + return actual == expected.value + + @pytest.mark.parametrize("primitive", Primitive) + @pytest.mark.parametrize( + "approach,paramstyle,parameter_structure", approach_paramstyle_combinations + ) + def test_primitive_single( + self, + approach, + paramstyle, + parameter_structure, + primitive: Primitive, + inline_table, + ): + """When ParameterApproach.INLINE is passed, inferrence will not be used. + When ParameterApproach.NATIVE is passed, primitive inputs will be inferred. + """ + + if parameter_structure == ParameterStructure.NAMED: + params = {"p": primitive.value} + elif parameter_structure == ParameterStructure.POSITIONAL: + params = [primitive.value] + + result = self._get_one_result(params, approach, paramstyle, parameter_structure) + + assert self._eq(result.col, primitive) + + @pytest.mark.parametrize( + "parameter_structure", (ParameterStructure.NAMED, ParameterStructure.POSITIONAL) + ) + @pytest.mark.parametrize( + "primitive,dbsql_parameter_cls", + [ + (Primitive.NONE, VoidParameter), + (Primitive.BOOL, BooleanParameter), + (Primitive.INT, IntegerParameter), + (Primitive.BIGINT, BigIntegerParameter), + (Primitive.STRING, StringParameter), + (Primitive.DECIMAL, DecimalParameter), + (Primitive.DATE, DateParameter), + (Primitive.TIMESTAMP, TimestampParameter), + (Primitive.DOUBLE, DoubleParameter), + (Primitive.FLOAT, FloatParameter), + (Primitive.SMALLINT, SmallIntParameter), + (PrimitiveExtra.TIMESTAMP_NTZ, TimestampNTZParameter), + (PrimitiveExtra.TINYINT, TinyIntParameter), + ], + ) + def test_dbsqlparameter_single( + self, + primitive: Primitive, + dbsql_parameter_cls: Type[TDbsqlParameter], + parameter_structure: ParameterStructure, + ): + dbsql_param = dbsql_parameter_cls( + value=primitive.value, # type: ignore + name="p" if parameter_structure == ParameterStructure.NAMED else None, + ) + + params = [dbsql_param] + result = self._get_one_result( + params, ParameterApproach.NATIVE, ParamStyle.NAMED, parameter_structure + ) + assert self._eq(result.col, primitive) + + @pytest.mark.parametrize("use_inline_params", (True, False, "silent")) + def test_use_inline_off_by_default_with_warning(self, use_inline_params, caplog): + """ + use_inline_params should be False by default. + If a user explicitly sets use_inline_params, don't warn them about it. + """ + + extra_args = {"use_inline_params": use_inline_params} if use_inline_params else {} + + with self.connection(extra_params=extra_args) as conn: + with conn.cursor() as cursor: + with self.patch_server_supports_native_params(supports_native_params=True): + cursor.execute("SELECT %(p)s", parameters={"p": 1}) + if use_inline_params is True: + assert ( + "Consider using native parameters." in caplog.text + ), "Log message should be suppressed" + elif use_inline_params == "silent": + assert ( + "Consider using native parameters." not in caplog.text + ), "Log message should not be supressed" + + def test_positional_native_params_with_defaults(self): + query = "SELECT ? col" + with self.cursor() as cursor: + result = cursor.execute(query, parameters=[1]).fetchone() + + assert result.col == 1 + + @pytest.mark.parametrize( + "params", + ( + [ + StringParameter(value="foo"), + StringParameter(value="bar"), + StringParameter(value="baz"), + ], + ["foo", "bar", "baz"], + ), + ) + def test_positional_native_multiple(self, params): + query = "SELECT ? `foo`, ? `bar`, ? `baz`" + + with self.cursor(extra_params={"use_inline_params": False}) as cursor: + result = cursor.execute(query, params).fetchone() + + expected = [i.value if isinstance(i, DbsqlParameterBase) else i for i in params] + outcome = [result.foo, result.bar, result.baz] + + assert set(outcome) == set(expected) + + def test_readme_example(self): + with self.cursor() as cursor: + result = cursor.execute( + "SELECT :param `p`, * FROM RANGE(10)", {"param": "foo"} + ).fetchall() + + assert len(result) == 10 + assert result[0].p == "foo" + + +class TestInlineParameterSyntax(PySQLPytestTestCase): + """The inline parameter approach uses pyformat markers""" + + def test_params_as_dict(self): + query = "SELECT %(foo)s foo, %(bar)s bar, %(baz)s baz" + params = {"foo": 1, "bar": 2, "baz": 3} + + with self.connection(extra_params={"use_inline_params": True}) as conn: + with conn.cursor() as cursor: + result = cursor.execute(query, parameters=params).fetchone() + + assert result.foo == 1 + assert result.bar == 2 + assert result.baz == 3 + + def test_params_as_sequence(self): + """One side-effect of ParamEscaper using Python string interpolation to inline the values + is that it can work with "ordinal" parameters, but only if a user writes parameter markers + that are not defined with PEP-249. This test exists to prove that it works in the ideal case. + """ + + # `%s` is not a valid paramstyle per PEP-249 + query = "SELECT %s foo, %s bar, %s baz" + params = (1, 2, 3) + + with self.connection(extra_params={"use_inline_params": True}) as conn: + with conn.cursor() as cursor: + result = cursor.execute(query, parameters=params).fetchone() + assert result.foo == 1 + assert result.bar == 2 + assert result.baz == 3 + + def test_inline_ordinals_can_break_sql(self): + """With inline mode, ordinal parameters can break the SQL syntax + because `%` symbols are used to wildcard match within LIKE statements. This test + just proves that's the case. + """ + query = "SELECT 'samsonite', %s WHERE 'samsonite' LIKE '%sonite'" + params = ["luggage"] + with self.cursor(extra_params={"use_inline_params": True}) as cursor: + with pytest.raises(TypeError, match="not enough arguments for format string"): + cursor.execute(query, parameters=params) + + def test_inline_named_dont_break_sql(self): + """With inline mode, ordinal parameters can break the SQL syntax + because `%` symbols are used to wildcard match within LIKE statements. This test + just proves that's the case. + """ + query = """ + with base as (SELECT 'x(one)sonite' as `col_1`) + SELECT col_1 FROM base WHERE col_1 LIKE CONCAT(%(one)s, 'onite') + """ + params = {"one": "%(one)s"} + with self.cursor(extra_params={"use_inline_params": True}) as cursor: + result = cursor.execute(query, parameters=params).fetchone() + print("hello") + + def test_native_ordinals_dont_break_sql(self): + """This test accompanies test_inline_ordinals_can_break_sql to prove that ordinal + parameters work in native mode for the exact same query, if we use the right marker `?` + """ + query = "SELECT 'samsonite', ? WHERE 'samsonite' LIKE '%sonite'" + params = ["luggage"] + with self.cursor(extra_params={"use_inline_params": False}) as cursor: + result = cursor.execute(query, parameters=params).fetchone() + + assert result.samsonite == "samsonite" + assert result.luggage == "luggage" + + def test_inline_like_wildcard_breaks(self): + """One flaw with the ParameterEscaper is that it fails if a query contains + a SQL LIKE wildcard %. This test proves that's the case. + """ + query = "SELECT 1 `col` WHERE 'foo' LIKE '%'" + params = {"param": "bar"} + with self.cursor(extra_params={"use_inline_params": True}) as cursor: + with pytest.raises(ValueError, match="unsupported format character"): + result = cursor.execute(query, parameters=params).fetchone() + + def test_native_like_wildcard_works(self): + """This is a mirror of test_inline_like_wildcard_breaks that proves that LIKE + wildcards work under the native approach. + """ + query = "SELECT 1 `col` WHERE 'foo' LIKE '%'" + params = {"param": "bar"} + with self.cursor(extra_params={"use_inline_params": False}) as cursor: + result = cursor.execute(query, parameters=params).fetchone() + + assert result.col == 1 diff --git a/tests/unit/test_auth.py b/tests/unit/test_auth.py index c52f9790..d6541525 100644 --- a/tests/unit/test_auth.py +++ b/tests/unit/test_auth.py @@ -1,103 +1,188 @@ import unittest - -from databricks.sql.auth.auth import AccessTokenAuthProvider, BasicAuthProvider, AuthProvider, ExternalAuthProvider -from databricks.sql.auth.auth import get_python_sql_connector_auth_provider +import pytest +from typing import Optional +from unittest.mock import patch + +from databricks.sql.auth.auth import ( + AccessTokenAuthProvider, + AuthProvider, + ExternalAuthProvider, + AuthType, +) +from databricks.sql.auth.auth import get_python_sql_connector_auth_provider, PYSQL_OAUTH_CLIENT_ID +from databricks.sql.auth.oauth import OAuthManager +from databricks.sql.auth.authenticators import DatabricksOAuthProvider +from databricks.sql.auth.endpoint import ( + CloudType, + InHouseOAuthEndpointCollection, + AzureOAuthEndpointCollection, +) from databricks.sql.auth.authenticators import CredentialsProvider, HeaderFactory +from databricks.sql.experimental.oauth_persistence import OAuthPersistenceCache class Auth(unittest.TestCase): - def test_access_token_provider(self): access_token = "aBc2" auth = AccessTokenAuthProvider(access_token=access_token) - http_request = {'myKey': 'myVal'} - auth.add_headers(http_request) - self.assertEqual(http_request['Authorization'], 'Bearer aBc2') - self.assertEqual(len(http_request.keys()), 2) - self.assertEqual(http_request['myKey'], 'myVal') - - def test_basic_auth_provider(self): - username = "moderakh" - password = "Elevate Databricks 123!!!" - auth = BasicAuthProvider(username=username, password=password) - - http_request = {'myKey': 'myVal'} + http_request = {"myKey": "myVal"} auth.add_headers(http_request) - - self.assertEqual(http_request['Authorization'], 'Basic bW9kZXJha2g6RWxldmF0ZSBEYXRhYnJpY2tzIDEyMyEhIQ==') + self.assertEqual(http_request["Authorization"], "Bearer aBc2") self.assertEqual(len(http_request.keys()), 2) - self.assertEqual(http_request['myKey'], 'myVal') + self.assertEqual(http_request["myKey"], "myVal") def test_noop_auth_provider(self): auth = AuthProvider() - http_request = {'myKey': 'myVal'} + http_request = {"myKey": "myVal"} auth.add_headers(http_request) self.assertEqual(len(http_request.keys()), 1) - self.assertEqual(http_request['myKey'], 'myVal') + self.assertEqual(http_request["myKey"], "myVal") + + @patch.object(OAuthManager, "check_and_refresh_access_token") + @patch.object(OAuthManager, "get_tokens") + def test_oauth_auth_provider(self, mock_get_tokens, mock_check_and_refresh): + client_id = "mock-id" + scopes = ["offline_access", "sql"] + access_token = "mock_token" + refresh_token = "mock_refresh_token" + mock_get_tokens.return_value = (access_token, refresh_token) + mock_check_and_refresh.return_value = (access_token, refresh_token, False) + + params = [ + ( + CloudType.AWS, + "foo.cloud.databricks.com", + False, + InHouseOAuthEndpointCollection, + "offline_access sql", + ), + ( + CloudType.AZURE, + "foo.1.azuredatabricks.net", + True, + AzureOAuthEndpointCollection, + f"{AzureOAuthEndpointCollection.DATATRICKS_AZURE_APP}/user_impersonation offline_access", + ), + ( + CloudType.AZURE, + "foo.1.azuredatabricks.net", + False, + InHouseOAuthEndpointCollection, + "offline_access sql", + ), + ( + CloudType.GCP, + "foo.gcp.databricks.com", + False, + InHouseOAuthEndpointCollection, + "offline_access sql", + ), + ] + + for ( + cloud_type, + host, + use_azure_auth, + expected_endpoint_type, + expected_scopes, + ) in params: + with self.subTest(cloud_type.value): + oauth_persistence = OAuthPersistenceCache() + auth_provider = DatabricksOAuthProvider( + hostname=host, + oauth_persistence=oauth_persistence, + redirect_port_range=[8020], + client_id=client_id, + scopes=scopes, + auth_type=AuthType.AZURE_OAUTH.value + if use_azure_auth + else AuthType.DATABRICKS_OAUTH.value, + ) + + self.assertIsInstance( + auth_provider.oauth_manager.idp_endpoint, expected_endpoint_type + ) + self.assertEqual(auth_provider.oauth_manager.port_range, [8020]) + self.assertEqual(auth_provider.oauth_manager.client_id, client_id) + self.assertEqual( + oauth_persistence.read(host).refresh_token, refresh_token + ) + mock_get_tokens.assert_called_with(hostname=host, scope=expected_scopes) + + headers = {} + auth_provider.add_headers(headers) + self.assertEqual(headers["Authorization"], f"Bearer {access_token}") def test_external_provider(self): class MyProvider(CredentialsProvider): - def auth_type(self) -> str: - return "mine" + def auth_type(self) -> str: + return "mine" - def __call__(self, *args, **kwargs) -> HeaderFactory: - return lambda: {"foo": "bar"} + def __call__(self, *args, **kwargs) -> HeaderFactory: + return lambda: {"foo": "bar"} auth = ExternalAuthProvider(MyProvider()) - http_request = {'myKey': 'myVal'} + http_request = {"myKey": "myVal"} auth.add_headers(http_request) - self.assertEqual(http_request['foo'], 'bar') + self.assertEqual(http_request["foo"], "bar") self.assertEqual(len(http_request.keys()), 2) - self.assertEqual(http_request['myKey'], 'myVal') + self.assertEqual(http_request["myKey"], "myVal") def test_get_python_sql_connector_auth_provider_access_token(self): hostname = "moderakh-test.cloud.databricks.com" - kwargs = {'access_token': 'dpi123'} + kwargs = {"access_token": "dpi123"} auth_provider = get_python_sql_connector_auth_provider(hostname, **kwargs) self.assertTrue(type(auth_provider).__name__, "AccessTokenAuthProvider") headers = {} auth_provider.add_headers(headers) - self.assertEqual(headers['Authorization'], 'Bearer dpi123') + self.assertEqual(headers["Authorization"], "Bearer dpi123") def test_get_python_sql_connector_auth_provider_external(self): - class MyProvider(CredentialsProvider): - def auth_type(self) -> str: - return "mine" + def auth_type(self) -> str: + return "mine" - def __call__(self, *args, **kwargs) -> HeaderFactory: - return lambda: {"foo": "bar"} + def __call__(self, *args, **kwargs) -> HeaderFactory: + return lambda: {"foo": "bar"} hostname = "moderakh-test.cloud.databricks.com" - kwargs = {'credentials_provider': MyProvider()} + kwargs = {"credentials_provider": MyProvider()} auth_provider = get_python_sql_connector_auth_provider(hostname, **kwargs) self.assertTrue(type(auth_provider).__name__, "ExternalAuthProvider") headers = {} auth_provider.add_headers(headers) - self.assertEqual(headers['foo'], 'bar') - - def test_get_python_sql_connector_auth_provider_username_password(self): - username = "moderakh" - password = "Elevate Databricks 123!!!" - hostname = "moderakh-test.cloud.databricks.com" - kwargs = {'_username': username, '_password': password} - auth_provider = get_python_sql_connector_auth_provider(hostname, **kwargs) - self.assertTrue(type(auth_provider).__name__, "BasicAuthProvider") - - headers = {} - auth_provider.add_headers(headers) - self.assertEqual(headers['Authorization'], 'Basic bW9kZXJha2g6RWxldmF0ZSBEYXRhYnJpY2tzIDEyMyEhIQ==') + self.assertEqual(headers["foo"], "bar") def test_get_python_sql_connector_auth_provider_noop(self): tls_client_cert_file = "fake.cert" use_cert_as_auth = "abc" hostname = "moderakh-test.cloud.databricks.com" - kwargs = {'_tls_client_cert_file': tls_client_cert_file, '_use_cert_as_auth': use_cert_as_auth} + kwargs = { + "_tls_client_cert_file": tls_client_cert_file, + "_use_cert_as_auth": use_cert_as_auth, + } auth_provider = get_python_sql_connector_auth_provider(hostname, **kwargs) self.assertTrue(type(auth_provider).__name__, "CredentialProvider") + + def test_get_python_sql_connector_basic_auth(self): + kwargs = { + "username": "username", + "password": "password", + } + with self.assertRaises(ValueError) as e: + get_python_sql_connector_auth_provider("foo.cloud.databricks.com", **kwargs) + self.assertIn("Username/password authentication is no longer supported", str(e.exception)) + + @patch.object(DatabricksOAuthProvider, "_initial_get_token") + def test_get_python_sql_connector_default_auth(self, mock__initial_get_token): + hostname = "foo.cloud.databricks.com" + auth_provider = get_python_sql_connector_auth_provider(hostname) + self.assertTrue(type(auth_provider).__name__, "DatabricksOAuthProvider") + self.assertTrue(auth_provider._client_id,PYSQL_OAUTH_CLIENT_ID) + diff --git a/tests/unit/tests.py b/tests/unit/test_client.py similarity index 80% rename from tests/unit/tests.py rename to tests/unit/test_client.py index 74274373..c86a9f7f 100644 --- a/tests/unit/tests.py +++ b/tests/unit/test_client.py @@ -2,10 +2,20 @@ import re import sys import unittest -from unittest.mock import patch, MagicMock, Mock +from unittest.mock import patch, MagicMock, Mock, PropertyMock import itertools from decimal import Decimal from datetime import datetime, date +from uuid import UUID + +from databricks.sql.thrift_api.TCLIService.ttypes import ( + TOpenSessionResp, + TExecuteStatementResp, + TOperationHandle, + THandleIdentifier, + TOperationType +) +from databricks.sql.thrift_backend import ThriftBackend import databricks.sql import databricks.sql.client as client @@ -16,6 +26,51 @@ from tests.unit.test_thrift_backend import ThriftBackendTestSuite from tests.unit.test_arrow_queue import ArrowQueueSuite +class ThriftBackendMockFactory: + + @classmethod + def new(cls): + ThriftBackendMock = Mock(spec=ThriftBackend) + ThriftBackendMock.return_value = ThriftBackendMock + + cls.apply_property_to_mock(ThriftBackendMock, staging_allowed_local_path=None) + MockTExecuteStatementResp = MagicMock(spec=TExecuteStatementResp()) + + cls.apply_property_to_mock( + MockTExecuteStatementResp, + description=None, + arrow_queue=None, + is_staging_operation=False, + command_handle=b"\x22", + has_been_closed_server_side=True, + has_more_rows=True, + lz4_compressed=True, + arrow_schema_bytes=b"schema", + ) + + ThriftBackendMock.execute_command.return_value = MockTExecuteStatementResp + + return ThriftBackendMock + + @classmethod + def apply_property_to_mock(self, mock_obj, **kwargs): + """ + Apply a property to a mock object. + """ + + for key, value in kwargs.items(): + if value is not None: + kwargs = {"return_value": value} + else: + kwargs = {} + + prop = PropertyMock(**kwargs) + setattr(type(mock_obj), key, prop) + + + + + class ClientTestSuite(unittest.TestCase): """ @@ -32,20 +87,22 @@ class ClientTestSuite(unittest.TestCase): @patch("%s.client.ThriftBackend" % PACKAGE_NAME) def test_close_uses_the_correct_session_id(self, mock_client_class): instance = mock_client_class.return_value - instance.open_session.return_value = b'\x22' + + mock_open_session_resp = MagicMock(spec=TOpenSessionResp)() + mock_open_session_resp.sessionHandle.sessionId = b'\x22' + instance.open_session.return_value = mock_open_session_resp connection = databricks.sql.connect(**self.DUMMY_CONNECTION_ARGS) connection.close() # Check the close session request has an id of x22 - close_session_id = instance.close_session.call_args[0][0] + close_session_id = instance.close_session.call_args[0][0].sessionId self.assertEqual(close_session_id, b'\x22') @patch("%s.client.ThriftBackend" % PACKAGE_NAME) def test_auth_args(self, mock_client_class): # Test that the following auth args work: # token = foo, - # token = None, _username = foo, _password = bar # token = None, _tls_client_cert_file = something, _use_cert_as_auth = True connection_args = [ { @@ -53,13 +110,6 @@ def test_auth_args(self, mock_client_class): "http_path": None, "access_token": "tok", }, - { - "server_hostname": "foo", - "http_path": None, - "_username": "foo", - "_password": "bar", - "access_token": None, - }, { "server_hostname": "foo", "http_path": None, @@ -71,7 +121,7 @@ def test_auth_args(self, mock_client_class): for args in connection_args: connection = databricks.sql.connect(**args) - host, port, http_path, _ = mock_client_class.call_args[0] + host, port, http_path, *_ = mock_client_class.call_args[0] self.assertEqual(args["server_hostname"], host) self.assertEqual(args["http_path"], http_path) connection.close() @@ -84,14 +134,6 @@ def test_http_header_passthrough(self, mock_client_class): call_args = mock_client_class.call_args[0][3] self.assertIn(("foo", "bar"), call_args) - @patch("%s.client.ThriftBackend" % PACKAGE_NAME) - def test_authtoken_passthrough(self, mock_client_class): - databricks.sql.connect(**self.DUMMY_CONNECTION_ARGS) - - headers = mock_client_class.call_args[0][3] - - self.assertIn(("Authorization", "Bearer tok"), headers) - @patch("%s.client.ThriftBackend" % PACKAGE_NAME) def test_tls_arg_passthrough(self, mock_client_class): databricks.sql.connect( @@ -123,9 +165,9 @@ def test_useragent_header(self, mock_client_class): http_headers = mock_client_class.call_args[0][3] self.assertIn(user_agent_header_with_entry, http_headers) - @patch("%s.client.ThriftBackend" % PACKAGE_NAME) + @patch("%s.client.ThriftBackend" % PACKAGE_NAME, ThriftBackendMockFactory.new()) @patch("%s.client.ResultSet" % PACKAGE_NAME) - def test_closing_connection_closes_commands(self, mock_result_set_class, mock_client_class): + def test_closing_connection_closes_commands(self, mock_result_set_class): # Test once with has_been_closed_server side, once without for closed in (True, False): with self.subTest(closed=closed): @@ -185,10 +227,11 @@ def test_closing_result_set_hard_closes_commands(self): @patch("%s.client.ResultSet" % PACKAGE_NAME) def test_executing_multiple_commands_uses_the_most_recent_command(self, mock_result_set_class): + mock_result_sets = [Mock(), Mock()] mock_result_set_class.side_effect = mock_result_sets - cursor = client.Cursor(Mock(), Mock()) + cursor = client.Cursor(connection=Mock(), thrift_backend=ThriftBackendMockFactory.new()) cursor.execute("SELECT 1;") cursor.execute("SELECT 1;") @@ -227,13 +270,16 @@ def test_context_manager_closes_cursor(self): @patch("%s.client.ThriftBackend" % PACKAGE_NAME) def test_context_manager_closes_connection(self, mock_client_class): instance = mock_client_class.return_value - instance.open_session.return_value = b'\x22' + + mock_open_session_resp = MagicMock(spec=TOpenSessionResp)() + mock_open_session_resp.sessionHandle.sessionId = b'\x22' + instance.open_session.return_value = mock_open_session_resp with databricks.sql.connect(**self.DUMMY_CONNECTION_ARGS) as connection: pass # Check the close session request has an id of x22 - close_session_id = instance.close_session.call_args[0][0] + close_session_id = instance.close_session.call_args[0][0].sessionId self.assertEqual(close_session_id, b'\x22') def dict_product(self, dicts): @@ -315,7 +361,7 @@ def test_cancel_command_calls_the_backend(self): mock_op_handle = Mock() cursor.active_op_handle = mock_op_handle cursor.cancel() - self.assertTrue(mock_thrift_backend.cancel_command.called_with(mock_op_handle)) + mock_thrift_backend.cancel_command.assert_called_with(mock_op_handle) @patch("databricks.sql.client.logger") def test_cancel_command_will_issue_warning_for_cancel_with_no_executing_command( @@ -363,39 +409,39 @@ def test_initial_namespace_passthrough(self, mock_client_class): self.assertEqual(mock_client_class.return_value.open_session.call_args[0][2], mock_schem) def test_execute_parameter_passthrough(self): - mock_thrift_backend = Mock() + mock_thrift_backend = ThriftBackendMockFactory.new() cursor = client.Cursor(Mock(), mock_thrift_backend) - tests = [("SELECT %(string_v)s", "SELECT 'foo_12345'", { - "string_v": "foo_12345" - }), ("SELECT %(x)s", "SELECT NULL", { - "x": None - }), ("SELECT %(int_value)d", "SELECT 48", { - "int_value": 48 - }), ("SELECT %(float_value).2f", "SELECT 48.20", { - "float_value": 48.2 - }), ("SELECT %(iter)s", "SELECT (1,2,3,4,5)", { - "iter": [1, 2, 3, 4, 5] - }), - ("SELECT %(datetime)s", "SELECT '2022-02-01 10:23:00.000000'", { - "datetime": datetime(2022, 2, 1, 10, 23) - }), ("SELECT %(date)s", "SELECT '2022-02-01'", { - "date": date(2022, 2, 1) - })] + tests = [ + ("SELECT %(string_v)s", "SELECT 'foo_12345'", {"string_v": "foo_12345"}), + ("SELECT %(x)s", "SELECT NULL", {"x": None}), + ("SELECT %(int_value)d", "SELECT 48", {"int_value": 48}), + ("SELECT %(float_value).2f", "SELECT 48.20", {"float_value": 48.2}), + ("SELECT %(iter)s", "SELECT (1,2,3,4,5)", {"iter": [1, 2, 3, 4, 5]}), + ( + "SELECT %(datetime)s", + "SELECT '2022-02-01 10:23:00.000000'", + {"datetime": datetime(2022, 2, 1, 10, 23)}, + ), + ("SELECT %(date)s", "SELECT '2022-02-01'", {"date": date(2022, 2, 1)}), + ] for query, expected_query, params in tests: cursor.execute(query, parameters=params) - self.assertEqual(mock_thrift_backend.execute_command.call_args[1]["operation"], - expected_query) + self.assertEqual( + mock_thrift_backend.execute_command.call_args[1]["operation"], + expected_query, + ) + @patch("%s.client.ThriftBackend" % PACKAGE_NAME) @patch("%s.client.ResultSet" % PACKAGE_NAME) def test_executemany_parameter_passhthrough_and_uses_last_result_set( - self, mock_result_set_class): + self, mock_result_set_class, mock_thrift_backend): # Create a new mock result set each time the class is instantiated mock_result_set_instances = [Mock(), Mock(), Mock()] mock_result_set_class.side_effect = mock_result_set_instances - mock_thrift_backend = Mock() - cursor = client.Cursor(Mock(), mock_thrift_backend) + mock_thrift_backend = ThriftBackendMockFactory.new() + cursor = client.Cursor(Mock(), mock_thrift_backend()) params = [{"x": None}, {"x": "foo1"}, {"x": "bar2"}] expected_queries = ["SELECT NULL", "SELECT 'foo1'", "SELECT 'bar2'"] @@ -434,6 +480,7 @@ def test_rollback_not_supported(self, mock_thrift_backend_class): with self.assertRaises(NotSupportedError): c.rollback() + @unittest.skip("JDW: skipping winter 2024 as we're about to rewrite this interface") @patch("%s.client.ThriftBackend" % PACKAGE_NAME) def test_row_number_respected(self, mock_thrift_backend_class): def make_fake_row_slice(n_rows): @@ -458,6 +505,7 @@ def make_fake_row_slice(n_rows): cursor.fetchmany_arrow(6) self.assertEqual(cursor.rownumber, 29) + @unittest.skip("JDW: skipping winter 2024 as we're about to rewrite this interface") @patch("%s.client.ThriftBackend" % PACKAGE_NAME) def test_disable_pandas_respected(self, mock_thrift_backend_class): mock_thrift_backend = mock_thrift_backend_class.return_value @@ -509,7 +557,10 @@ def test_column_name_api(self): @patch("%s.client.ThriftBackend" % PACKAGE_NAME) def test_finalizer_closes_abandoned_connection(self, mock_client_class): instance = mock_client_class.return_value - instance.open_session.return_value = b'\x22' + + mock_open_session_resp = MagicMock(spec=TOpenSessionResp)() + mock_open_session_resp.sessionHandle.sessionId = b'\x22' + instance.open_session.return_value = mock_open_session_resp databricks.sql.connect(**self.DUMMY_CONNECTION_ARGS) @@ -517,13 +568,16 @@ def test_finalizer_closes_abandoned_connection(self, mock_client_class): gc.collect() # Check the close session request has an id of x22 - close_session_id = instance.close_session.call_args[0][0] + close_session_id = instance.close_session.call_args[0][0].sessionId self.assertEqual(close_session_id, b'\x22') @patch("%s.client.ThriftBackend" % PACKAGE_NAME) def test_cursor_keeps_connection_alive(self, mock_client_class): instance = mock_client_class.return_value - instance.open_session.return_value = b'\x22' + + mock_open_session_resp = MagicMock(spec=TOpenSessionResp)() + mock_open_session_resp.sessionHandle.sessionId = b'\x22' + instance.open_session.return_value = mock_open_session_resp connection = databricks.sql.connect(**self.DUMMY_CONNECTION_ARGS) cursor = connection.cursor() @@ -534,20 +588,40 @@ def test_cursor_keeps_connection_alive(self, mock_client_class): self.assertEqual(instance.close_session.call_count, 0) cursor.close() - @patch("%s.client.ThriftBackend" % PACKAGE_NAME) + @patch("%s.utils.ExecuteResponse" % PACKAGE_NAME, autospec=True) @patch("%s.client.Cursor._handle_staging_operation" % PACKAGE_NAME) - @patch("%s.utils.ExecuteResponse" % PACKAGE_NAME) + @patch("%s.client.ThriftBackend" % PACKAGE_NAME) def test_staging_operation_response_is_handled(self, mock_client_class, mock_handle_staging_operation, mock_execute_response): # If server sets ExecuteResponse.is_staging_operation True then _handle_staging_operation should be called - mock_execute_response.is_staging_operation = True - + + ThriftBackendMockFactory.apply_property_to_mock(mock_execute_response, is_staging_operation=True) + mock_client_class.execute_command.return_value = mock_execute_response + mock_client_class.return_value = mock_client_class + connection = databricks.sql.connect(**self.DUMMY_CONNECTION_ARGS) cursor = connection.cursor() cursor.execute("Text of some staging operation command;") connection.close() - mock_handle_staging_operation.assert_called_once_with() + mock_handle_staging_operation.call_count == 1 + + @patch("%s.client.ThriftBackend" % PACKAGE_NAME, ThriftBackendMockFactory.new()) + def test_access_current_query_id(self): + operation_id = 'EE6A8778-21FC-438B-92D8-96AC51EE3821' + + connection = databricks.sql.connect(**self.DUMMY_CONNECTION_ARGS) + cursor = connection.cursor() + + self.assertIsNone(cursor.query_id) + + cursor.active_op_handle = TOperationHandle( + operationId=THandleIdentifier(guid=UUID(operation_id).bytes, secret=0x00), + operationType=TOperationType.EXECUTE_STATEMENT) + self.assertEqual(cursor.query_id.upper(), operation_id.upper()) + + cursor.close() + self.assertIsNone(cursor.query_id) if __name__ == '__main__': diff --git a/tests/unit/test_cloud_fetch_queue.py b/tests/unit/test_cloud_fetch_queue.py new file mode 100644 index 00000000..acd0c392 --- /dev/null +++ b/tests/unit/test_cloud_fetch_queue.py @@ -0,0 +1,301 @@ +import pyarrow +import unittest +from unittest.mock import MagicMock, patch + +from databricks.sql.thrift_api.TCLIService.ttypes import TSparkArrowResultLink +import databricks.sql.utils as utils +from databricks.sql.types import SSLOptions + +class CloudFetchQueueSuite(unittest.TestCase): + + def create_result_link( + self, + file_link: str = "fileLink", + start_row_offset: int = 0, + row_count: int = 8000, + bytes_num: int = 20971520 + ): + return TSparkArrowResultLink(file_link, None, start_row_offset, row_count, bytes_num) + + def create_result_links(self, num_files: int, start_row_offset: int = 0): + result_links = [] + for i in range(num_files): + file_link = "fileLink_" + str(i) + result_link = self.create_result_link(file_link=file_link, start_row_offset=start_row_offset) + result_links.append(result_link) + start_row_offset += result_link.rowCount + return result_links + + @staticmethod + def make_arrow_table(): + batch = [[0, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]] + n_cols = len(batch[0]) if batch else 0 + schema = pyarrow.schema({"col%s" % i: pyarrow.uint32() for i in range(n_cols)}) + cols = [[batch[row][col] for row in range(len(batch))] for col in range(n_cols)] + return pyarrow.Table.from_pydict(dict(zip(schema.names, cols)), schema=schema) + + @staticmethod + def get_schema_bytes(): + schema = pyarrow.schema({"col%s" % i: pyarrow.uint32() for i in range(4)}) + sink = pyarrow.BufferOutputStream() + writer = pyarrow.ipc.RecordBatchStreamWriter(sink, schema) + writer.close() + return sink.getvalue().to_pybytes() + + + @patch("databricks.sql.utils.CloudFetchQueue._create_next_table", return_value=[None, None]) + def test_initializer_adds_links(self, mock_create_next_table): + schema_bytes = MagicMock() + result_links = self.create_result_links(10) + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=result_links, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + + assert len(queue.download_manager._pending_links) == 10 + assert len(queue.download_manager._download_tasks) == 0 + mock_create_next_table.assert_called() + + def test_initializer_no_links_to_add(self): + schema_bytes = MagicMock() + result_links = [] + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=result_links, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + + assert len(queue.download_manager._pending_links) == 0 + assert len(queue.download_manager._download_tasks) == 0 + assert queue.table is None + + @patch("databricks.sql.cloudfetch.download_manager.ResultFileDownloadManager.get_next_downloaded_file", return_value=None) + def test_create_next_table_no_download(self, mock_get_next_downloaded_file): + queue = utils.CloudFetchQueue( + MagicMock(), + result_links=[], + max_download_threads=10, + ssl_options=SSLOptions(), + ) + + assert queue._create_next_table() is None + mock_get_next_downloaded_file.assert_called_with(0) + + @patch("databricks.sql.utils.create_arrow_table_from_arrow_file") + @patch("databricks.sql.cloudfetch.download_manager.ResultFileDownloadManager.get_next_downloaded_file", + return_value=MagicMock(file_bytes=b"1234567890", row_count=4)) + def test_initializer_create_next_table_success(self, mock_get_next_downloaded_file, mock_create_arrow_table): + mock_create_arrow_table.return_value = self.make_arrow_table() + schema_bytes, description = MagicMock(), MagicMock() + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=[], + description=description, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + expected_result = self.make_arrow_table() + + mock_get_next_downloaded_file.assert_called_with(0) + mock_create_arrow_table.assert_called_with(b"1234567890", description) + assert queue.table == expected_result + assert queue.table.num_rows == 4 + assert queue.table_row_index == 0 + assert queue.start_row_index == 4 + + table = queue._create_next_table() + assert table == expected_result + assert table.num_rows == 4 + assert queue.start_row_index == 8 + + @patch("databricks.sql.utils.CloudFetchQueue._create_next_table") + def test_next_n_rows_0_rows(self, mock_create_next_table): + mock_create_next_table.return_value = self.make_arrow_table() + schema_bytes, description = MagicMock(), MagicMock() + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=[], + description=description, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + assert queue.table == self.make_arrow_table() + assert queue.table.num_rows == 4 + assert queue.table_row_index == 0 + + result = queue.next_n_rows(0) + assert result.num_rows == 0 + assert queue.table_row_index == 0 + assert result == self.make_arrow_table()[0:0] + + @patch("databricks.sql.utils.CloudFetchQueue._create_next_table") + def test_next_n_rows_partial_table(self, mock_create_next_table): + mock_create_next_table.return_value = self.make_arrow_table() + schema_bytes, description = MagicMock(), MagicMock() + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=[], + description=description, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + assert queue.table == self.make_arrow_table() + assert queue.table.num_rows == 4 + assert queue.table_row_index == 0 + + result = queue.next_n_rows(3) + assert result.num_rows == 3 + assert queue.table_row_index == 3 + assert result == self.make_arrow_table()[:3] + + @patch("databricks.sql.utils.CloudFetchQueue._create_next_table") + def test_next_n_rows_more_than_one_table(self, mock_create_next_table): + mock_create_next_table.return_value = self.make_arrow_table() + schema_bytes, description = MagicMock(), MagicMock() + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=[], + description=description, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + assert queue.table == self.make_arrow_table() + assert queue.table.num_rows == 4 + assert queue.table_row_index == 0 + + result = queue.next_n_rows(7) + assert result.num_rows == 7 + assert queue.table_row_index == 3 + assert result == pyarrow.concat_tables([self.make_arrow_table(), self.make_arrow_table()])[:7] + + @patch("databricks.sql.utils.CloudFetchQueue._create_next_table") + def test_next_n_rows_only_one_table_returned(self, mock_create_next_table): + mock_create_next_table.side_effect = [self.make_arrow_table(), None] + schema_bytes, description = MagicMock(), MagicMock() + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=[], + description=description, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + assert queue.table == self.make_arrow_table() + assert queue.table.num_rows == 4 + assert queue.table_row_index == 0 + + result = queue.next_n_rows(7) + assert result.num_rows == 4 + assert result == self.make_arrow_table() + + @patch("databricks.sql.utils.CloudFetchQueue._create_next_table", return_value=None) + def test_next_n_rows_empty_table(self, mock_create_next_table): + schema_bytes = self.get_schema_bytes() + description = MagicMock() + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=[], + description=description, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + assert queue.table is None + + result = queue.next_n_rows(100) + mock_create_next_table.assert_called() + assert result == pyarrow.ipc.open_stream(bytearray(schema_bytes)).read_all() + + @patch("databricks.sql.utils.CloudFetchQueue._create_next_table") + def test_remaining_rows_empty_table_fully_returned(self, mock_create_next_table): + mock_create_next_table.side_effect = [self.make_arrow_table(), None, 0] + schema_bytes, description = MagicMock(), MagicMock() + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=[], + description=description, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + assert queue.table == self.make_arrow_table() + assert queue.table.num_rows == 4 + queue.table_row_index = 4 + + result = queue.remaining_rows() + assert result.num_rows == 0 + assert result == self.make_arrow_table()[0:0] + + @patch("databricks.sql.utils.CloudFetchQueue._create_next_table") + def test_remaining_rows_partial_table_fully_returned(self, mock_create_next_table): + mock_create_next_table.side_effect = [self.make_arrow_table(), None] + schema_bytes, description = MagicMock(), MagicMock() + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=[], + description=description, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + assert queue.table == self.make_arrow_table() + assert queue.table.num_rows == 4 + queue.table_row_index = 2 + + result = queue.remaining_rows() + assert result.num_rows == 2 + assert result == self.make_arrow_table()[2:] + + @patch("databricks.sql.utils.CloudFetchQueue._create_next_table") + def test_remaining_rows_one_table_fully_returned(self, mock_create_next_table): + mock_create_next_table.side_effect = [self.make_arrow_table(), None] + schema_bytes, description = MagicMock(), MagicMock() + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=[], + description=description, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + assert queue.table == self.make_arrow_table() + assert queue.table.num_rows == 4 + assert queue.table_row_index == 0 + + result = queue.remaining_rows() + assert result.num_rows == 4 + assert result == self.make_arrow_table() + + @patch("databricks.sql.utils.CloudFetchQueue._create_next_table") + def test_remaining_rows_multiple_tables_fully_returned(self, mock_create_next_table): + mock_create_next_table.side_effect = [self.make_arrow_table(), self.make_arrow_table(), None] + schema_bytes, description = MagicMock(), MagicMock() + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=[], + description=description, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + assert queue.table == self.make_arrow_table() + assert queue.table.num_rows == 4 + queue.table_row_index = 3 + + result = queue.remaining_rows() + assert mock_create_next_table.call_count == 3 + assert result.num_rows == 5 + assert result == pyarrow.concat_tables([self.make_arrow_table(), self.make_arrow_table()])[3:] + + @patch("databricks.sql.utils.CloudFetchQueue._create_next_table", return_value=None) + def test_remaining_rows_empty_table(self, mock_create_next_table): + schema_bytes = self.get_schema_bytes() + description = MagicMock() + queue = utils.CloudFetchQueue( + schema_bytes, + result_links=[], + description=description, + max_download_threads=10, + ssl_options=SSLOptions(), + ) + assert queue.table is None + + result = queue.remaining_rows() + assert result == pyarrow.ipc.open_stream(bytearray(schema_bytes)).read_all() diff --git a/tests/unit/test_column_queue.py b/tests/unit/test_column_queue.py new file mode 100644 index 00000000..130b589b --- /dev/null +++ b/tests/unit/test_column_queue.py @@ -0,0 +1,22 @@ +from databricks.sql.utils import ColumnQueue, ColumnTable + + +class TestColumnQueueSuite: + @staticmethod + def make_column_table(table): + n_cols = len(table) if table else 0 + return ColumnTable(table, [f"col_{i}" for i in range(n_cols)]) + + def test_fetchmany_respects_n_rows(self): + column_table = self.make_column_table([[0, 3, 6, 9], [1, 4, 7, 10], [2, 5, 8, 11]]) + column_queue = ColumnQueue(column_table) + + assert column_queue.next_n_rows(2) == column_table.slice(0, 2) + assert column_queue.next_n_rows(2) == column_table.slice(2, 2) + + def test_fetch_remaining_rows_respects_n_rows(self): + column_table = self.make_column_table([[0, 3, 6, 9], [1, 4, 7, 10], [2, 5, 8, 11]]) + column_queue = ColumnQueue(column_table) + + assert column_queue.next_n_rows(2) == column_table.slice(0, 2) + assert column_queue.remaining_rows() == column_table.slice(2, 2) diff --git a/tests/unit/test_download_manager.py b/tests/unit/test_download_manager.py new file mode 100644 index 00000000..a11bc8d4 --- /dev/null +++ b/tests/unit/test_download_manager.py @@ -0,0 +1,63 @@ +import unittest +from unittest.mock import patch, MagicMock + +import databricks.sql.cloudfetch.download_manager as download_manager +from databricks.sql.types import SSLOptions +from databricks.sql.thrift_api.TCLIService.ttypes import TSparkArrowResultLink + + +class DownloadManagerTests(unittest.TestCase): + """ + Unit tests for checking download manager logic. + """ + + def create_download_manager(self, links, max_download_threads=10, lz4_compressed=True): + return download_manager.ResultFileDownloadManager( + links, + max_download_threads, + lz4_compressed, + ssl_options=SSLOptions(), + ) + + def create_result_link( + self, + file_link: str = "fileLink", + start_row_offset: int = 0, + row_count: int = 8000, + bytes_num: int = 20971520 + ): + return TSparkArrowResultLink(file_link, None, start_row_offset, row_count, bytes_num) + + def create_result_links(self, num_files: int, start_row_offset: int = 0): + result_links = [] + for i in range(num_files): + file_link = "fileLink_" + str(i) + result_link = self.create_result_link(file_link=file_link, start_row_offset=start_row_offset) + result_links.append(result_link) + start_row_offset += result_link.rowCount + return result_links + + def test_add_file_links_zero_row_count(self): + links = [self.create_result_link(row_count=0, bytes_num=0)] + manager = self.create_download_manager(links) + + assert len(manager._pending_links) == 0 # the only link supplied contains no data, so should be skipped + assert len(manager._download_tasks) == 0 + + def test_add_file_links_success(self): + links = self.create_result_links(num_files=10) + manager = self.create_download_manager(links) + + assert len(manager._pending_links) == len(links) + assert len(manager._download_tasks) == 0 + + @patch("concurrent.futures.ThreadPoolExecutor.submit") + def test_schedule_downloads(self, mock_submit): + max_download_threads = 4 + links = self.create_result_links(num_files=10) + manager = self.create_download_manager(links, max_download_threads=max_download_threads) + + manager._schedule_downloads() + assert mock_submit.call_count == max_download_threads + assert len(manager._pending_links) == len(links) - max_download_threads + assert len(manager._download_tasks) == max_download_threads diff --git a/tests/unit/test_downloader.py b/tests/unit/test_downloader.py new file mode 100644 index 00000000..7075ef6c --- /dev/null +++ b/tests/unit/test_downloader.py @@ -0,0 +1,119 @@ +import unittest +from unittest.mock import Mock, patch, MagicMock + +import requests + +import databricks.sql.cloudfetch.downloader as downloader +from databricks.sql.exc import Error +from databricks.sql.types import SSLOptions + + +def create_response(**kwargs) -> requests.Response: + result = requests.Response() + for k, v in kwargs.items(): + setattr(result, k, v) + return result + + +class DownloaderTests(unittest.TestCase): + """ + Unit tests for checking downloader logic. + """ + + @patch('time.time', return_value=1000) + def test_run_link_expired(self, mock_time): + settings = Mock() + result_link = Mock() + # Already expired + result_link.expiryTime = 999 + d = downloader.ResultSetDownloadHandler(settings, result_link, ssl_options=SSLOptions()) + + with self.assertRaises(Error) as context: + d.run() + self.assertTrue('link has expired' in context.exception.message) + + mock_time.assert_called_once() + + @patch('time.time', return_value=1000) + def test_run_link_past_expiry_buffer(self, mock_time): + settings = Mock(link_expiry_buffer_secs=5) + result_link = Mock() + # Within the expiry buffer time + result_link.expiryTime = 1004 + d = downloader.ResultSetDownloadHandler(settings, result_link, ssl_options=SSLOptions()) + + with self.assertRaises(Error) as context: + d.run() + self.assertTrue('link has expired' in context.exception.message) + + mock_time.assert_called_once() + + @patch('requests.Session', return_value=MagicMock(get=MagicMock(return_value=None))) + @patch('time.time', return_value=1000) + def test_run_get_response_not_ok(self, mock_time, mock_session): + mock_session.return_value.get.return_value = create_response(status_code=404) + + settings = Mock(link_expiry_buffer_secs=0, download_timeout=0) + settings.download_timeout = 0 + settings.use_proxy = False + result_link = Mock(expiryTime=1001) + + d = downloader.ResultSetDownloadHandler(settings, result_link, ssl_options=SSLOptions()) + with self.assertRaises(requests.exceptions.HTTPError) as context: + d.run() + self.assertTrue('404' in str(context.exception)) + + @patch('requests.Session', return_value=MagicMock(get=MagicMock(return_value=None))) + @patch('time.time', return_value=1000) + def test_run_uncompressed_successful(self, mock_time, mock_session): + file_bytes = b"1234567890" * 10 + mock_session.return_value.get.return_value = create_response(status_code=200, _content=file_bytes) + + settings = Mock(link_expiry_buffer_secs=0, download_timeout=0, use_proxy=False) + settings.is_lz4_compressed = False + result_link = Mock(bytesNum=100, expiryTime=1001) + + d = downloader.ResultSetDownloadHandler(settings, result_link, ssl_options=SSLOptions()) + file = d.run() + + assert file.file_bytes == b"1234567890" * 10 + + @patch('requests.Session', return_value=MagicMock(get=MagicMock(return_value=MagicMock(ok=True)))) + @patch('time.time', return_value=1000) + def test_run_compressed_successful(self, mock_time, mock_session): + file_bytes = b"1234567890" * 10 + compressed_bytes = b'\x04"M\x18h@d\x00\x00\x00\x00\x00\x00\x00#\x14\x00\x00\x00\xaf1234567890\n\x00BP67890\x00\x00\x00\x00' + mock_session.return_value.get.return_value = create_response(status_code=200, _content=compressed_bytes) + + settings = Mock(link_expiry_buffer_secs=0, download_timeout=0, use_proxy=False) + settings.is_lz4_compressed = True + result_link = Mock(bytesNum=100, expiryTime=1001) + + d = downloader.ResultSetDownloadHandler(settings, result_link, ssl_options=SSLOptions()) + file = d.run() + + assert file.file_bytes == b"1234567890" * 10 + + @patch('requests.Session.get', side_effect=ConnectionError('foo')) + @patch('time.time', return_value=1000) + def test_download_connection_error(self, mock_time, mock_session): + settings = Mock(link_expiry_buffer_secs=0, use_proxy=False, is_lz4_compressed=True) + result_link = Mock(bytesNum=100, expiryTime=1001) + mock_session.return_value.get.return_value.content = \ + b'\x04"M\x18h@d\x00\x00\x00\x00\x00\x00\x00#\x14\x00\x00\x00\xaf1234567890\n\x00BP67890\x00\x00\x00\x00' + + d = downloader.ResultSetDownloadHandler(settings, result_link, ssl_options=SSLOptions()) + with self.assertRaises(ConnectionError): + d.run() + + @patch('requests.Session.get', side_effect=TimeoutError('foo')) + @patch('time.time', return_value=1000) + def test_download_timeout(self, mock_time, mock_session): + settings = Mock(link_expiry_buffer_secs=0, use_proxy=False, is_lz4_compressed=True) + result_link = Mock(bytesNum=100, expiryTime=1001) + mock_session.return_value.get.return_value.content = \ + b'\x04"M\x18h@d\x00\x00\x00\x00\x00\x00\x00#\x14\x00\x00\x00\xaf1234567890\n\x00BP67890\x00\x00\x00\x00' + + d = downloader.ResultSetDownloadHandler(settings, result_link, ssl_options=SSLOptions()) + with self.assertRaises(TimeoutError): + d.run() diff --git a/tests/unit/test_endpoint.py b/tests/unit/test_endpoint.py new file mode 100644 index 00000000..1f7d7cdd --- /dev/null +++ b/tests/unit/test_endpoint.py @@ -0,0 +1,124 @@ +import unittest +import os +import pytest + +from unittest.mock import patch + +from databricks.sql.auth.auth import AuthType +from databricks.sql.auth.endpoint import ( + infer_cloud_from_host, + CloudType, + get_oauth_endpoints, + AzureOAuthEndpointCollection, +) + +aws_host = "foo-bar.cloud.databricks.com" +azure_host = "foo-bar.1.azuredatabricks.net" +azure_cn_host = "foo-bar2.databricks.azure.cn" +gcp_host = "foo.1.gcp.databricks.com" + + +class EndpointTest(unittest.TestCase): + def test_infer_cloud_from_host(self): + param_list = [ + (CloudType.AWS, aws_host), + (CloudType.AZURE, azure_host), + (None, "foo.example.com"), + ] + + for expected_type, host in param_list: + with self.subTest(expected_type or "None", expected_type=expected_type): + self.assertEqual(infer_cloud_from_host(host), expected_type) + self.assertEqual( + infer_cloud_from_host(f"https://{host}/to/path"), expected_type + ) + + def test_oauth_endpoint(self): + scopes = ["offline_access", "sql", "admin"] + scopes2 = ["sql", "admin"] + azure_scope = ( + f"{AzureOAuthEndpointCollection.DATATRICKS_AZURE_APP}/user_impersonation" + ) + + param_list = [ + ( + CloudType.AWS, + aws_host, + False, + f"https://{aws_host}/oidc/oauth2/v2.0/authorize", + f"https://{aws_host}/oidc/.well-known/oauth-authorization-server", + scopes, + scopes2, + ), + ( + CloudType.AZURE, + azure_cn_host, + False, + f"https://{azure_cn_host}/oidc/oauth2/v2.0/authorize", + "https://login.microsoftonline.com/organizations/v2.0/.well-known/openid-configuration", + [azure_scope, "offline_access"], + [azure_scope], + ), + ( + CloudType.AZURE, + azure_host, + True, + f"https://{azure_host}/oidc/oauth2/v2.0/authorize", + "https://login.microsoftonline.com/organizations/v2.0/.well-known/openid-configuration", + [azure_scope, "offline_access"], + [azure_scope], + ), + ( + CloudType.AZURE, + azure_host, + False, + f"https://{azure_host}/oidc/oauth2/v2.0/authorize", + f"https://{azure_host}/oidc/.well-known/oauth-authorization-server", + scopes, + scopes2, + ), + ( + CloudType.GCP, + gcp_host, + False, + f"https://{gcp_host}/oidc/oauth2/v2.0/authorize", + f"https://{gcp_host}/oidc/.well-known/oauth-authorization-server", + scopes, + scopes2, + ), + ] + + for ( + cloud_type, + host, + use_azure_auth, + expected_auth_url, + expected_config_url, + expected_scopes, + expected_scope2, + ) in param_list: + with self.subTest(cloud_type): + endpoint = get_oauth_endpoints(host, use_azure_auth) + self.assertEqual( + endpoint.get_authorization_url(host), expected_auth_url + ) + self.assertEqual( + endpoint.get_openid_config_url(host), expected_config_url + ) + self.assertEqual(endpoint.get_scopes_mapping(scopes), expected_scopes) + self.assertEqual(endpoint.get_scopes_mapping(scopes2), expected_scope2) + + @patch.dict( + os.environ, + {"DATABRICKS_AZURE_TENANT_ID": "052ee82f-b79d-443c-8682-3ec1749e56b0"}, + ) + def test_azure_oauth_scope_mappings_from_different_tenant_id(self): + scopes = ["offline_access", "sql", "all"] + endpoint = get_oauth_endpoints(azure_host, True) + self.assertEqual( + endpoint.get_scopes_mapping(scopes), + [ + "052ee82f-b79d-443c-8682-3ec1749e56b0/user_impersonation", + "offline_access", + ], + ) diff --git a/tests/unit/test_init_file.py b/tests/unit/test_init_file.py new file mode 100644 index 00000000..75b15ac1 --- /dev/null +++ b/tests/unit/test_init_file.py @@ -0,0 +1,19 @@ +import hashlib + + +class TestInitFile: + """ + Micro test to confirm the contents of `databricks/__init__.py` does not change. + + Also see https://github.com/databricks/databricks-sdk-py/issues/343#issuecomment-1866029118. + """ + + def test_init_file_contents(self): + with open("src/databricks/__init__.py") as f: + init_file_contents = f.read() + + # This hash is the expected hash of the contents of `src/databricks/__init__.py`. + # It must not change, or else parallel package installation may lead to clobbered and invalid files. + expected_sha1 = "2772edbf52e517542acf8c039479c4b57b6ca2cd" + actual_sha1 = hashlib.sha1(init_file_contents.encode("utf-8")).hexdigest() + assert expected_sha1 == actual_sha1 diff --git a/tests/unit/test_oauth_persistence.py b/tests/unit/test_oauth_persistence.py index 10677c16..28b3cab3 100644 --- a/tests/unit/test_oauth_persistence.py +++ b/tests/unit/test_oauth_persistence.py @@ -1,8 +1,6 @@ import unittest -from databricks.sql.auth.auth import AccessTokenAuthProvider, BasicAuthProvider, AuthProvider -from databricks.sql.auth.auth import get_python_sql_connector_auth_provider from databricks.sql.experimental.oauth_persistence import DevOnlyFilePersistence, OAuthToken import tempfile import os diff --git a/tests/unit/test_param_escaper.py b/tests/unit/test_param_escaper.py index 6c1f1770..472a0843 100644 --- a/tests/unit/test_param_escaper.py +++ b/tests/unit/test_param_escaper.py @@ -1,104 +1,130 @@ from datetime import date, datetime import unittest, pytest, decimal +from typing import Any, Dict +from databricks.sql.parameters.native import dbsql_parameter_from_primitive -from databricks.sql.utils import ParamEscaper, inject_parameters +from databricks.sql.utils import ParamEscaper, inject_parameters, transform_paramstyle, ParameterStructure pe = ParamEscaper() -class TestIndividualFormatters(object): +class TestIndividualFormatters(object): # Test individual type escapers def test_escape_number_integer(self): - """This behaviour falls back to Python's default string formatting of numbers - """ + """This behaviour falls back to Python's default string formatting of numbers""" assert pe.escape_number(100) == 100 def test_escape_number_float(self): - """This behaviour falls back to Python's default string formatting of numbers - """ + """This behaviour falls back to Python's default string formatting of numbers""" assert pe.escape_number(100.1234) == 100.1234 def test_escape_number_decimal(self): - """This behaviour uses the string representation of a decimal - """ + """This behaviour uses the string representation of a decimal""" assert pe.escape_decimal(decimal.Decimal("124.32")) == "124.32" def test_escape_string_normal(self): - """ - """ + """ """ assert pe.escape_string("golly bob howdy") == "'golly bob howdy'" def test_escape_string_that_includes_special_characters(self): - """Tests for how special characters are treated. - - When passed a string, the `escape_string` method wraps it in single quotes - and escapes any special characters with a back stroke (\) - - Example: - - IN : his name was 'robert palmer' - OUT: 'his name was \'robert palmer\'' - """ - - # Testing for the presence of these characters: '"/\😂 - - assert pe.escape_string("his name was 'robert palmer'") == r"'his name was \'robert palmer\''" + r"""Tests for how special characters are treated. - # These tests represent the same user input in the several ways it can be written in Python - # Each argument to `escape_string` evaluates to the same bytes. But Python lets us write it differently. - assert pe.escape_string("his name was \"robert palmer\"") == "'his name was \"robert palmer\"'" - assert pe.escape_string('his name was "robert palmer"') == "'his name was \"robert palmer\"'" - assert pe.escape_string('his name was {}'.format('"robert palmer"')) == "'his name was \"robert palmer\"'" + When passed a string, the `escape_string` method wraps it in single quotes + and escapes any special characters with a back stroke (\) - assert pe.escape_string("his name was robert / palmer") == r"'his name was robert / palmer'" + Example: - # If you need to include a single backslash, use an r-string to prevent Python from raising a - # DeprecationWarning for an invalid escape sequence - assert pe.escape_string("his name was robert \\/ palmer") == r"'his name was robert \\/ palmer'" - assert pe.escape_string("his name was robert \\ palmer") == r"'his name was robert \\ palmer'" - assert pe.escape_string("his name was robert \\\\ palmer") == r"'his name was robert \\\\ palmer'" - - assert pe.escape_string("his name was robert palmer 😂") == r"'his name was robert palmer 😂'" - - # Adding the test from PR #56 to prove escape behaviour - - assert pe.escape_string("you're") == r"'you\'re'" - - # Adding this test from #51 to prove escape behaviour when the target string involves repeated SQL escape chars - assert pe.escape_string("cat\\'s meow") == r"'cat\\\'s meow'" - - # Tests from the docs: https://docs.databricks.com/sql/language-manual/data-types/string-type.html + IN : his name was 'robert palmer' + OUT: 'his name was \'robert palmer\'' + """ - assert pe.escape_string('Spark') == "'Spark'" - assert pe.escape_string("O'Connell") == r"'O\'Connell'" - assert pe.escape_string("Some\\nText") == r"'Some\\nText'" - assert pe.escape_string("Some\\\\nText") == r"'Some\\\\nText'" - assert pe.escape_string("서울시") == "'서울시'" - assert pe.escape_string("\\\\") == r"'\\\\'" + # Testing for the presence of these characters: '"/\😂 + + assert ( + pe.escape_string("his name was 'robert palmer'") + == r"'his name was \'robert palmer\''" + ) + + # These tests represent the same user input in the several ways it can be written in Python + # Each argument to `escape_string` evaluates to the same bytes. But Python lets us write it differently. + assert ( + pe.escape_string('his name was "robert palmer"') + == "'his name was \"robert palmer\"'" + ) + assert ( + pe.escape_string('his name was "robert palmer"') + == "'his name was \"robert palmer\"'" + ) + assert ( + pe.escape_string("his name was {}".format('"robert palmer"')) + == "'his name was \"robert palmer\"'" + ) + + assert ( + pe.escape_string("his name was robert / palmer") + == r"'his name was robert / palmer'" + ) + + # If you need to include a single backslash, use an r-string to prevent Python from raising a + # DeprecationWarning for an invalid escape sequence + assert ( + pe.escape_string("his name was robert \\/ palmer") + == r"'his name was robert \\/ palmer'" + ) + assert ( + pe.escape_string("his name was robert \\ palmer") + == r"'his name was robert \\ palmer'" + ) + assert ( + pe.escape_string("his name was robert \\\\ palmer") + == r"'his name was robert \\\\ palmer'" + ) + + assert ( + pe.escape_string("his name was robert palmer 😂") + == r"'his name was robert palmer 😂'" + ) + + # Adding the test from PR #56 to prove escape behaviour + + assert pe.escape_string("you're") == r"'you\'re'" + + # Adding this test from #51 to prove escape behaviour when the target string involves repeated SQL escape chars + assert pe.escape_string("cat\\'s meow") == r"'cat\\\'s meow'" + + # Tests from the docs: https://docs.databricks.com/sql/language-manual/data-types/string-type.html + + assert pe.escape_string("Spark") == "'Spark'" + assert pe.escape_string("O'Connell") == r"'O\'Connell'" + assert pe.escape_string("Some\\nText") == r"'Some\\nText'" + assert pe.escape_string("Some\\\\nText") == r"'Some\\\\nText'" + assert pe.escape_string("서울시") == "'서울시'" + assert pe.escape_string("\\\\") == r"'\\\\'" def test_escape_date_time(self): - INPUT = datetime(1991,8,3,21,55) + INPUT = datetime(1991, 8, 3, 21, 55) FORMAT = "%Y-%m-%d %H:%M:%S" OUTPUT = "'1991-08-03 21:55:00'" assert pe.escape_datetime(INPUT, FORMAT) == OUTPUT def test_escape_date(self): - INPUT = date(1991,8,3) + INPUT = date(1991, 8, 3) FORMAT = "%Y-%m-%d" OUTPUT = "'1991-08-03'" assert pe.escape_datetime(INPUT, FORMAT) == OUTPUT def test_escape_sequence_integer(self): - assert pe.escape_sequence([1,2,3,4]) == "(1,2,3,4)" + assert pe.escape_sequence([1, 2, 3, 4]) == "(1,2,3,4)" def test_escape_sequence_float(self): - assert pe.escape_sequence([1.1,2.2,3.3,4.4]) == "(1.1,2.2,3.3,4.4)" + assert pe.escape_sequence([1.1, 2.2, 3.3, 4.4]) == "(1.1,2.2,3.3,4.4)" def test_escape_sequence_string(self): - assert pe.escape_sequence( - ["his", "name", "was", "robert", "palmer"]) == \ - "('his','name','was','robert','palmer')" + assert ( + pe.escape_sequence(["his", "name", "was", "robert", "palmer"]) + == "('his','name','was','robert','palmer')" + ) def test_escape_sequence_sequence_of_strings(self): # This is not valid SQL. @@ -109,9 +135,7 @@ def test_escape_sequence_sequence_of_strings(self): class TestFullQueryEscaping(object): - def test_simple(self): - INPUT = """ SELECT field1, @@ -140,7 +164,6 @@ def test_simple(self): @unittest.skipUnless(False, "Thrift server supports native parameter binding.") def test_only_bind_in_where_clause(self): - INPUT = """ SELECT %(field)s, @@ -153,3 +176,50 @@ def test_only_bind_in_where_clause(self): with pytest.raises(Exception): inject_parameters(INPUT, pe.escape_args(args)) + + +class TestInlineToNativeTransformer(object): + @pytest.mark.parametrize( + ("label", "query", "params", "expected"), + ( + ("no effect", "SELECT 1", {}, "SELECT 1"), + ("one marker", "%(param)s", {"param": ""}, ":param"), + ( + "multiple markers", + "%(foo)s %(bar)s %(baz)s", + {"foo": None, "bar": None, "baz": None}, + ":foo :bar :baz", + ), + ( + "sql query", + "SELECT * FROM table WHERE field = %(param)s AND other_field IN (%(list)s)", + {"param": None, "list": None}, + "SELECT * FROM table WHERE field = :param AND other_field IN (:list)", + ), + ( + "query with like wildcard", + 'select * from table where field like "%"', + {}, + 'select * from table where field like "%"' + ), + ( + "query with named param and like wildcard", + 'select :param from table where field like "%"', + {"param": None}, + 'select :param from table where field like "%"' + ), + ( + "query with doubled wildcards", + 'select 1 where '' like "%%"', + {"param": None}, + 'select 1 where '' like "%%"', + ) + ), + ) + def test_transformer( + self, label: str, query: str, params: Dict[str, Any], expected: str + ): + + _params = [dbsql_parameter_from_primitive(value=value, name=name) for name, value in params.items()] + output = transform_paramstyle(query, _params, param_structure=ParameterStructure.NAMED) + assert output == expected diff --git a/tests/unit/test_parameters.py b/tests/unit/test_parameters.py new file mode 100644 index 00000000..eec921e4 --- /dev/null +++ b/tests/unit/test_parameters.py @@ -0,0 +1,204 @@ +import datetime +from decimal import Decimal +from enum import Enum +from typing import Type + +import pytest +import pytz + +from databricks.sql.client import Connection +from databricks.sql.parameters import ( + BigIntegerParameter, + BooleanParameter, + DateParameter, + DecimalParameter, + DoubleParameter, + FloatParameter, + IntegerParameter, + SmallIntParameter, + StringParameter, + TimestampNTZParameter, + TimestampParameter, + TinyIntParameter, + VoidParameter, +) +from databricks.sql.parameters.native import ( + TDbsqlParameter, + TSparkParameterValue, + dbsql_parameter_from_primitive, +) +from databricks.sql.thrift_api.TCLIService import ttypes +from databricks.sql.thrift_api.TCLIService.ttypes import ( + TOpenSessionResp, + TSessionHandle, + TSparkParameterValue, +) + + +class TestSessionHandleChecks(object): + @pytest.mark.parametrize( + "test_input,expected", + [ + ( + TOpenSessionResp( + serverProtocolVersion=ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V7, + sessionHandle=TSessionHandle(1, None), + ), + ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V7, + ), + # Ensure that protocol version inside sessionhandle takes precedence. + ( + TOpenSessionResp( + serverProtocolVersion=ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V7, + sessionHandle=TSessionHandle( + 1, ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V8 + ), + ), + ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V8, + ), + ], + ) + def test_get_protocol_version_fallback_behavior(self, test_input, expected): + assert Connection.get_protocol_version(test_input) == expected + + @pytest.mark.parametrize( + "test_input,expected", + [ + ( + None, + False, + ), + ( + ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V7, + False, + ), + ( + ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V8, + True, + ), + ], + ) + def test_parameters_enabled(self, test_input, expected): + assert Connection.server_parameterized_queries_enabled(test_input) == expected + + +@pytest.mark.parametrize( + "value,expected", + ( + (Decimal("10.00"), "DECIMAL(4,2)"), + (Decimal("123456789123456789.123456789123456789"), "DECIMAL(36,18)"), + (Decimal(".12345678912345678912345678912345678912"), "DECIMAL(38,38)"), + (Decimal("123456789.123456789"), "DECIMAL(18,9)"), + (Decimal("12345678912345678912345678912345678912"), "DECIMAL(38,0)"), + (Decimal("1234.56"), "DECIMAL(6,2)"), + ), +) +def test_calculate_decimal_cast_string(value, expected): + p = DecimalParameter(value) + assert p._cast_expr() == expected + + +class Primitive(Enum): + """These are the inferrable types. This Enum is used for parametrized tests.""" + + NONE = None + BOOL = True + INT = 50 + BIGINT = 2147483648 + STRING = "Hello" + DECIMAL = Decimal("1234.56") + DATE = datetime.date(2023, 9, 6) + TIMESTAMP = datetime.datetime(2023, 9, 6, 3, 14, 27, 843, tzinfo=pytz.UTC) + DOUBLE = 3.14 + FLOAT = 3.15 + SMALLINT = 51 + + +class TestDbsqlParameter: + @pytest.mark.parametrize( + "_type, prim, expect_cast_expr", + ( + (DecimalParameter, Primitive.DECIMAL, "DECIMAL(6,2)"), + (IntegerParameter, Primitive.INT, "INT"), + (StringParameter, Primitive.STRING, "STRING"), + (BigIntegerParameter, Primitive.BIGINT, "BIGINT"), + (BooleanParameter, Primitive.BOOL, "BOOLEAN"), + (DateParameter, Primitive.DATE, "DATE"), + (DoubleParameter, Primitive.DOUBLE, "DOUBLE"), + (FloatParameter, Primitive.FLOAT, "FLOAT"), + (VoidParameter, Primitive.NONE, "VOID"), + (SmallIntParameter, Primitive.INT, "SMALLINT"), + (TimestampParameter, Primitive.TIMESTAMP, "TIMESTAMP"), + (TimestampNTZParameter, Primitive.TIMESTAMP, "TIMESTAMP_NTZ"), + (TinyIntParameter, Primitive.INT, "TINYINT"), + ), + ) + def test_cast_expression( + self, _type: TDbsqlParameter, prim: Primitive, expect_cast_expr: str + ): + p = _type(prim.value) + assert p._cast_expr() == expect_cast_expr + + @pytest.mark.parametrize( + "t, prim", + ( + (DecimalParameter, Primitive.DECIMAL), + (IntegerParameter, Primitive.INT), + (StringParameter, Primitive.STRING), + (BigIntegerParameter, Primitive.BIGINT), + (BooleanParameter, Primitive.BOOL), + (DateParameter, Primitive.DATE), + (DoubleParameter, Primitive.DOUBLE), + (FloatParameter, Primitive.FLOAT), + (VoidParameter, Primitive.NONE), + (SmallIntParameter, Primitive.INT), + (TimestampParameter, Primitive.TIMESTAMP), + (TimestampNTZParameter, Primitive.TIMESTAMP), + (TinyIntParameter, Primitive.INT), + ), + ) + def test_tspark_param_value(self, t: TDbsqlParameter, prim): + p: TDbsqlParameter = t(prim.value) + output = p._tspark_param_value() + + if prim == Primitive.NONE: + assert output == None + else: + assert output == TSparkParameterValue(stringValue=str(prim.value)) + + def test_tspark_param_named(self): + p = dbsql_parameter_from_primitive(Primitive.INT.value, name="p") + tsp = p.as_tspark_param(named=True) + + assert tsp.name == "p" + assert tsp.ordinal is False + + def test_tspark_param_ordinal(self): + p = dbsql_parameter_from_primitive(Primitive.INT.value, name="p") + tsp = p.as_tspark_param(named=False) + + assert tsp.name is None + assert tsp.ordinal is True + + @pytest.mark.parametrize( + "_type, prim", + ( + (DecimalParameter, Primitive.DECIMAL), + (IntegerParameter, Primitive.INT), + (StringParameter, Primitive.STRING), + (BigIntegerParameter, Primitive.BIGINT), + (BooleanParameter, Primitive.BOOL), + (DateParameter, Primitive.DATE), + (FloatParameter, Primitive.FLOAT), + (VoidParameter, Primitive.NONE), + (TimestampParameter, Primitive.TIMESTAMP), + ), + ) + def test_inference(self, _type: TDbsqlParameter, prim: Primitive): + """This method only tests inferrable types. + + Not tested are TinyIntParameter, SmallIntParameter DoubleParameter and TimestampNTZParameter + """ + + inferred_type = dbsql_parameter_from_primitive(prim.value) + assert isinstance(inferred_type, _type) diff --git a/tests/unit/test_retry.py b/tests/unit/test_retry.py new file mode 100644 index 00000000..798bac2e --- /dev/null +++ b/tests/unit/test_retry.py @@ -0,0 +1,55 @@ +from os import error +import time +from unittest.mock import Mock, patch +import pytest +from requests import Request +from urllib3 import HTTPResponse +from databricks.sql.auth.retry import DatabricksRetryPolicy, RequestHistory + + +class TestRetry: + + @pytest.fixture() + def retry_policy(self) -> DatabricksRetryPolicy: + return DatabricksRetryPolicy( + delay_min=1, + delay_max=30, + stop_after_attempts_count=3, + stop_after_attempts_duration=900, + delay_default=2, + force_dangerous_codes=[], + ) + + @pytest.fixture() + def error_history(self) -> RequestHistory: + return RequestHistory( + method="POST", url=None, error=None, status=503, redirect_location=None + ) + + @patch("time.sleep") + def test_sleep__no_retry_after(self, t_mock, retry_policy, error_history): + retry_policy._retry_start_time = time.time() + retry_policy.history = [error_history, error_history] + retry_policy.sleep(HTTPResponse(status=503)) + t_mock.assert_called_with(2) + + @patch("time.sleep") + def test_sleep__retry_after_is_binding(self, t_mock, retry_policy, error_history): + retry_policy._retry_start_time = time.time() + retry_policy.history = [error_history, error_history] + retry_policy.sleep(HTTPResponse(status=503, headers={"Retry-After": "3"})) + t_mock.assert_called_with(3) + + @patch("time.sleep") + def test_sleep__retry_after_present_but_not_binding(self, t_mock, retry_policy, error_history): + retry_policy._retry_start_time = time.time() + retry_policy.history = [error_history, error_history] + retry_policy.sleep(HTTPResponse(status=503, headers={"Retry-After": "1"})) + t_mock.assert_called_with(2) + + @patch("time.sleep") + def test_sleep__retry_after_surpassed(self, t_mock, retry_policy, error_history): + retry_policy._retry_start_time = time.time() + retry_policy.history = [error_history, error_history, error_history] + retry_policy.sleep(HTTPResponse(status=503, headers={"Retry-After": "3"})) + t_mock.assert_called_with(4) diff --git a/tests/unit/test_thrift_backend.py b/tests/unit/test_thrift_backend.py index 1c2e589b..0333766c 100644 --- a/tests/unit/test_thrift_backend.py +++ b/tests/unit/test_thrift_backend.py @@ -4,10 +4,13 @@ import unittest from unittest.mock import patch, MagicMock, Mock from ssl import CERT_NONE, CERT_REQUIRED +from urllib3 import HTTPSConnectionPool import pyarrow import databricks.sql +from databricks.sql import utils +from databricks.sql.types import SSLOptions from databricks.sql.thrift_api.TCLIService import ttypes from databricks.sql import * from databricks.sql.auth.authenticators import AuthProvider @@ -15,12 +18,12 @@ def retry_policy_factory(): - return { # (type, default, min, max) - "_retry_delay_min": (float, 1, None, None), - "_retry_delay_max": (float, 60, None, None), - "_retry_stop_after_attempts_count": (int, 30, None, None), - "_retry_stop_after_attempts_duration": (float, 900, None, None), - "_retry_delay_default": (float, 5, 1, 60) + return { # (type, default, min, max) + "_retry_delay_min": (float, 1, None, None), + "_retry_delay_max": (float, 60, None, None), + "_retry_stop_after_attempts_count": (int, 30, None, None), + "_retry_stop_after_attempts_duration": (float, 900, None, None), + "_retry_delay_default": (float, 5, 1, 60), } @@ -34,14 +37,17 @@ class ThriftBackendTestSuite(unittest.TestCase): operation_handle = ttypes.TOperationHandle( operationId=ttypes.THandleIdentifier(guid=0x33, secret=0x35), - operationType=ttypes.TOperationType.EXECUTE_STATEMENT) + operationType=ttypes.TOperationType.EXECUTE_STATEMENT, + ) session_handle = ttypes.TSessionHandle( - sessionId=ttypes.THandleIdentifier(guid=0x36, secret=0x37)) + sessionId=ttypes.THandleIdentifier(guid=0x36, secret=0x37) + ) open_session_resp = ttypes.TOpenSessionResp( status=okay_status, - serverProtocolVersion=ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V4) + serverProtocolVersion=ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V4, + ) metadata_resp = ttypes.TGetResultSetMetadataResp( status=okay_status, @@ -50,8 +56,11 @@ class ThriftBackendTestSuite(unittest.TestCase): ) execute_response_types = [ - ttypes.TExecuteStatementResp, ttypes.TGetCatalogsResp, ttypes.TGetSchemasResp, - ttypes.TGetTablesResp, ttypes.TGetColumnsResp + ttypes.TExecuteStatementResp, + ttypes.TGetCatalogsResp, + ttypes.TGetSchemasResp, + ttypes.TGetTablesResp, + ttypes.TGetColumnsResp, ] def test_make_request_checks_thrift_status_code(self): @@ -60,15 +69,17 @@ def test_make_request_checks_thrift_status_code(self): mock_method = Mock() mock_method.__name__ = "method name" mock_method.return_value = mock_response - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) with self.assertRaises(DatabaseError): thrift_backend.make_request(mock_method, Mock()) def _make_type_desc(self, type): - return ttypes.TTypeDesc(types=[ttypes.TTypeEntry(ttypes.TPrimitiveTypeEntry(type=type))]) + return ttypes.TTypeDesc( + types=[ttypes.TTypeEntry(ttypes.TTAllowedParameterValueEntry(type=type))] + ) def _make_fake_thrift_backend(self): - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) thrift_backend._hive_schema_to_arrow_schema = Mock() thrift_backend._hive_schema_to_description = Mock() thrift_backend._create_arrow_table = MagicMock() @@ -78,13 +89,17 @@ def _make_fake_thrift_backend(self): def test_hive_schema_to_arrow_schema_preserves_column_names(self): columns = [ ttypes.TColumnDesc( - columnName="column 1", typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE)), + columnName="column 1", typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE) + ), ttypes.TColumnDesc( - columnName="column 2", typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE)), + columnName="column 2", typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE) + ), ttypes.TColumnDesc( - columnName="column 2", typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE)), + columnName="column 2", typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE) + ), ttypes.TColumnDesc( - columnName="", typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE)) + columnName="", typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE) + ), ] t_table_schema = ttypes.TTableSchema(columns) @@ -95,7 +110,7 @@ def test_hive_schema_to_arrow_schema_preserves_column_names(self): self.assertEqual(arrow_schema.field(2).name, "column 2") self.assertEqual(arrow_schema.field(3).name, "") - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_bad_protocol_versions_are_rejected(self, tcli_service_client_cass): t_http_client_instance = tcli_service_client_cass.return_value bad_protocol_versions = [ @@ -114,7 +129,8 @@ def test_bad_protocol_versions_are_rejected(self, tcli_service_client_cass): for protocol_version in bad_protocol_versions: t_http_client_instance.OpenSession.return_value = ttypes.TOpenSessionResp( - status=self.okay_status, serverProtocolVersion=protocol_version) + status=self.okay_status, serverProtocolVersion=protocol_version + ) with self.assertRaises(OperationalError) as cm: thrift_backend = self._make_fake_thrift_backend() @@ -122,25 +138,26 @@ def test_bad_protocol_versions_are_rejected(self, tcli_service_client_cass): self.assertIn("expected server to use a protocol version", str(cm.exception)) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_okay_protocol_versions_succeed(self, tcli_service_client_cass): t_http_client_instance = tcli_service_client_cass.return_value good_protocol_versions = [ ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V2, ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V3, - ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V4 + ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V4, ] for protocol_version in good_protocol_versions: t_http_client_instance.OpenSession.return_value = ttypes.TOpenSessionResp( - status=self.okay_status, serverProtocolVersion=protocol_version) + status=self.okay_status, serverProtocolVersion=protocol_version + ) thrift_backend = self._make_fake_thrift_backend() thrift_backend.open_session({}, None, None) @patch("databricks.sql.auth.thrift_http_client.THttpClient") def test_headers_are_set(self, t_http_client_class): - ThriftBackend("foo", 123, "bar", [("header", "value")], auth_provider=AuthProvider()) + ThriftBackend("foo", 123, "bar", [("header", "value")], auth_provider=AuthProvider(), ssl_options=SSLOptions()) t_http_client_class.return_value.setCustomHeaders.assert_called_with({"header": "value"}) def test_proxy_headers_are_set(self): @@ -150,85 +167,184 @@ def test_proxy_headers_are_set(self): fake_proxy_spec = "https://someuser:somepassword@8.8.8.8:12340" parsed_proxy = urlparse(fake_proxy_spec) - + try: - result = THttpClient.basic_proxy_auth_header(parsed_proxy) + result = THttpClient.basic_proxy_auth_headers(parsed_proxy) except TypeError as e: assert False - assert isinstance(result, type(str())) + assert isinstance(result, type(dict())) + assert isinstance(result.get('proxy-authorization'), type(str())) @patch("databricks.sql.auth.thrift_http_client.THttpClient") - @patch("databricks.sql.thrift_backend.create_default_context") + @patch("databricks.sql.types.create_default_context") def test_tls_cert_args_are_propagated(self, mock_create_default_context, t_http_client_class): mock_cert_key_file = Mock() mock_cert_key_password = Mock() mock_trusted_ca_file = Mock() mock_cert_file = Mock() + mock_ssl_options = SSLOptions( + tls_client_cert_file=mock_cert_file, + tls_client_cert_key_file=mock_cert_key_file, + tls_client_cert_key_password=mock_cert_key_password, + tls_trusted_ca_file=mock_trusted_ca_file, + ) + mock_ssl_context = mock_ssl_options.create_ssl_context() + mock_create_default_context.assert_called_once_with(cafile=mock_trusted_ca_file) + ThriftBackend( "foo", 123, - "bar", [], + "bar", + [], auth_provider=AuthProvider(), - _tls_client_cert_file=mock_cert_file, - _tls_client_cert_key_file=mock_cert_key_file, - _tls_client_cert_key_password=mock_cert_key_password, - _tls_trusted_ca_file=mock_trusted_ca_file) + ssl_options=mock_ssl_options, + ) - mock_create_default_context.assert_called_once_with(cafile=mock_trusted_ca_file) - mock_ssl_context = mock_create_default_context.return_value mock_ssl_context.load_cert_chain.assert_called_once_with( - certfile=mock_cert_file, keyfile=mock_cert_key_file, password=mock_cert_key_password) + certfile=mock_cert_file, keyfile=mock_cert_key_file, password=mock_cert_key_password + ) self.assertTrue(mock_ssl_context.check_hostname) self.assertEqual(mock_ssl_context.verify_mode, CERT_REQUIRED) - self.assertEqual(t_http_client_class.call_args[1]["ssl_context"], mock_ssl_context) + self.assertEqual(t_http_client_class.call_args[1]["ssl_options"], mock_ssl_options) + + @patch("databricks.sql.types.create_default_context") + def test_tls_cert_args_are_used_by_http_client(self, mock_create_default_context): + from databricks.sql.auth.thrift_http_client import THttpClient + + mock_cert_key_file = Mock() + mock_cert_key_password = Mock() + mock_trusted_ca_file = Mock() + mock_cert_file = Mock() + + mock_ssl_options = SSLOptions( + tls_verify=True, + tls_client_cert_file=mock_cert_file, + tls_client_cert_key_file=mock_cert_key_file, + tls_client_cert_key_password=mock_cert_key_password, + tls_trusted_ca_file=mock_trusted_ca_file, + ) + + http_client = THttpClient( + auth_provider=None, + uri_or_host="https://example.com", + ssl_options=mock_ssl_options, + ) + + self.assertEqual(http_client.scheme, 'https') + self.assertEqual(http_client.certfile, mock_ssl_options.tls_client_cert_file) + self.assertEqual(http_client.keyfile, mock_ssl_options.tls_client_cert_key_file) + self.assertIsNotNone(http_client.certfile) + mock_create_default_context.assert_called() + + http_client.open() + + conn_pool = http_client._THttpClient__pool + self.assertIsInstance(conn_pool, HTTPSConnectionPool) + self.assertEqual(conn_pool.cert_reqs, CERT_REQUIRED) + self.assertEqual(conn_pool.ca_certs, mock_ssl_options.tls_trusted_ca_file) + self.assertEqual(conn_pool.cert_file, mock_ssl_options.tls_client_cert_file) + self.assertEqual(conn_pool.key_file, mock_ssl_options.tls_client_cert_key_file) + self.assertEqual(conn_pool.key_password, mock_ssl_options.tls_client_cert_key_password) + + def test_tls_no_verify_is_respected_by_http_client(self): + from databricks.sql.auth.thrift_http_client import THttpClient + + http_client = THttpClient( + auth_provider=None, + uri_or_host="https://example.com", + ssl_options=SSLOptions(tls_verify=False), + ) + self.assertEqual(http_client.scheme, 'https') + + http_client.open() + + conn_pool = http_client._THttpClient__pool + self.assertIsInstance(conn_pool, HTTPSConnectionPool) + self.assertEqual(conn_pool.cert_reqs, CERT_NONE) @patch("databricks.sql.auth.thrift_http_client.THttpClient") - @patch("databricks.sql.thrift_backend.create_default_context") + @patch("databricks.sql.types.create_default_context") def test_tls_no_verify_is_respected(self, mock_create_default_context, t_http_client_class): - ThriftBackend("foo", 123, "bar", [], auth_provider=AuthProvider(), _tls_no_verify=True) + mock_ssl_options = SSLOptions(tls_verify=False) + mock_ssl_context = mock_ssl_options.create_ssl_context() + mock_create_default_context.assert_called() + + ThriftBackend("foo", 123, "bar", [], auth_provider=AuthProvider(), ssl_options=mock_ssl_options) - mock_ssl_context = mock_create_default_context.return_value self.assertFalse(mock_ssl_context.check_hostname) self.assertEqual(mock_ssl_context.verify_mode, CERT_NONE) - self.assertEqual(t_http_client_class.call_args[1]["ssl_context"], mock_ssl_context) + self.assertEqual(t_http_client_class.call_args[1]["ssl_options"], mock_ssl_options) @patch("databricks.sql.auth.thrift_http_client.THttpClient") - @patch("databricks.sql.thrift_backend.create_default_context") - def test_tls_verify_hostname_is_respected(self, mock_create_default_context, - t_http_client_class): - ThriftBackend("foo", 123, "bar", [], auth_provider=AuthProvider(), _tls_verify_hostname=False) + @patch("databricks.sql.types.create_default_context") + def test_tls_verify_hostname_is_respected( + self, mock_create_default_context, t_http_client_class + ): + mock_ssl_options = SSLOptions(tls_verify_hostname=False) + mock_ssl_context = mock_ssl_options.create_ssl_context() + mock_create_default_context.assert_called() + + ThriftBackend( + "foo", 123, "bar", [], auth_provider=AuthProvider(), ssl_options=mock_ssl_options + ) - mock_ssl_context = mock_create_default_context.return_value self.assertFalse(mock_ssl_context.check_hostname) self.assertEqual(mock_ssl_context.verify_mode, CERT_REQUIRED) - self.assertEqual(t_http_client_class.call_args[1]["ssl_context"], mock_ssl_context) + self.assertEqual(t_http_client_class.call_args[1]["ssl_options"], mock_ssl_options) @patch("databricks.sql.auth.thrift_http_client.THttpClient") def test_port_and_host_are_respected(self, t_http_client_class): - ThriftBackend("hostname", 123, "path_value", [], auth_provider=AuthProvider()) - self.assertEqual(t_http_client_class.call_args[1]["uri_or_host"], - "https://hostname:123/path_value") + ThriftBackend("hostname", 123, "path_value", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) + self.assertEqual( + t_http_client_class.call_args[1]["uri_or_host"], "https://hostname:123/path_value" + ) + + @patch("databricks.sql.auth.thrift_http_client.THttpClient") + def test_host_with_https_does_not_duplicate(self, t_http_client_class): + ThriftBackend("https://hostname", 123, "path_value", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) + self.assertEqual( + t_http_client_class.call_args[1]["uri_or_host"], "https://hostname:123/path_value" + ) + + @patch("databricks.sql.auth.thrift_http_client.THttpClient") + def test_host_with_trailing_backslash_does_not_duplicate(self, t_http_client_class): + ThriftBackend("https://hostname/", 123, "path_value", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) + self.assertEqual( + t_http_client_class.call_args[1]["uri_or_host"], "https://hostname:123/path_value" + ) @patch("databricks.sql.auth.thrift_http_client.THttpClient") def test_socket_timeout_is_propagated(self, t_http_client_class): - ThriftBackend("hostname", 123, "path_value", [], auth_provider=AuthProvider(), _socket_timeout=129) + ThriftBackend( + "hostname", 123, "path_value", [], auth_provider=AuthProvider(), ssl_options=SSLOptions(), _socket_timeout=129 + ) self.assertEqual(t_http_client_class.return_value.setTimeout.call_args[0][0], 129 * 1000) - ThriftBackend("hostname", 123, "path_value", [], auth_provider=AuthProvider(), _socket_timeout=0) + ThriftBackend( + "hostname", 123, "path_value", [], auth_provider=AuthProvider(), ssl_options=SSLOptions(), _socket_timeout=0 + ) self.assertEqual(t_http_client_class.return_value.setTimeout.call_args[0][0], 0) - ThriftBackend("hostname", 123, "path_value", [], auth_provider=AuthProvider(), _socket_timeout=None) + ThriftBackend("hostname", 123, "path_value", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) + self.assertEqual(t_http_client_class.return_value.setTimeout.call_args[0][0], 900 * 1000) + ThriftBackend( + "hostname", 123, "path_value", [], auth_provider=AuthProvider(), ssl_options=SSLOptions(), _socket_timeout=None + ) self.assertEqual(t_http_client_class.return_value.setTimeout.call_args[0][0], None) def test_non_primitive_types_raise_error(self): columns = [ ttypes.TColumnDesc( - columnName="column 1", typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE)), + columnName="column 1", typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE) + ), ttypes.TColumnDesc( columnName="column 2", - typeDesc=ttypes.TTypeDesc(types=[ - ttypes.TTypeEntry(userDefinedTypeEntry=ttypes.TUserDefinedTypeEntry("foo")) - ])) + typeDesc=ttypes.TTypeDesc( + types=[ + ttypes.TTypeEntry(userDefinedTypeEntry=ttypes.TUserDefinedTypeEntry("foo")) + ] + ), + ), ] t_table_schema = ttypes.TTableSchema(columns) @@ -242,50 +358,66 @@ def test_hive_schema_to_description_preserves_column_names_and_types(self): # canary test columns = [ ttypes.TColumnDesc( - columnName="column 1", typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE)), + columnName="column 1", typeDesc=self._make_type_desc(ttypes.TTypeId.INT_TYPE) + ), ttypes.TColumnDesc( - columnName="column 2", typeDesc=self._make_type_desc(ttypes.TTypeId.BOOLEAN_TYPE)), + columnName="column 2", typeDesc=self._make_type_desc(ttypes.TTypeId.BOOLEAN_TYPE) + ), ttypes.TColumnDesc( - columnName="column 2", typeDesc=self._make_type_desc(ttypes.TTypeId.MAP_TYPE)), + columnName="column 2", typeDesc=self._make_type_desc(ttypes.TTypeId.MAP_TYPE) + ), ttypes.TColumnDesc( - columnName="", typeDesc=self._make_type_desc(ttypes.TTypeId.STRUCT_TYPE)) + columnName="", typeDesc=self._make_type_desc(ttypes.TTypeId.STRUCT_TYPE) + ), ] t_table_schema = ttypes.TTableSchema(columns) description = ThriftBackend._hive_schema_to_description(t_table_schema) - self.assertEqual(description, [ - ("column 1", "int", None, None, None, None, None), - ("column 2", "boolean", None, None, None, None, None), - ("column 2", "map", None, None, None, None, None), - ("", "struct", None, None, None, None, None), - ]) + self.assertEqual( + description, + [ + ("column 1", "int", None, None, None, None, None), + ("column 2", "boolean", None, None, None, None, None), + ("column 2", "map", None, None, None, None, None), + ("", "struct", None, None, None, None, None), + ], + ) def test_hive_schema_to_description_preserves_scale_and_precision(self): columns = [ ttypes.TColumnDesc( columnName="column 1", - typeDesc=ttypes.TTypeDesc(types=[ - ttypes.TTypeEntry( - ttypes.TPrimitiveTypeEntry( - type=ttypes.TTypeId.DECIMAL_TYPE, - typeQualifiers=ttypes.TTypeQualifiers( - qualifiers={ - "precision": ttypes.TTypeQualifierValue(i32Value=10), - "scale": ttypes.TTypeQualifierValue(i32Value=100), - }))) - ])), + typeDesc=ttypes.TTypeDesc( + types=[ + ttypes.TTypeEntry( + ttypes.TTAllowedParameterValueEntry( + type=ttypes.TTypeId.DECIMAL_TYPE, + typeQualifiers=ttypes.TTypeQualifiers( + qualifiers={ + "precision": ttypes.TTypeQualifierValue(i32Value=10), + "scale": ttypes.TTypeQualifierValue(i32Value=100), + } + ), + ) + ) + ] + ), + ), ] t_table_schema = ttypes.TTableSchema(columns) description = ThriftBackend._hive_schema_to_description(t_table_schema) - self.assertEqual(description, [ - ("column 1", "decimal", None, None, 10, 100, None), - ]) + self.assertEqual( + description, + [ + ("column 1", "decimal", None, None, 10, 100, None), + ], + ) def test_make_request_checks_status_code(self): error_codes = [ttypes.TStatusCode.ERROR_STATUS, ttypes.TStatusCode.INVALID_HANDLE_STATUS] - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) for code in error_codes: mock_error_response = Mock() @@ -296,8 +428,9 @@ def test_make_request_checks_status_code(self): self.assertIn("a detailed error message", str(cm.exception)) success_codes = [ - ttypes.TStatusCode.SUCCESS_STATUS, ttypes.TStatusCode.SUCCESS_WITH_INFO_STATUS, - ttypes.TStatusCode.STILL_EXECUTING_STATUS + ttypes.TStatusCode.SUCCESS_STATUS, + ttypes.TStatusCode.SUCCESS_WITH_INFO_STATUS, + ttypes.TStatusCode.STILL_EXECUTING_STATUS, ] for code in success_codes: @@ -314,69 +447,84 @@ def test_handle_execute_response_checks_operation_state_in_direct_results(self): operationStatus=ttypes.TGetOperationStatusResp( status=self.okay_status, operationState=ttypes.TOperationState.ERROR_STATE, - errorMessage="some information about the error"), + errorMessage="some information about the error", + ), resultSetMetadata=None, resultSet=None, - closeOperation=None)) - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + closeOperation=None, + ), + ) + thrift_backend = ThriftBackend( + "foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions() + ) with self.assertRaises(DatabaseError) as cm: thrift_backend._handle_execute_response(t_execute_resp, Mock()) self.assertIn("some information about the error", str(cm.exception)) - def test_handle_execute_response_sets_compression_in_direct_results(self): + @patch("databricks.sql.utils.ResultSetQueueFactory.build_queue", return_value=Mock()) + def test_handle_execute_response_sets_compression_in_direct_results(self, build_queue): for resp_type in self.execute_response_types: - lz4Compressed=Mock() - resultSet=MagicMock() + lz4Compressed = Mock() + resultSet = MagicMock() resultSet.results.startRowOffset = 0 t_execute_resp = resp_type( status=Mock(), operationHandle=Mock(), directResults=ttypes.TSparkDirectResults( - operationStatus= Mock(), + operationStatus=Mock(), resultSetMetadata=ttypes.TGetResultSetMetadataResp( status=self.okay_status, resultFormat=ttypes.TSparkRowSetType.ARROW_BASED_SET, schema=MagicMock(), arrowSchema=MagicMock(), - lz4Compressed=lz4Compressed), + lz4Compressed=lz4Compressed, + ), resultSet=resultSet, - closeOperation=None)) - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + closeOperation=None, + ), + ) + thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) execute_response = thrift_backend._handle_execute_response(t_execute_resp, Mock()) self.assertEqual(execute_response.lz4_compressed, lz4Compressed) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_handle_execute_response_checks_operation_state_in_polls(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value error_resp = ttypes.TGetOperationStatusResp( status=self.okay_status, operationState=ttypes.TOperationState.ERROR_STATE, - errorMessage="some information about the error") + errorMessage="some information about the error", + ) closed_resp = ttypes.TGetOperationStatusResp( - status=self.okay_status, operationState=ttypes.TOperationState.CLOSED_STATE) + status=self.okay_status, operationState=ttypes.TOperationState.CLOSED_STATE + ) - for op_state_resp, exec_resp_type in itertools.product([error_resp, closed_resp], - self.execute_response_types): + for op_state_resp, exec_resp_type in itertools.product( + [error_resp, closed_resp], self.execute_response_types + ): with self.subTest(op_state_resp=op_state_resp, exec_resp_type=exec_resp_type): tcli_service_instance = tcli_service_class.return_value t_execute_resp = exec_resp_type( status=self.okay_status, directResults=None, - operationHandle=self.operation_handle) + operationHandle=self.operation_handle, + ) tcli_service_instance.GetOperationStatus.return_value = op_state_resp - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions() + ) with self.assertRaises(DatabaseError) as cm: thrift_backend._handle_execute_response(t_execute_resp, Mock()) if op_state_resp.errorMessage: self.assertIn(op_state_resp.errorMessage, str(cm.exception)) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_get_status_uses_display_message_if_available(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value @@ -387,21 +535,23 @@ def test_get_status_uses_display_message_if_available(self, tcli_service_class): operationState=ttypes.TOperationState.ERROR_STATE, errorMessage="foo", displayMessage=display_message, - diagnosticInfo=diagnostic_info) + diagnosticInfo=diagnostic_info, + ) t_execute_resp = ttypes.TExecuteStatementResp( - status=self.okay_status, directResults=None, operationHandle=self.operation_handle) + status=self.okay_status, directResults=None, operationHandle=self.operation_handle + ) tcli_service_instance.GetOperationStatus.return_value = t_get_operation_status_resp tcli_service_instance.ExecuteStatement.return_value = t_execute_resp - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) with self.assertRaises(DatabaseError) as cm: thrift_backend.execute_command(Mock(), Mock(), 100, 100, Mock(), Mock()) self.assertEqual(display_message, str(cm.exception)) self.assertIn(diagnostic_info, str(cm.exception.message_with_context())) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_direct_results_uses_display_message_if_available(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value @@ -412,7 +562,8 @@ def test_direct_results_uses_display_message_if_available(self, tcli_service_cla operationState=ttypes.TOperationState.ERROR_STATE, errorMessage="foo", displayMessage=display_message, - diagnosticInfo=diagnostic_info) + diagnosticInfo=diagnostic_info, + ) t_execute_resp = ttypes.TExecuteStatementResp( status=self.okay_status, @@ -420,11 +571,13 @@ def test_direct_results_uses_display_message_if_available(self, tcli_service_cla operationStatus=t_get_operation_status_resp, resultSetMetadata=None, resultSet=None, - closeOperation=None)) + closeOperation=None, + ), + ) tcli_service_instance.ExecuteStatement.return_value = t_execute_resp - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) with self.assertRaises(DatabaseError) as cm: thrift_backend.execute_command(Mock(), Mock(), 100, 100, Mock(), Mock()) @@ -439,7 +592,9 @@ def test_handle_execute_response_checks_direct_results_for_error_statuses(self): operationStatus=ttypes.TGetOperationStatusResp(status=self.bad_status), resultSetMetadata=None, resultSet=None, - closeOperation=None)) + closeOperation=None, + ), + ) resp_2 = resp_type( status=self.okay_status, @@ -447,7 +602,9 @@ def test_handle_execute_response_checks_direct_results_for_error_statuses(self): operationStatus=None, resultSetMetadata=ttypes.TGetResultSetMetadataResp(status=self.bad_status), resultSet=None, - closeOperation=None)) + closeOperation=None, + ), + ) resp_3 = resp_type( status=self.okay_status, @@ -455,7 +612,9 @@ def test_handle_execute_response_checks_direct_results_for_error_statuses(self): operationStatus=None, resultSetMetadata=None, resultSet=ttypes.TFetchResultsResp(status=self.bad_status), - closeOperation=None)) + closeOperation=None, + ), + ) resp_4 = resp_type( status=self.okay_status, @@ -463,17 +622,21 @@ def test_handle_execute_response_checks_direct_results_for_error_statuses(self): operationStatus=None, resultSetMetadata=None, resultSet=None, - closeOperation=ttypes.TCloseOperationResp(status=self.bad_status))) + closeOperation=ttypes.TCloseOperationResp(status=self.bad_status), + ), + ) for error_resp in [resp_1, resp_2, resp_3, resp_4]: with self.subTest(error_resp=error_resp): - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions() + ) with self.assertRaises(DatabaseError) as cm: thrift_backend._handle_execute_response(error_resp, Mock()) self.assertIn("this is a bad error", str(cm.exception)) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_handle_execute_response_can_handle_without_direct_results(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value @@ -497,17 +660,24 @@ def test_handle_execute_response_can_handle_without_direct_results(self, tcli_se ) op_state_3 = ttypes.TGetOperationStatusResp( - status=self.okay_status, operationState=ttypes.TOperationState.FINISHED_STATE) + status=self.okay_status, operationState=ttypes.TOperationState.FINISHED_STATE + ) tcli_service_instance.GetResultSetMetadata.return_value = self.metadata_resp tcli_service_instance.GetOperationStatus.side_effect = [ - op_state_1, op_state_2, op_state_3 + op_state_1, + op_state_2, + op_state_3, ] - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions() + ) results_message_response = thrift_backend._handle_execute_response( - execute_resp, Mock()) - self.assertEqual(results_message_response.status, - ttypes.TOperationState.FINISHED_STATE) + execute_resp, Mock() + ) + self.assertEqual( + results_message_response.status, ttypes.TOperationState.FINISHED_STATE + ) def test_handle_execute_response_can_handle_with_direct_results(self): result_set_metadata_mock = Mock() @@ -519,16 +689,20 @@ def test_handle_execute_response_can_handle_with_direct_results(self): ), resultSetMetadata=result_set_metadata_mock, resultSet=Mock(), - closeOperation=Mock()) + closeOperation=Mock(), + ) for resp_type in self.execute_response_types: with self.subTest(resp_type=resp_type): execute_resp = resp_type( status=self.okay_status, directResults=direct_results_message, - operationHandle=self.operation_handle) + operationHandle=self.operation_handle, + ) - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend( + "foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions() + ) thrift_backend._results_message_to_execute_response = Mock() thrift_backend._handle_execute_response(execute_resp, Mock()) @@ -538,7 +712,7 @@ def test_handle_execute_response_can_handle_with_direct_results(self): ttypes.TOperationState.FINISHED_STATE, ) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_use_arrow_schema_if_available(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value arrow_schema_mock = MagicMock(name="Arrow schema mock") @@ -548,7 +722,8 @@ def test_use_arrow_schema_if_available(self, tcli_service_class): status=self.okay_status, resultFormat=ttypes.TSparkRowSetType.ARROW_BASED_SET, schema=hive_schema_mock, - arrowSchema=arrow_schema_mock) + arrowSchema=arrow_schema_mock, + ) t_execute_resp = ttypes.TExecuteStatementResp( status=self.okay_status, @@ -562,7 +737,7 @@ def test_use_arrow_schema_if_available(self, tcli_service_class): self.assertEqual(execute_response.arrow_schema_bytes, arrow_schema_mock) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_fall_back_to_hive_schema_if_no_arrow_schema(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value hive_schema_mock = MagicMock(name="Hive schema mock") @@ -571,7 +746,8 @@ def test_fall_back_to_hive_schema_if_no_arrow_schema(self, tcli_service_class): status=self.okay_status, resultFormat=ttypes.TSparkRowSetType.ARROW_BASED_SET, arrowSchema=None, - schema=hive_schema_mock) + schema=hive_schema_mock, + ) t_execute_resp = ttypes.TExecuteStatementResp( status=self.okay_status, @@ -583,14 +759,18 @@ def test_fall_back_to_hive_schema_if_no_arrow_schema(self, tcli_service_class): thrift_backend = self._make_fake_thrift_backend() thrift_backend._handle_execute_response(t_execute_resp, Mock()) - self.assertEqual(hive_schema_mock, - thrift_backend._hive_schema_to_arrow_schema.call_args[0][0]) + self.assertEqual( + hive_schema_mock, thrift_backend._hive_schema_to_arrow_schema.call_args[0][0] + ) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.utils.ResultSetQueueFactory.build_queue", return_value=Mock()) + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_handle_execute_response_reads_has_more_rows_in_direct_results( - self, tcli_service_class): - for has_more_rows, resp_type in itertools.product([True, False], - self.execute_response_types): + self, tcli_service_class, build_queue + ): + for has_more_rows, resp_type in itertools.product( + [True, False], self.execute_response_types + ): with self.subTest(has_more_rows=has_more_rows, resp_type=resp_type): tcli_service_instance = tcli_service_class.return_value results_mock = Mock() @@ -606,11 +786,13 @@ def test_handle_execute_response_reads_has_more_rows_in_direct_results( hasMoreRows=has_more_rows, results=results_mock, ), - closeOperation=Mock()) + closeOperation=Mock(), + ) execute_resp = resp_type( status=self.okay_status, directResults=direct_results_message, - operationHandle=self.operation_handle) + operationHandle=self.operation_handle, + ) tcli_service_instance.GetResultSetMetadata.return_value = self.metadata_resp thrift_backend = self._make_fake_thrift_backend() @@ -619,11 +801,14 @@ def test_handle_execute_response_reads_has_more_rows_in_direct_results( self.assertEqual(has_more_rows, execute_response.has_more_rows) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.utils.ResultSetQueueFactory.build_queue", return_value=Mock()) + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_handle_execute_response_reads_has_more_rows_in_result_response( - self, tcli_service_class): - for has_more_rows, resp_type in itertools.product([True, False], - self.execute_response_types): + self, tcli_service_class, build_queue + ): + for has_more_rows, resp_type in itertools.product( + [True, False], self.execute_response_types + ): with self.subTest(has_more_rows=has_more_rows, resp_type=resp_type): tcli_service_instance = tcli_service_class.return_value results_mock = MagicMock() @@ -632,18 +817,23 @@ def test_handle_execute_response_reads_has_more_rows_in_result_response( execute_resp = resp_type( status=self.okay_status, directResults=None, - operationHandle=self.operation_handle) + operationHandle=self.operation_handle, + ) fetch_results_resp = ttypes.TFetchResultsResp( status=self.okay_status, hasMoreRows=has_more_rows, results=results_mock, + resultSetMetadata=ttypes.TGetResultSetMetadataResp( + resultFormat=ttypes.TSparkRowSetType.ARROW_BASED_SET + ), ) operation_status_resp = ttypes.TGetOperationStatusResp( status=self.okay_status, operationState=ttypes.TOperationState.FINISHED_STATE, - errorMessage="some information about the error") + errorMessage="some information about the error", + ) tcli_service_instance.FetchResults.return_value = fetch_results_resp tcli_service_instance.GetOperationStatus.return_value = operation_status_resp @@ -658,11 +848,12 @@ def test_handle_execute_response_reads_has_more_rows_in_result_response( expected_row_start_offset=0, lz4_compressed=False, arrow_schema_bytes=Mock(), - description=Mock()) + description=Mock(), + ) self.assertEqual(has_more_rows, has_more_rows_resp) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_arrow_batches_row_count_are_respected(self, tcli_service_class): # make some semi-real arrow batches and check the number of rows is correct in the queue tcli_service_instance = tcli_service_class.return_value @@ -674,16 +865,27 @@ def test_arrow_batches_row_count_are_respected(self, tcli_service_class): rows=[], arrowBatches=[ ttypes.TSparkArrowBatch(batch=bytearray(), rowCount=15) for _ in range(10) - ])) + ], + ), + resultSetMetadata=ttypes.TGetResultSetMetadataResp( + resultFormat=ttypes.TSparkRowSetType.ARROW_BASED_SET + ), + ) tcli_service_instance.FetchResults.return_value = t_fetch_results_resp - schema = pyarrow.schema([ - pyarrow.field("column1", pyarrow.int32()), - pyarrow.field("column2", pyarrow.string()), - pyarrow.field("column3", pyarrow.float64()), - pyarrow.field("column3", pyarrow.binary()) - ]).serialize().to_pybytes() - - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + schema = ( + pyarrow.schema( + [ + pyarrow.field("column1", pyarrow.int32()), + pyarrow.field("column2", pyarrow.string()), + pyarrow.field("column3", pyarrow.float64()), + pyarrow.field("column3", pyarrow.binary()), + ] + ) + .serialize() + .to_pybytes() + ) + + thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) arrow_queue, has_more_results = thrift_backend.fetch_results( op_handle=Mock(), max_rows=1, @@ -691,16 +893,17 @@ def test_arrow_batches_row_count_are_respected(self, tcli_service_class): expected_row_start_offset=0, lz4_compressed=False, arrow_schema_bytes=schema, - description=MagicMock()) + description=MagicMock(), + ) self.assertEqual(arrow_queue.n_valid_rows, 15 * 10) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_execute_statement_calls_client_and_handle_execute_response(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value response = Mock() tcli_service_instance.ExecuteStatement.return_value = response - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) thrift_backend._handle_execute_response = Mock() cursor_mock = Mock() @@ -713,12 +916,12 @@ def test_execute_statement_calls_client_and_handle_execute_response(self, tcli_s # Check response handling thrift_backend._handle_execute_response.assert_called_with(response, cursor_mock) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_get_catalogs_calls_client_and_handle_execute_response(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value response = Mock() tcli_service_instance.GetCatalogs.return_value = response - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) thrift_backend._handle_execute_response = Mock() cursor_mock = Mock() @@ -730,12 +933,12 @@ def test_get_catalogs_calls_client_and_handle_execute_response(self, tcli_servic # Check response handling thrift_backend._handle_execute_response.assert_called_with(response, cursor_mock) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_get_schemas_calls_client_and_handle_execute_response(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value response = Mock() tcli_service_instance.GetSchemas.return_value = response - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) thrift_backend._handle_execute_response = Mock() cursor_mock = Mock() @@ -745,7 +948,8 @@ def test_get_schemas_calls_client_and_handle_execute_response(self, tcli_service 200, cursor_mock, catalog_name="catalog_pattern", - schema_name="schema_pattern") + schema_name="schema_pattern", + ) # Check call to client req = tcli_service_instance.GetSchemas.call_args[0][0] get_direct_results = ttypes.TSparkGetDirectResults(maxRows=100, maxBytes=200) @@ -755,12 +959,12 @@ def test_get_schemas_calls_client_and_handle_execute_response(self, tcli_service # Check response handling thrift_backend._handle_execute_response.assert_called_with(response, cursor_mock) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_get_tables_calls_client_and_handle_execute_response(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value response = Mock() tcli_service_instance.GetTables.return_value = response - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) thrift_backend._handle_execute_response = Mock() cursor_mock = Mock() @@ -772,7 +976,8 @@ def test_get_tables_calls_client_and_handle_execute_response(self, tcli_service_ catalog_name="catalog_pattern", schema_name="schema_pattern", table_name="table_pattern", - table_types=["type1", "type2"]) + table_types=["type1", "type2"], + ) # Check call to client req = tcli_service_instance.GetTables.call_args[0][0] get_direct_results = ttypes.TSparkGetDirectResults(maxRows=100, maxBytes=200) @@ -784,12 +989,12 @@ def test_get_tables_calls_client_and_handle_execute_response(self, tcli_service_ # Check response handling thrift_backend._handle_execute_response.assert_called_with(response, cursor_mock) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_get_columns_calls_client_and_handle_execute_response(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value response = Mock() tcli_service_instance.GetColumns.return_value = response - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) thrift_backend._handle_execute_response = Mock() cursor_mock = Mock() @@ -801,7 +1006,8 @@ def test_get_columns_calls_client_and_handle_execute_response(self, tcli_service catalog_name="catalog_pattern", schema_name="schema_pattern", table_name="table_pattern", - column_name="column_pattern") + column_name="column_pattern", + ) # Check call to client req = tcli_service_instance.GetColumns.call_args[0][0] get_direct_results = ttypes.TSparkGetDirectResults(maxRows=100, maxBytes=200) @@ -813,39 +1019,43 @@ def test_get_columns_calls_client_and_handle_execute_response(self, tcli_service # Check response handling thrift_backend._handle_execute_response.assert_called_with(response, cursor_mock) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_open_session_user_provided_session_id_optional(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value tcli_service_instance.OpenSession.return_value = self.open_session_resp - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) thrift_backend.open_session({}, None, None) self.assertEqual(len(tcli_service_instance.OpenSession.call_args_list), 1) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_op_handle_respected_in_close_command(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) thrift_backend.close_command(self.operation_handle) - self.assertEqual(tcli_service_instance.CloseOperation.call_args[0][0].operationHandle, - self.operation_handle) + self.assertEqual( + tcli_service_instance.CloseOperation.call_args[0][0].operationHandle, + self.operation_handle, + ) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_session_handle_respected_in_close_session(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) thrift_backend.close_session(self.session_handle) - self.assertEqual(tcli_service_instance.CloseSession.call_args[0][0].sessionHandle, - self.session_handle) + self.assertEqual( + tcli_service_instance.CloseSession.call_args[0][0].sessionHandle, self.session_handle + ) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_non_arrow_non_column_based_set_triggers_exception(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value results_mock = Mock() results_mock.startRowOffset = 0 execute_statement_resp = ttypes.TExecuteStatementResp( - status=self.okay_status, directResults=None, operationHandle=self.operation_handle) + status=self.okay_status, directResults=None, operationHandle=self.operation_handle + ) metadata_resp = ttypes.TGetResultSetMetadataResp( status=self.okay_status, @@ -855,7 +1065,8 @@ def test_non_arrow_non_column_based_set_triggers_exception(self, tcli_service_cl operation_status_resp = ttypes.TGetOperationStatusResp( status=self.okay_status, operationState=ttypes.TOperationState.FINISHED_STATE, - errorMessage="some information about the error") + errorMessage="some information about the error", + ) tcli_service_instance.ExecuteStatement.return_value = execute_statement_resp tcli_service_instance.GetResultSetMetadata.return_value = metadata_resp @@ -868,15 +1079,16 @@ def test_non_arrow_non_column_based_set_triggers_exception(self, tcli_service_cl def test_create_arrow_table_raises_error_for_unsupported_type(self): t_row_set = ttypes.TRowSet() - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) with self.assertRaises(OperationalError): thrift_backend._create_arrow_table(t_row_set, Mock(), None, Mock()) - @patch.object(ThriftBackend, "_convert_arrow_based_set_to_arrow_table") - @patch.object(ThriftBackend, "_convert_column_based_set_to_arrow_table") - def test_create_arrow_table_calls_correct_conversion_method(self, convert_col_mock, - convert_arrow_mock): - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + @patch("databricks.sql.thrift_backend.convert_arrow_based_set_to_arrow_table") + @patch("databricks.sql.thrift_backend.convert_column_based_set_to_arrow_table") + def test_create_arrow_table_calls_correct_conversion_method( + self, convert_col_mock, convert_arrow_mock + ): + thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) convert_arrow_mock.return_value = (MagicMock(), Mock()) convert_col_mock.return_value = (MagicMock(), Mock()) @@ -898,38 +1110,47 @@ def test_create_arrow_table_calls_correct_conversion_method(self, convert_col_mo @patch("lz4.frame.decompress") @patch("pyarrow.ipc.open_stream") def test_convert_arrow_based_set_to_arrow_table(self, open_stream_mock, lz4_decompress_mock): - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) - - lz4_decompress_mock.return_value = bytearray('Testing','utf-8') - - schema = pyarrow.schema([ - pyarrow.field("column1", pyarrow.int32()), - ]).serialize().to_pybytes() - - arrow_batches = [ttypes.TSparkArrowBatch(batch=bytearray('Testing','utf-8'), rowCount=1) for _ in range(10)] - thrift_backend._convert_arrow_based_set_to_arrow_table(arrow_batches, False, schema) + thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) + + lz4_decompress_mock.return_value = bytearray("Testing", "utf-8") + + schema = ( + pyarrow.schema( + [ + pyarrow.field("column1", pyarrow.int32()), + ] + ) + .serialize() + .to_pybytes() + ) + + arrow_batches = [ + ttypes.TSparkArrowBatch(batch=bytearray("Testing", "utf-8"), rowCount=1) + for _ in range(10) + ] + utils.convert_arrow_based_set_to_arrow_table(arrow_batches, False, schema) lz4_decompress_mock.assert_not_called() - thrift_backend._convert_arrow_based_set_to_arrow_table(arrow_batches, True, schema) + utils.convert_arrow_based_set_to_arrow_table(arrow_batches, True, schema) lz4_decompress_mock.assert_called() - def test_convert_column_based_set_to_arrow_table_without_nulls(self): # Deliberately duplicate the column name to check that dups work field_names = ["column1", "column2", "column3", "column3"] - description = [(name, ) for name in field_names] + description = [(name,) for name in field_names] t_cols = [ ttypes.TColumn(i32Val=ttypes.TI32Column(values=[1, 2, 3], nulls=bytes(1))), ttypes.TColumn( - stringVal=ttypes.TStringColumn(values=["s1", "s2", "s3"], nulls=bytes(1))), + stringVal=ttypes.TStringColumn(values=["s1", "s2", "s3"], nulls=bytes(1)) + ), ttypes.TColumn(doubleVal=ttypes.TDoubleColumn(values=[1.15, 2.2, 3.3], nulls=bytes(1))), ttypes.TColumn( - binaryVal=ttypes.TBinaryColumn(values=[b'\x11', b'\x22', b'\x33'], nulls=bytes(1))) + binaryVal=ttypes.TBinaryColumn(values=[b"\x11", b"\x22", b"\x33"], nulls=bytes(1)) + ), ] - arrow_table, n_rows = ThriftBackend._convert_column_based_set_to_arrow_table( - t_cols, description) + arrow_table, n_rows = utils.convert_column_based_set_to_arrow_table(t_cols, description) self.assertEqual(n_rows, 3) # Check schema, column names and types @@ -947,48 +1168,50 @@ def test_convert_column_based_set_to_arrow_table_without_nulls(self): self.assertEqual(arrow_table.column(0).to_pylist(), [1, 2, 3]) self.assertEqual(arrow_table.column(1).to_pylist(), ["s1", "s2", "s3"]) self.assertEqual(arrow_table.column(2).to_pylist(), [1.15, 2.2, 3.3]) - self.assertEqual(arrow_table.column(3).to_pylist(), [b'\x11', b'\x22', b'\x33']) + self.assertEqual(arrow_table.column(3).to_pylist(), [b"\x11", b"\x22", b"\x33"]) def test_convert_column_based_set_to_arrow_table_with_nulls(self): field_names = ["column1", "column2", "column3", "column3"] - description = [(name, ) for name in field_names] + description = [(name,) for name in field_names] t_cols = [ ttypes.TColumn(i32Val=ttypes.TI32Column(values=[1, 2, 3], nulls=bytes([1]))), ttypes.TColumn( - stringVal=ttypes.TStringColumn(values=["s1", "s2", "s3"], nulls=bytes([2]))), + stringVal=ttypes.TStringColumn(values=["s1", "s2", "s3"], nulls=bytes([2])) + ), ttypes.TColumn( - doubleVal=ttypes.TDoubleColumn(values=[1.15, 2.2, 3.3], nulls=bytes([4]))), + doubleVal=ttypes.TDoubleColumn(values=[1.15, 2.2, 3.3], nulls=bytes([4])) + ), ttypes.TColumn( - binaryVal=ttypes.TBinaryColumn( - values=[b'\x11', b'\x22', b'\x33'], nulls=bytes([3]))) + binaryVal=ttypes.TBinaryColumn(values=[b"\x11", b"\x22", b"\x33"], nulls=bytes([3])) + ), ] - arrow_table, n_rows = ThriftBackend._convert_column_based_set_to_arrow_table( - t_cols, description) + arrow_table, n_rows = utils.convert_column_based_set_to_arrow_table(t_cols, description) self.assertEqual(n_rows, 3) # Check data self.assertEqual(arrow_table.column(0).to_pylist(), [None, 2, 3]) self.assertEqual(arrow_table.column(1).to_pylist(), ["s1", None, "s3"]) self.assertEqual(arrow_table.column(2).to_pylist(), [1.15, 2.2, None]) - self.assertEqual(arrow_table.column(3).to_pylist(), [None, None, b'\x33']) + self.assertEqual(arrow_table.column(3).to_pylist(), [None, None, b"\x33"]) def test_convert_column_based_set_to_arrow_table_uses_types_from_col_set(self): field_names = ["column1", "column2", "column3", "column3"] - description = [(name, ) for name in field_names] + description = [(name,) for name in field_names] t_cols = [ ttypes.TColumn(i32Val=ttypes.TI32Column(values=[1, 2, 3], nulls=bytes(1))), ttypes.TColumn( - stringVal=ttypes.TStringColumn(values=["s1", "s2", "s3"], nulls=bytes(1))), + stringVal=ttypes.TStringColumn(values=["s1", "s2", "s3"], nulls=bytes(1)) + ), ttypes.TColumn(doubleVal=ttypes.TDoubleColumn(values=[1.15, 2.2, 3.3], nulls=bytes(1))), ttypes.TColumn( - binaryVal=ttypes.TBinaryColumn(values=[b'\x11', b'\x22', b'\x33'], nulls=bytes(1))) + binaryVal=ttypes.TBinaryColumn(values=[b"\x11", b"\x22", b"\x33"], nulls=bytes(1)) + ), ] - arrow_table, n_rows = ThriftBackend._convert_column_based_set_to_arrow_table( - t_cols, description) + arrow_table, n_rows = utils.convert_column_based_set_to_arrow_table(t_cols, description) self.assertEqual(n_rows, 3) # Check schema, column names and types @@ -1006,9 +1229,9 @@ def test_convert_column_based_set_to_arrow_table_uses_types_from_col_set(self): self.assertEqual(arrow_table.column(0).to_pylist(), [1, 2, 3]) self.assertEqual(arrow_table.column(1).to_pylist(), ["s1", "s2", "s3"]) self.assertEqual(arrow_table.column(2).to_pylist(), [1.15, 2.2, 3.3]) - self.assertEqual(arrow_table.column(3).to_pylist(), [b'\x11', b'\x22', b'\x33']) + self.assertEqual(arrow_table.column(3).to_pylist(), [b"\x11", b"\x22", b"\x33"]) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_cancel_command_uses_active_op_handle(self, tcli_service_class): tcli_service_instance = tcli_service_class.return_value @@ -1016,8 +1239,10 @@ def test_cancel_command_uses_active_op_handle(self, tcli_service_class): active_op_handle_mock = Mock() thrift_backend.cancel_command(active_op_handle_mock) - self.assertEqual(tcli_service_instance.CancelOperation.call_args[0][0].operationHandle, - active_op_handle_mock) + self.assertEqual( + tcli_service_instance.CancelOperation.call_args[0][0].operationHandle, + active_op_handle_mock, + ) def test_handle_execute_response_sets_active_op_handle(self): thrift_backend = self._make_fake_thrift_backend() @@ -1031,11 +1256,12 @@ def test_handle_execute_response_sets_active_op_handle(self): self.assertEqual(mock_resp.operationHandle, mock_cursor.active_op_handle) - @patch("thrift.transport.THttpClient.THttpClient") + @patch("databricks.sql.auth.thrift_http_client.THttpClient") @patch("databricks.sql.thrift_api.TCLIService.TCLIService.Client.GetOperationStatus") @patch("databricks.sql.thrift_backend._retry_policy", new_callable=retry_policy_factory) def test_make_request_will_retry_GetOperationStatus( - self, mock_retry_policy, mock_GetOperationStatus, t_transport_class): + self, mock_retry_policy, mock_GetOperationStatus, t_transport_class + ): import thrift, errno from databricks.sql.thrift_api.TCLIService.TCLIService import Client @@ -1059,17 +1285,21 @@ def test_make_request_will_retry_GetOperationStatus( thrift_backend = ThriftBackend( "foobar", 443, - "path", [], + "path", + [], auth_provider=AuthProvider(), + ssl_options=SSLOptions(), _retry_stop_after_attempts_count=EXPECTED_RETRIES, - _retry_delay_default=1) - + _retry_delay_default=1, + ) with self.assertRaises(RequestError) as cm: thrift_backend.make_request(client.GetOperationStatus, req) - self.assertEqual(NoRetryReason.OUT_OF_ATTEMPTS.value, cm.exception.context["no-retry-reason"]) - self.assertEqual(f'{EXPECTED_RETRIES}/{EXPECTED_RETRIES}', cm.exception.context["attempt"]) + self.assertEqual( + NoRetryReason.OUT_OF_ATTEMPTS.value, cm.exception.context["no-retry-reason"] + ) + self.assertEqual(f"{EXPECTED_RETRIES}/{EXPECTED_RETRIES}", cm.exception.context["attempt"]) # Unusual OSError code mock_GetOperationStatus.side_effect = OSError(errno.EEXIST, "File does not exist") @@ -1085,24 +1315,58 @@ def test_make_request_will_retry_GetOperationStatus( self.assertEqual(cm.output[1], cm.output[0]) # The warnings should include this text - self.assertIn(f"{this_gos_name} failed with code {errno.EEXIST} and will attempt to retry", cm.output[0]) + self.assertIn( + f"{this_gos_name} failed with code {errno.EEXIST} and will attempt to retry", + cm.output[0], + ) + @patch("databricks.sql.thrift_api.TCLIService.TCLIService.Client.GetOperationStatus") + @patch("databricks.sql.thrift_backend._retry_policy", new_callable=retry_policy_factory) + def test_make_request_will_retry_GetOperationStatus_for_http_error( + self, mock_retry_policy, mock_gos + ): - @patch("thrift.transport.THttpClient.THttpClient") - def test_make_request_wont_retry_if_headers_not_present(self, t_transport_class): - t_transport_instance = t_transport_class.return_value - t_transport_instance.code = 429 - t_transport_instance.headers = {"foo": "bar"} - mock_method = Mock() - mock_method.__name__ = "method name" - mock_method.side_effect = Exception("This method fails") + import urllib3.exceptions - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + mock_gos.side_effect = urllib3.exceptions.HTTPError("Read timed out") - with self.assertRaises(OperationalError) as cm: - thrift_backend.make_request(mock_method, Mock()) + import thrift, errno + from databricks.sql.thrift_api.TCLIService.TCLIService import Client + from databricks.sql.exc import RequestError + from databricks.sql.utils import NoRetryReason + from databricks.sql.auth.thrift_http_client import THttpClient - self.assertIn("This method fails", str(cm.exception.message_with_context())) + this_gos_name = "GetOperationStatus" + mock_gos.__name__ = this_gos_name + + protocol = thrift.protocol.TBinaryProtocol.TBinaryProtocol(THttpClient) + client = Client(protocol) + + req = ttypes.TGetOperationStatusReq( + operationHandle=self.operation_handle, + getProgressUpdate=False, + ) + + EXPECTED_RETRIES = 2 + + thrift_backend = ThriftBackend( + "foobar", + 443, + "path", + [], + auth_provider=AuthProvider(), + ssl_options=SSLOptions(), + _retry_stop_after_attempts_count=EXPECTED_RETRIES, + _retry_delay_default=1, + ) + + with self.assertRaises(RequestError) as cm: + thrift_backend.make_request(client.GetOperationStatus, req) + + self.assertEqual( + NoRetryReason.OUT_OF_ATTEMPTS.value, cm.exception.context["no-retry-reason"] + ) + self.assertEqual(f"{EXPECTED_RETRIES}/{EXPECTED_RETRIES}", cm.exception.context["attempt"]) @patch("thrift.transport.THttpClient.THttpClient") def test_make_request_wont_retry_if_error_code_not_429_or_503(self, t_transport_class): @@ -1113,7 +1377,7 @@ def test_make_request_wont_retry_if_error_code_not_429_or_503(self, t_transport_ mock_method.__name__ = "method name" mock_method.side_effect = Exception("This method fails") - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) with self.assertRaises(OperationalError) as cm: thrift_backend.make_request(mock_method, Mock()) @@ -1123,7 +1387,8 @@ def test_make_request_wont_retry_if_error_code_not_429_or_503(self, t_transport_ @patch("databricks.sql.auth.thrift_http_client.THttpClient") @patch("databricks.sql.thrift_backend._retry_policy", new_callable=retry_policy_factory) def test_make_request_will_retry_stop_after_attempts_count_if_retryable( - self, mock_retry_policy, t_transport_class): + self, mock_retry_policy, t_transport_class + ): t_transport_instance = t_transport_class.return_value t_transport_instance.code = 429 t_transport_instance.headers = {"Retry-After": "0"} @@ -1134,11 +1399,14 @@ def test_make_request_will_retry_stop_after_attempts_count_if_retryable( thrift_backend = ThriftBackend( "foobar", 443, - "path", [], + "path", + [], auth_provider=AuthProvider(), + ssl_options=SSLOptions(), _retry_stop_after_attempts_count=14, _retry_delay_max=0, - _retry_delay_min=0) + _retry_delay_min=0, + ) with self.assertRaises(OperationalError) as cm: thrift_backend.make_request(mock_method, Mock()) @@ -1155,17 +1423,25 @@ def test_make_request_will_read_error_message_headers_if_set(self, t_transport_c mock_method.__name__ = "method name" mock_method.side_effect = Exception("This method fails") - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) - - error_headers = [[("x-thriftserver-error-message", "thrift server error message")], - [("x-databricks-error-or-redirect-message", "databricks error message")], - [("x-databricks-error-or-redirect-message", "databricks error message"), - ("x-databricks-reason-phrase", "databricks error reason")], - [("x-thriftserver-error-message", "thrift server error message"), - ("x-databricks-error-or-redirect-message", "databricks error message"), - ("x-databricks-reason-phrase", "databricks error reason")], - [("x-thriftserver-error-message", "thrift server error message"), - ("x-databricks-error-or-redirect-message", "databricks error message")]] + thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) + + error_headers = [ + [("x-thriftserver-error-message", "thrift server error message")], + [("x-databricks-error-or-redirect-message", "databricks error message")], + [ + ("x-databricks-error-or-redirect-message", "databricks error message"), + ("x-databricks-reason-phrase", "databricks error reason"), + ], + [ + ("x-thriftserver-error-message", "thrift server error message"), + ("x-databricks-error-or-redirect-message", "databricks error message"), + ("x-databricks-reason-phrase", "databricks error reason"), + ], + [ + ("x-thriftserver-error-message", "thrift server error message"), + ("x-databricks-error-or-redirect-message", "databricks error message"), + ], + ] for headers in error_headers: t_transport_instance.headers = dict(headers) @@ -1176,16 +1452,17 @@ def test_make_request_will_read_error_message_headers_if_set(self, t_transport_c self.assertIn(header[1], str(cm.exception)) @staticmethod - def make_table_and_desc(height, n_decimal_cols, width, precision, scale, int_constant, - decimal_constant): + def make_table_and_desc( + height, n_decimal_cols, width, precision, scale, int_constant, decimal_constant + ): int_col = [int_constant for _ in range(height)] decimal_col = [decimal_constant for _ in range(height)] data = OrderedDict({"col{}".format(i): int_col for i in range(width - n_decimal_cols)}) decimals = OrderedDict({"col_dec{}".format(i): decimal_col for i in range(n_decimal_cols)}) data.update(decimals) - int_desc = ([("", "int")] * (width - n_decimal_cols)) - decimal_desc = ([("", "decimal", None, None, precision, scale, None)] * n_decimal_cols) + int_desc = [("", "int")] * (width - n_decimal_cols) + decimal_desc = [("", "decimal", None, None, precision, scale, None)] * n_decimal_cols description = int_desc + decimal_desc table = pyarrow.Table.from_pydict(data) @@ -1201,30 +1478,39 @@ def test_arrow_decimal_conversion(self): for n_decimal_cols in [0, 1, 10]: for height in [0, 1, 10]: with self.subTest(n_decimal_cols=n_decimal_cols, height=height): - table, description = self.make_table_and_desc(height, n_decimal_cols, width, - precision, scale, int_constant, - decimal_constant) - decimal_converted_table = ThriftBackend._convert_decimals_in_arrow_table( - table, description) + table, description = self.make_table_and_desc( + height, + n_decimal_cols, + width, + precision, + scale, + int_constant, + decimal_constant, + ) + decimal_converted_table = utils.convert_decimals_in_arrow_table( + table, description + ) for i in range(width): if height > 0: if i < width - n_decimal_cols: self.assertEqual( - decimal_converted_table.field(i).type, pyarrow.int64()) + decimal_converted_table.field(i).type, pyarrow.int64() + ) else: self.assertEqual( decimal_converted_table.field(i).type, - pyarrow.decimal128(precision=precision, scale=scale)) + pyarrow.decimal128(precision=precision, scale=scale), + ) int_col = [int_constant for _ in range(height)] decimal_col = [Decimal(decimal_constant) for _ in range(height)] expected_result = OrderedDict( - {"col{}".format(i): int_col - for i in range(width - n_decimal_cols)}) + {"col{}".format(i): int_col for i in range(width - n_decimal_cols)} + ) decimals = OrderedDict( - {"col_dec{}".format(i): decimal_col - for i in range(n_decimal_cols)}) + {"col_dec{}".format(i): decimal_col for i in range(n_decimal_cols)} + ) expected_result.update(decimals) self.assertEqual(decimal_converted_table.to_pydict(), expected_result) @@ -1235,32 +1521,34 @@ def test_retry_args_passthrough(self, mock_http_client): "_retry_delay_min": 6, "_retry_delay_max": 10, "_retry_stop_after_attempts_count": 1, - "_retry_stop_after_attempts_duration": 100 + "_retry_stop_after_attempts_duration": 100, } - backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), **retry_delay_args) - for (arg, val) in retry_delay_args.items(): + backend = ThriftBackend( + "foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions(), **retry_delay_args + ) + for arg, val in retry_delay_args.items(): self.assertEqual(getattr(backend, arg), val) @patch("thrift.transport.THttpClient.THttpClient") def test_retry_args_bounding(self, mock_http_client): retry_delay_test_args_and_expected_values = {} - for (k, (_, _, min, max)) in databricks.sql.thrift_backend._retry_policy.items(): + for k, (_, _, min, max) in databricks.sql.thrift_backend._retry_policy.items(): retry_delay_test_args_and_expected_values[k] = ((min - 1, min), (max + 1, max)) for i in range(2): retry_delay_args = { - k: v[i][0] - for (k, v) in retry_delay_test_args_and_expected_values.items() + k: v[i][0] for (k, v) in retry_delay_test_args_and_expected_values.items() } - backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), **retry_delay_args) + backend = ThriftBackend( + "foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions(), **retry_delay_args + ) retry_delay_expected_vals = { - k: v[i][1] - for (k, v) in retry_delay_test_args_and_expected_values.items() + k: v[i][1] for (k, v) in retry_delay_test_args_and_expected_values.items() } - for (arg, val) in retry_delay_expected_vals.items(): + for arg, val in retry_delay_expected_vals.items(): self.assertEqual(getattr(backend, arg), val) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_configuration_passthrough(self, tcli_client_class): tcli_service_instance = tcli_client_class.return_value tcli_service_instance.OpenSession.return_value = self.open_session_resp @@ -1269,21 +1557,21 @@ def test_configuration_passthrough(self, tcli_client_class): "spark.thriftserver.arrowBasedRowSet.timestampAsString": "false", "foo": "bar", "baz": "True", - "42": "42" + "42": "42", } - backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) backend.open_session(mock_config, None, None) open_session_req = tcli_client_class.return_value.OpenSession.call_args[0][0] self.assertEqual(open_session_req.configuration, expected_config) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_cant_set_timestamp_as_string_to_true(self, tcli_client_class): tcli_service_instance = tcli_client_class.return_value tcli_service_instance.OpenSession.return_value = self.open_session_resp mock_config = {"spark.thriftserver.arrowBasedRowSet.timestampAsString": True} - backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) with self.assertRaises(databricks.sql.Error) as cm: backend.open_session(mock_config, None, None) @@ -1295,19 +1583,21 @@ def _construct_open_session_with_namespace(self, can_use_multiple_cats, cat, sch status=self.okay_status, serverProtocolVersion=ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V4, canUseMultipleCatalogs=can_use_multiple_cats, - initialNamespace=ttypes.TNamespace(catalogName=cat, schemaName=schem)) + initialNamespace=ttypes.TNamespace(catalogName=cat, schemaName=schem), + ) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_initial_namespace_passthrough_to_open_session(self, tcli_client_class): tcli_service_instance = tcli_client_class.return_value - backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) initial_cat_schem_args = [("cat", None), (None, "schem"), ("cat", "schem")] for cat, schem in initial_cat_schem_args: with self.subTest(cat=cat, schem=schem): - tcli_service_instance.OpenSession.return_value = \ + tcli_service_instance.OpenSession.return_value = ( self._construct_open_session_with_namespace(True, cat, schem) + ) backend.open_session({}, cat, schem) @@ -1315,22 +1605,22 @@ def test_initial_namespace_passthrough_to_open_session(self, tcli_client_class): self.assertEqual(open_session_req.initialNamespace.catalogName, cat) self.assertEqual(open_session_req.initialNamespace.schemaName, schem) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_can_use_multiple_catalogs_is_set_in_open_session_req(self, tcli_client_class): tcli_service_instance = tcli_client_class.return_value tcli_service_instance.OpenSession.return_value = self.open_session_resp - backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) backend.open_session({}, None, None) open_session_req = tcli_client_class.return_value.OpenSession.call_args[0][0] self.assertTrue(open_session_req.canUseMultipleCatalogs) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_can_use_multiple_catalogs_is_false_fails_with_initial_catalog(self, tcli_client_class): tcli_service_instance = tcli_client_class.return_value - backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) # If the initial catalog is set, but server returns canUseMultipleCatalogs=False, we # expect failure. If the initial catalog isn't set, then canUseMultipleCatalogs=False # is fine @@ -1338,48 +1628,55 @@ def test_can_use_multiple_catalogs_is_false_fails_with_initial_catalog(self, tcl passing_ns_args = [(None, None), (None, "schem")] for cat, schem in failing_ns_args: - tcli_service_instance.OpenSession.return_value = \ + tcli_service_instance.OpenSession.return_value = ( self._construct_open_session_with_namespace(False, cat, schem) + ) with self.assertRaises(InvalidServerResponseError) as cm: backend.open_session({}, cat, schem) - self.assertIn("server does not support multiple catalogs", str(cm.exception), - "incorrect error thrown for initial namespace {}".format((cat, schem))) + self.assertIn( + "server does not support multiple catalogs", + str(cm.exception), + "incorrect error thrown for initial namespace {}".format((cat, schem)), + ) for cat, schem in passing_ns_args: - tcli_service_instance.OpenSession.return_value = \ + tcli_service_instance.OpenSession.return_value = ( self._construct_open_session_with_namespace(False, cat, schem) + ) backend.open_session({}, cat, schem) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) def test_protocol_v3_fails_if_initial_namespace_set(self, tcli_client_class): tcli_service_instance = tcli_client_class.return_value - tcli_service_instance.OpenSession.return_value = \ - ttypes.TOpenSessionResp( - status=self.okay_status, - serverProtocolVersion=ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V3, - canUseMultipleCatalogs=True, - initialNamespace=ttypes.TNamespace(catalogName="cat", schemaName="schem") - ) + tcli_service_instance.OpenSession.return_value = ttypes.TOpenSessionResp( + status=self.okay_status, + serverProtocolVersion=ttypes.TProtocolVersion.SPARK_CLI_SERVICE_PROTOCOL_V3, + canUseMultipleCatalogs=True, + initialNamespace=ttypes.TNamespace(catalogName="cat", schemaName="schem"), + ) - backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider()) + backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions()) with self.assertRaises(InvalidServerResponseError) as cm: backend.open_session({}, "cat", "schem") - self.assertIn("Setting initial namespace not supported by the DBR version", - str(cm.exception)) + self.assertIn( + "Setting initial namespace not supported by the DBR version", str(cm.exception) + ) - @patch("databricks.sql.thrift_backend.TCLIService.Client") + @patch("databricks.sql.thrift_backend.TCLIService.Client", autospec=True) @patch("databricks.sql.thrift_backend.ThriftBackend._handle_execute_response") - def test_execute_command_sets_complex_type_fields_correctly(self, mock_handle_execute_response, - tcli_service_class): + def test_execute_command_sets_complex_type_fields_correctly( + self, mock_handle_execute_response, tcli_service_class + ): tcli_service_instance = tcli_service_class.return_value # Iterate through each possible combination of native types (True, False and unset) - for (complex, timestamp, decimals) in itertools.product( - [True, False, None], [True, False, None], [True, False, None]): + for complex, timestamp, decimals in itertools.product( + [True, False, None], [True, False, None], [True, False, None] + ): complex_arg_types = {} if complex is not None: complex_arg_types["_use_arrow_native_complex_types"] = complex @@ -1388,18 +1685,26 @@ def test_execute_command_sets_complex_type_fields_correctly(self, mock_handle_ex if decimals is not None: complex_arg_types["_use_arrow_native_decimals"] = decimals - thrift_backend = ThriftBackend("foobar", 443, "path", [], auth_provider=AuthProvider(), **complex_arg_types) + thrift_backend = ThriftBackend( + "foobar", 443, "path", [], auth_provider=AuthProvider(), ssl_options=SSLOptions(), **complex_arg_types + ) thrift_backend.execute_command(Mock(), Mock(), 100, 100, Mock(), Mock()) t_execute_statement_req = tcli_service_instance.ExecuteStatement.call_args[0][0] # If the value is unset, the native type should default to True - self.assertEqual(t_execute_statement_req.useArrowNativeTypes.timestampAsArrow, - complex_arg_types.get("_use_arrow_native_timestamps", True)) - self.assertEqual(t_execute_statement_req.useArrowNativeTypes.decimalAsArrow, - complex_arg_types.get("_use_arrow_native_decimals", True)) - self.assertEqual(t_execute_statement_req.useArrowNativeTypes.complexTypesAsArrow, - complex_arg_types.get("_use_arrow_native_complex_types", True)) + self.assertEqual( + t_execute_statement_req.useArrowNativeTypes.timestampAsArrow, + complex_arg_types.get("_use_arrow_native_timestamps", True), + ) + self.assertEqual( + t_execute_statement_req.useArrowNativeTypes.decimalAsArrow, + complex_arg_types.get("_use_arrow_native_decimals", True), + ) + self.assertEqual( + t_execute_statement_req.useArrowNativeTypes.complexTypesAsArrow, + complex_arg_types.get("_use_arrow_native_complex_types", True), + ) self.assertFalse(t_execute_statement_req.useArrowNativeTypes.intervalTypesAsArrow) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main()