From d42ecc782249c8a61fc3b3a907af0d4ce7a1d564 Mon Sep 17 00:00:00 2001 From: Ash Berlin-Taylor Date: Tue, 12 Nov 2024 12:09:19 +0000 Subject: [PATCH] Fix completion/linting/type checking with VSCode/pyright (#43899) Pyright (the type engine powering VSCode's python extension) doesn't treat `airflow` as a namespace package because of the `airflow/__init__.py` and it doesn't want to/can't support detecting the `__path__ = ...` method of making it an explicit namespace package, so we are left with no option but to create Yet Another Stub File. Tested by pytting `reveal_type(FAB_VERSION); reveal_type(TaskSDKDag)` inside `_upgrade_outdated_dag_access_control` in `airflow/model/dag.py` -- before this change it was reporting both as Unknown. And to continue to keep Pycharm happy we have to have the `__path__` stanzas without future annotations in all the "empty" files. Ugly, but at least it works --- Dockerfile | 2 +- Dockerfile.ci | 2 +- providers/src/airflow/__init__.py | 21 +++++++++++++++++++++ providers/src/airflow/providers/__init__.py | 11 ++--------- pyproject.toml | 1 + scripts/docker/install_airflow.sh | 2 +- task_sdk/pyproject.toml | 5 +++++ task_sdk/src/airflow/__init__.py | 21 +++++++++++++++++++++ 8 files changed, 53 insertions(+), 12 deletions(-) create mode 100644 providers/src/airflow/__init__.py create mode 100644 task_sdk/src/airflow/__init__.py diff --git a/Dockerfile b/Dockerfile index 48d53af815d67..5ca9949b0213d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -890,7 +890,7 @@ function install_airflow() { # Similarly we need _a_ file for task_sdk too mkdir -p ./task_sdk/src/airflow/sdk/ - touch ./task_sdk/src/airflow/__init__.py + touch ./task_sdk/src/airflow/sdk/__init__.py trap 'rm -f ./providers/src/airflow/providers/__init__.py ./task_sdk/src/airflow/__init__.py 2>/dev/null' EXIT diff --git a/Dockerfile.ci b/Dockerfile.ci index 5adb2d6210ab3..943270aec693a 100644 --- a/Dockerfile.ci +++ b/Dockerfile.ci @@ -660,7 +660,7 @@ function install_airflow() { # Similarly we need _a_ file for task_sdk too mkdir -p ./task_sdk/src/airflow/sdk/ - touch ./task_sdk/src/airflow/__init__.py + touch ./task_sdk/src/airflow/sdk/__init__.py trap 'rm -f ./providers/src/airflow/providers/__init__.py ./task_sdk/src/airflow/__init__.py 2>/dev/null' EXIT diff --git a/providers/src/airflow/__init__.py b/providers/src/airflow/__init__.py new file mode 100644 index 0000000000000..716fea5d47253 --- /dev/null +++ b/providers/src/airflow/__init__.py @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Pycharm needs to see this line. VSCode/pyright doesn't care about it, but this file needs to exist +# https://github.com/microsoft/pyright/issues/9439#issuecomment-2468990559 +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/providers/src/airflow/providers/__init__.py b/providers/src/airflow/providers/__init__.py index adfa8eb4b98ce..716fea5d47253 100644 --- a/providers/src/airflow/providers/__init__.py +++ b/providers/src/airflow/providers/__init__.py @@ -16,13 +16,6 @@ # specific language governing permissions and limitations # under the License. -# We do not use "from __future__ import annotations" here because it is not supported -# by Pycharm when we want to make sure all imports in airflow work from namespace packages -# Adding it automatically is excluded in pyproject.toml via I002 ruff rule exclusion - -# Make `airflow` a namespace package, supporting installing -# airflow.providers.* in different locations (i.e. one in site, and one in user -# lib.) This is required by some IDEs to resolve the import paths. -# -# Note: this file is not installed or distributed in any distribution! +# Pycharm needs to see this line. VSCode/pyright doesn't care about it, but this file needs to exist +# https://github.com/microsoft/pyright/issues/9439#issuecomment-2468990559 __path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore diff --git a/pyproject.toml b/pyproject.toml index c5361472fc187..6b23735cb3e16 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -317,6 +317,7 @@ testing = ["dev", "providers.tests", "task_sdk.tests", "tests_common", "tests"] "airflow/models/__init__.py" = ["F401", "TCH004"] "airflow/models/sqla_models.py" = ["F401"] "providers/src/airflow/providers/__init__.py" = ["I002"] +"providers/src/airflow/__init__.py" = ["I002"] # The test_python.py is needed because adding __future__.annotations breaks runtime checks that are # needed for the test to work diff --git a/scripts/docker/install_airflow.sh b/scripts/docker/install_airflow.sh index 769279c2cb0f4..2975c50c2d611 100644 --- a/scripts/docker/install_airflow.sh +++ b/scripts/docker/install_airflow.sh @@ -54,7 +54,7 @@ function install_airflow() { # Similarly we need _a_ file for task_sdk too mkdir -p ./task_sdk/src/airflow/sdk/ - touch ./task_sdk/src/airflow/__init__.py + touch ./task_sdk/src/airflow/sdk/__init__.py trap 'rm -f ./providers/src/airflow/providers/__init__.py ./task_sdk/src/airflow/__init__.py 2>/dev/null' EXIT diff --git a/task_sdk/pyproject.toml b/task_sdk/pyproject.toml index 37ea2d300ab4a..f290dfb17fdbc 100644 --- a/task_sdk/pyproject.toml +++ b/task_sdk/pyproject.toml @@ -33,6 +33,8 @@ build-backend = "hatchling.build" [tool.hatch.build.targets.wheel] packages = ["src/airflow"] +# This file only exists to make pyright/VSCode happy, don't ship it +exclude = ["src/airflow/__init__.py"] [tool.ruff] extend = "../pyproject.toml" @@ -46,6 +48,9 @@ namespace-packages = ["src/airflow"] "src/airflow/sdk/__init__.py" = ["TCH004"] +# Pycharm barfs if this "stub" file has future imports +"src/airflow/__init__.py" = ["I002"] + [tool.uv] dev-dependencies = [ "kgb>=7.1.1", diff --git a/task_sdk/src/airflow/__init__.py b/task_sdk/src/airflow/__init__.py new file mode 100644 index 0000000000000..716fea5d47253 --- /dev/null +++ b/task_sdk/src/airflow/__init__.py @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +# Pycharm needs to see this line. VSCode/pyright doesn't care about it, but this file needs to exist +# https://github.com/microsoft/pyright/issues/9439#issuecomment-2468990559 +__path__ = __import__("pkgutil").extend_path(__path__, __name__) # type: ignore