[Python APIView] add support for pyproject.toml managed packages (#9923)

* update apiview tree parser tspconfig * update package.json * add usage decorator * add service decorator temporarily for codegen * update tspconfig * initial generated code * add global review line token * add blank lines * fix function node * handle multi and single lines * indent multi-line params extra * regen for has_prefix_space * add blank line after class * fix spacing/newlines in class * make multi line args children * print key node item type * add to create review line * add ReviewLines * fix multi-line methods output w/ ReviewLines * add diagnostic * fix tests * fix more class parsing tests * get functions tests passing * fix more tests * fix all class tests * fix dataclass tests * [TEMP] remove extra punctuation from add type test * pass apiview through node constructors * fix diagnostics test * add back multi line pylint error * fix metadata map test * remove to test txt * add context end line and parser header * add NavigateToId * update tests with apiview * add navigation display name * add context end and related to line tests * fix bug * clean up TODOs * add render classes for nav icons * update readme w/ info * re-gen from tsp * run black * black * add tsp-location * rever apiview tsp to main, move all changes to separate pr * newline at end of package lock * update version * [temp] print cwd for failing ci * add init to _generated + remove temp cwd printing * add isodate/typing-extensions to setup.py * add azure-core to setup.py * add external link token kind * test source url * black * add is context end to class and remove from single line functions * address comments * update readme * update tspconfig commit * travis comments, fix function special arg indent + remove lstrip from tests * travis comments, update tests to count relatedToLine and isContextEndLine * add relatedline/endcontext count to more tests * update apiview tree parser tspconfig * update package.json * add usage decorator * update tspconfig * initial generated code * add global review line token * add blank lines * fix function node * handle multi and single lines * indent multi-line params extra * regen for has_prefix_space * add blank line after class * fix spacing/newlines in class * make multi line args children * print key node item type * add to create review line * add ReviewLines * fix multi-line methods output w/ ReviewLines * add diagnostic * fix tests * fix more class parsing tests * get functions tests passing * fix more tests * fix all class tests * fix dataclass tests * [TEMP] remove extra punctuation from add type test * pass apiview through node constructors * fix diagnostics test * add back multi line pylint error * fix metadata map test * remove to test txt * add context end line and parser header * add NavigateToId * update tests with apiview * add navigation display name * add context end and related to line tests * fix bug * clean up TODOs * add render classes for nav icons * update readme w/ info * re-gen from tsp * run black * black * add tsp-location * rever apiview tsp to main, move all changes to separate pr * newline at end of package lock * update version * [temp] print cwd for failing ci * add init to _generated + remove temp cwd printing * add isodate/typing-extensions to setup.py * add azure-core to setup.py * add external link token kind * test source url * black * add is context end to class and remove from single line functions * address comments * update readme * update tspconfig commit * travis comments, fix function special arg indent + remove lstrip from tests * travis comments, update tests to count relatedToLine and isContextEndLine * add relatedline/endcontext count to more tests * [Python APIView] add support for pyproject.toml only packages * add pkginfo to setup.py * remove todo as provides_extra works if optional-dependencies are specified in pyproject.toml * update to get root namespace with source dir path + add tests * add tests for sdist/whl install w setup.py and pyproject * add manifest.in w/ *.md to apistubgentest * fix rebase conflict * fix test * fix test * black * changelog
Azure · Feb 27, 2025 · dc7da5f · dc7da5f
1 parent 2a36ecd
commit dc7da5f
Show file tree

Hide file tree

Showing 25 changed files with 1,086 additions and 369 deletions.
diff --git a/packages/python-packages/apistubgentest/MANIFEST.in b/packages/python-packages/apistubgentest/MANIFEST.in
@@ -0,0 +1,2 @@
+include *.md
+include LICENSE
diff --git a/packages/python-packages/apiview-stub-generator/CHANGELOG.md b/packages/python-packages/apiview-stub-generator/CHANGELOG.md
@@ -2,6 +2,7 @@
 
 ## Version 0.3.15 (Unreleased)
 Fixed issue where module-level overloads were not being parsed.
+Added support for parsing pyproject.toml-managed packages.
 
 ## Version 0.3.14 (2025-02-22)
 Update for tree token style parser.

diff --git a/packages/python-packages/apiview-stub-generator/apistub/__init__.py b/packages/python-packages/apiview-stub-generator/apistub/__init__.py
@@ -33,6 +33,8 @@ def console_entry_point():
     out_file_path = stub_generator.out_path
     # Generate JSON file name if outpath doesn't have json file name
     if not out_file_path.endswith(".json"):
-        out_file_path = os.path.join(stub_generator.out_path, "{0}_python.json".format(apiview.package_name))
+        out_file_path = os.path.join(
+            stub_generator.out_path, "{0}_python.json".format(apiview.package_name)
+        )
     with open(out_file_path, "w") as json_file:
         json_file.write(json_tokens)
diff --git a/...python-packages/apiview-stub-generator/apistub/_generated/treestyle/parser/_model_base.py b/...python-packages/apiview-stub-generator/apistub/_generated/treestyle/parser/_model_base.py
diff --git a/...ages/python-packages/apiview-stub-generator/apistub/_generated/treestyle/parser/_patch.py b/...ages/python-packages/apiview-stub-generator/apistub/_generated/treestyle/parser/_patch.py
@@ -8,7 +8,9 @@
 """
 from typing import List
 
-__all__: List[str] = []  # Add all objects you want publicly available to users at this package level
+__all__: List[str] = (
+    []
+)  # Add all objects you want publicly available to users at this package level
 
 
 def patch_sdk():

diff --git a/...hon-packages/apiview-stub-generator/apistub/_generated/treestyle/parser/_serialization.py b/...hon-packages/apiview-stub-generator/apistub/_generated/treestyle/parser/_serialization.py
diff --git a/...hon-packages/apiview-stub-generator/apistub/_generated/treestyle/parser/models/_models.py b/...hon-packages/apiview-stub-generator/apistub/_generated/treestyle/parser/models/_models.py
@@ -6,7 +6,17 @@
 # Changes may cause incorrect behavior and will be lost if the code is regenerated.
 # --------------------------------------------------------------------------
 
-from typing import Any, Dict, List, Literal, Mapping, Optional, TYPE_CHECKING, Union, overload
+from typing import (
+    Any,
+    Dict,
+    List,
+    Literal,
+    Mapping,
+    Optional,
+    TYPE_CHECKING,
+    Union,
+    overload,
+)
 
 from .. import _model_base
 from .._model_base import rest_field
@@ -60,7 +70,9 @@ def __init__(self, mapping: Mapping[str, Any]):
         :type mapping: Mapping[str, Any]
         """
 
-    def __init__(self, *args: Any, **kwargs: Any) -> None:  # pylint: disable=useless-super-delegation
+    def __init__(
+        self, *args: Any, **kwargs: Any
+    ) -> None:  # pylint: disable=useless-super-delegation
         super().__init__(*args, **kwargs)
 
 
@@ -103,18 +115,32 @@ class CodeFile(_model_base.Model):
     parser_version: str = rest_field(name="ParserVersion")
     """version of the APIview language parser used to create token file. Required."""
     language: Literal[
-        "C", "C++", "C#", "Go", "Java", "JavaScript", "Kotlin", "Python", "Swagger", "Swift", "TypeSpec"
+        "C",
+        "C++",
+        "C#",
+        "Go",
+        "Java",
+        "JavaScript",
+        "Kotlin",
+        "Python",
+        "Swagger",
+        "Swift",
+        "TypeSpec",
     ] = rest_field(name="Language")
     """Required. Is one of the following types: Literal[\"C\"], Literal[\"C++\"], Literal[\"C#\"],
      Literal[\"Go\"], Literal[\"Java\"], Literal[\"JavaScript\"], Literal[\"Kotlin\"],
      Literal[\"Python\"], Literal[\"Swagger\"], Literal[\"Swift\"], Literal[\"TypeSpec\"]"""
-    language_variant: Optional[Literal["None", "Spring", "Android"]] = rest_field(name="LanguageVariant")
+    language_variant: Optional[Literal["None", "Spring", "Android"]] = rest_field(
+        name="LanguageVariant"
+    )
     """Language variant is applicable only for java variants. Is one of the following types:
      Literal[\"None\"], Literal[\"Spring\"], Literal[\"Android\"]"""
     cross_language_package_id: Optional[str] = rest_field(name="CrossLanguagePackageId")
     review_lines: List["_models.ReviewLine"] = rest_field(name="ReviewLines")
     """Required."""
-    diagnostics: Optional[List["_models.CodeDiagnostic"]] = rest_field(name="Diagnostics")
+    diagnostics: Optional[List["_models.CodeDiagnostic"]] = rest_field(
+        name="Diagnostics"
+    )
     """Add any system generated comments. Each comment is linked to review line ID."""
     navigation: Optional[List["_models.NavigationItem"]] = rest_field(name="Navigation")
     """Navigation items are used to create a tree view in the navigation panel. Each navigation item
@@ -131,7 +157,17 @@ def __init__(
         package_version: str,
         parser_version: str,
         language: Literal[
-            "C", "C++", "C#", "Go", "Java", "JavaScript", "Kotlin", "Python", "Swagger", "Swift", "TypeSpec"
+            "C",
+            "C++",
+            "C#",
+            "Go",
+            "Java",
+            "JavaScript",
+            "Kotlin",
+            "Python",
+            "Swagger",
+            "Swift",
+            "TypeSpec",
         ],
         review_lines: List["_models.ReviewLine"],
         language_variant: Optional[Literal["None", "Spring", "Android"]] = None,
@@ -147,7 +183,9 @@ def __init__(self, mapping: Mapping[str, Any]):
         :type mapping: Mapping[str, Any]
         """
 
-    def __init__(self, *args: Any, **kwargs: Any) -> None:  # pylint: disable=useless-super-delegation
+    def __init__(
+        self, *args: Any, **kwargs: Any
+    ) -> None:  # pylint: disable=useless-super-delegation
         super().__init__(*args, **kwargs)
 
 
@@ -191,7 +229,9 @@ def __init__(self, mapping: Mapping[str, Any]):
         :type mapping: Mapping[str, Any]
         """
 
-    def __init__(self, *args: Any, **kwargs: Any) -> None:  # pylint: disable=useless-super-delegation
+    def __init__(
+        self, *args: Any, **kwargs: Any
+    ) -> None:  # pylint: disable=useless-super-delegation
         super().__init__(*args, **kwargs)
 
 
@@ -276,7 +316,9 @@ def __init__(self, mapping: Mapping[str, Any]):
         :type mapping: Mapping[str, Any]
         """
 
-    def __init__(self, *args: Any, **kwargs: Any) -> None:  # pylint: disable=useless-super-delegation
+    def __init__(
+        self, *args: Any, **kwargs: Any
+    ) -> None:  # pylint: disable=useless-super-delegation
         super().__init__(*args, **kwargs)
 
 
@@ -372,5 +414,7 @@ def __init__(self, mapping: Mapping[str, Any]):
         :type mapping: Mapping[str, Any]
         """
 
-    def __init__(self, *args: Any, **kwargs: Any) -> None:  # pylint: disable=useless-super-delegation
+    def __init__(
+        self, *args: Any, **kwargs: Any
+    ) -> None:  # pylint: disable=useless-super-delegation
         super().__init__(*args, **kwargs)
diff --git a/...thon-packages/apiview-stub-generator/apistub/_generated/treestyle/parser/models/_patch.py b/...thon-packages/apiview-stub-generator/apistub/_generated/treestyle/parser/models/_patch.py
@@ -15,7 +15,12 @@
 from apistub._node_index import NodeIndex
 from apistub._metadata_map import MetadataMap
 
-from ._models import CodeFile, ReviewToken as TokenImpl, ReviewLine as ReviewLineImpl, CodeDiagnostic as Diagnostic
+from ._models import (
+    CodeFile,
+    ReviewToken as TokenImpl,
+    ReviewLine as ReviewLineImpl,
+    CodeDiagnostic as Diagnostic,
+)
 from ._enums import TokenKind
 
 HEADER_TEXT = "# Package is parsed using apiview-stub-generator(version:{0}), Python version: {1}".format(
@@ -73,7 +78,15 @@ def get_root_path(cls):
                 path = os.path.split(path)[0]
         return None
 
-    def __init__(self, *, pkg_name="", namespace="", metadata_map=None, source_url=None, pkg_version=""):
+    def __init__(
+        self,
+        *,
+        pkg_name="",
+        namespace="",
+        metadata_map=None,
+        source_url=None,
+        pkg_version="",
+    ):
         self.metadata_map = metadata_map or MetadataMap("")
         self.review_lines: ReviewLines
         super().__init__(
@@ -94,7 +107,12 @@ def __init__(self, *, pkg_name="", namespace="", metadata_map=None, source_url=N
     def add_diagnostic(self, *, err, target_id):
         text = f"{err.message} [{err.symbol}]"
         self.diagnostics.append(
-            Diagnostic(level=err.level, text=text, help_link_uri=err.help_link, target_id=target_id)
+            Diagnostic(
+                level=err.level,
+                text=text,
+                help_link_uri=err.help_link,
+                target_id=target_id,
+            )
         )
 
     def add_navigation(self, navigation):
@@ -116,7 +134,9 @@ def generate_tokens(self):
 
 class ReviewToken(TokenImpl):
 
-    def __init__(self, *args, **kwargs) -> None:  # pylint: disable=useless-super-delegation
+    def __init__(
+        self, *args, **kwargs
+    ) -> None:  # pylint: disable=useless-super-delegation
         super().__init__(*args, **kwargs)
 
     def render(self):
@@ -273,9 +293,13 @@ def add_keyword(self, keyword, has_prefix_space=False, has_suffix_space=True):
         )
 
     def add_link(self, url, *, skip_diff=False):
-        self.add_token(ReviewToken(kind=TokenKind.EXTERNAL_URL, value=url, skip_diff=skip_diff))
+        self.add_token(
+            ReviewToken(kind=TokenKind.EXTERNAL_URL, value=url, skip_diff=skip_diff)
+        )
 
-    def add_string_literal(self, value, *, has_prefix_space=False, has_suffix_space=True):
+    def add_string_literal(
+        self, value, *, has_prefix_space=False, has_suffix_space=True
+    ):
         self.add_token(
             ReviewToken(
                 kind=TokenKind.STRING_LITERAL,
@@ -285,7 +309,9 @@ def add_string_literal(self, value, *, has_prefix_space=False, has_suffix_space=
             )
         )
 
-    def add_literal(self, value, *, has_prefix_space=False, has_suffix_space=True, skip_diff=False):
+    def add_literal(
+        self, value, *, has_prefix_space=False, has_suffix_space=True, skip_diff=False
+    ):
         self.add_token(
             ReviewToken(
                 kind=TokenKind.LITERAL,
@@ -317,7 +343,9 @@ def _add_token_for_type_name(
             token.navigate_to_id = navigate_to_id
         self.add_token(token)
 
-    def _add_type_token(self, type_name, apiview, has_prefix_space=False, has_suffix_space=True):
+    def _add_type_token(
+        self, type_name, apiview, has_prefix_space=False, has_suffix_space=True
+    ):
         # parse to get individual type name
         logging.debug("Generating tokens for type {}".format(type_name))
 
@@ -333,19 +361,27 @@ def _add_type_token(self, type_name, apiview, has_prefix_space=False, has_suffix
                 self.add_punctuation(prefix, has_suffix_space=False)
             # process parsed type name. internal or built in
             self._add_token_for_type_name(
-                parsed_type, apiview=apiview, has_prefix_space=has_prefix_space, has_suffix_space=has_suffix_space
+                parsed_type,
+                apiview=apiview,
+                has_prefix_space=has_prefix_space,
+                has_suffix_space=has_suffix_space,
             )
             postfix = type_name[index + len(parsed_type) :]
             # process remaining string in type recursively
             self._add_type_token(
-                postfix, apiview=apiview, has_prefix_space=has_prefix_space, has_suffix_space=has_suffix_space
+                postfix,
+                apiview=apiview,
+                has_prefix_space=has_prefix_space,
+                has_suffix_space=has_suffix_space,
             )
         else:
             # This is required group ending punctuations
             if type_name:  # if type name is empty, don't add punctuation
                 self.add_punctuation(type_name, has_suffix_space=False)
 
-    def add_type(self, type_name, apiview, has_prefix_space=False, has_suffix_space=True):
+    def add_type(
+        self, type_name, apiview, has_prefix_space=False, has_suffix_space=True
+    ):
         # TODO: add_type should require an ArgType or similar object so we can link *all* types
 
         # This method replace full qualified internal types to short name and generate tokens
@@ -357,12 +393,19 @@ def add_type(self, type_name, apiview, has_prefix_space=False, has_suffix_space=
         # Check if multiple types are listed with 'or' separator
         # Encode multiple types with or separator into Union
         if TYPE_OR_SEPARATOR in type_name:
-            types = [t.strip() for t in type_name.split(TYPE_OR_SEPARATOR) if t != TYPE_OR_SEPARATOR]
+            types = [
+                t.strip()
+                for t in type_name.split(TYPE_OR_SEPARATOR)
+                if t != TYPE_OR_SEPARATOR
+            ]
             # Make a Union of types if multiple types are present
             type_name = "Union[{}]".format(", ".join(types))
 
         self._add_type_token(
-            type_name, apiview=apiview, has_prefix_space=has_prefix_space, has_suffix_space=has_suffix_space
+            type_name,
+            apiview=apiview,
+            has_prefix_space=has_prefix_space,
+            has_suffix_space=has_suffix_space,
         )
 
     def render(self):

diff --git a/packages/python-packages/apiview-stub-generator/apistub/_metadata_map.py b/packages/python-packages/apiview-stub-generator/apistub/_metadata_map.py
@@ -30,7 +30,9 @@ def __init__(self, pkg_path, mapping_path=None):
             with open(mapping_path, "r") as json_file:
                 mapping = json.load(json_file)
                 self.cross_language_map = mapping.get("CrossLanguageDefinitionId", {})
-                self.cross_language_package_id = mapping.get("CrossLanguagePackageId", "")
+                self.cross_language_package_id = mapping.get(
+                    "CrossLanguagePackageId", ""
+                )
         except OSError:
             self.cross_language_map = {}
             self.cross_language_package_id = ""

diff --git a/packages/python-packages/apiview-stub-generator/apistub/_parsing_helpers.py b/packages/python-packages/apiview-stub-generator/apistub/_parsing_helpers.py
@@ -4,17 +4,17 @@
 
 if TYPE_CHECKING:
     from .nodes._module_node import ModuleNode
-    from .nodes._class_node import ClassNode   
+    from .nodes._class_node import ClassNode
 
 
 def parse_overloads(
     node: Union["ModuleNode", "ClassNode"],
     functions: List[astroid.FunctionDef],
     *,
-    is_module_level: bool = False
+    is_module_level: bool = False,
 ) -> List[FunctionNode]:
-    """ Uses AST parsing to look for @overload decorated functions
-        because inspect cannot see these. Returns a list of overloads given class or module node and function nodes to look through.
+    """Uses AST parsing to look for @overload decorated functions
+    because inspect cannot see these. Returns a list of overloads given class or module node and function nodes to look through.
     """
     overload_nodes = []
     for func in functions:
@@ -23,13 +23,24 @@ def parse_overloads(
         for ast_dec in func.decorators.nodes:
             try:
                 if ast_dec.name == "overload":
-                    overload_node = FunctionNode(node.namespace, node, node=func, is_module_level=is_module_level, apiview=node.apiview)
+                    overload_node = FunctionNode(
+                        node.namespace,
+                        node,
+                        node=func,
+                        is_module_level=is_module_level,
+                        apiview=node.apiview,
+                    )
                     overload_nodes.append(overload_node)
             except AttributeError:
                 continue
     return overload_nodes
 
-def add_overload_nodes(node: Union["ModuleNode", "ClassNode"], func_node: FunctionNode, overloads: List[FunctionNode]) -> None:
+
+def add_overload_nodes(
+    node: Union["ModuleNode", "ClassNode"],
+    func_node: FunctionNode,
+    overloads: List[FunctionNode],
+) -> None:
     """Gets overloads of the function node and appends them to the child nodes, then appends the function node."""
     func_overloads = [x for x in overloads if x.name == func_node.name]