fix(sdk): fix gen sdk (#1115)

TencentBlueKing · Nov 19, 2024 · 9a44238 · 9a44238
1 parent 561cd58
commit 9a44238
Show file tree

Hide file tree

Showing 9 changed files with 372 additions and 34 deletions.
diff --git a/src/dashboard/Dockerfile b/src/dashboard/Dockerfile
@@ -17,6 +17,12 @@ ADD build/requirements.txt /app/requirements.txt
 RUN pip install --upgrade pip
 RUN pip install -r requirements.txt
 
+# install mvn dep
+RUN mvn dependency:get -Dartifact=com.squareup.okhttp3:okhttp:4.12.0 && \
+    mvn dependency:get -Dartifact=com.fasterxml.jackson.core:jackson-databind:2.13.4.1 && \
+    mvn dependency:get -Dartifact=org.apache.maven.plugins:maven-compiler-plugin:3.8.1 && \
+    mvn dependency:get -Dartifact=org.apache.maven.plugins:maven-jar-plugin:3.1.0
+
 ADD build /app
 
 

diff --git a/src/dashboard/apigateway/apigateway/apps/support/api_sdk/distributors/maven.py b/src/dashboard/apigateway/apigateway/apps/support/api_sdk/distributors/maven.py
@@ -92,6 +92,7 @@ def distribute(self, output_dir: str, files: List[str]) -> DistributeResult:
                 stderr=subprocess.PIPE,
                 text=True,
                 check=False,
+                timeout=120,
             )
             logger.info("stdout:%s", completed_process.stdout)
             logger.info("stderr:%s", completed_process.stderr)

diff --git a/src/dashboard/apigateway/apigateway/apps/support/api_sdk/distributors/pypi.py b/src/dashboard/apigateway/apigateway/apps/support/api_sdk/distributors/pypi.py
@@ -17,7 +17,6 @@
 #
 import logging
 import os
-import sys
 from dataclasses import dataclass, field
 from subprocess import CalledProcessError, check_call
 from typing import ClassVar, List, Optional
@@ -89,7 +88,7 @@ def distribute(self, output_dir: str, files: List[str]) -> DistributeResult:
 
         try:
             check_call(
-                [sys.executable, "setup.py", "sdist", "upload", "-r", self.repository],
+                ["twine", "upload", "dist/*", "-r", self.repository],
                 env=env,
                 cwd=source_dir,
             )

diff --git a/src/dashboard/apigateway/apigateway/apps/support/api_sdk/packagers/maven.py b/src/dashboard/apigateway/apigateway/apps/support/api_sdk/packagers/maven.py
@@ -1,26 +1,25 @@
-#  -*- coding: utf-8 -*-
-#  #
-#  TencentBlueKing is pleased to support the open source community by making
-#  蓝鲸智云 - API 网关(BlueKing - APIGateway) available.
-#  Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
-#  Licensed under the MIT License (the "License"); you may not use this file except
-#  in compliance with the License. You may obtain a copy of the License at
-#  #
-#      http://opensource.org/licenses/MIT
-#  #
-#  Unless required by applicable law or agreed to in writing, software distributed under
-#  the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
-#  either express or implied. See the License for the specific language governing permissions and
-#  limitations under the License.
-#  #
-#  We undertake not to change the open source license (MIT license) applicable
-#  to the current version of the project delivered to anyone in the future.
-#  #
+#
+# TencentBlueKing is pleased to support the open source community by making
+# 蓝鲸智云 - API 网关(BlueKing - APIGateway) available.
+# Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved.
+# Licensed under the MIT License (the "License"); you may not use this file except
+# in compliance with the License. You may obtain a copy of the License at
+#
+#     http://opensource.org/licenses/MIT
+#
+# Unless required by applicable law or agreed to in writing, software distributed under
+# the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND,
+# either express or implied. See the License for the specific language governing permissions and
+# limitations under the License.
+#
+# We undertake not to change the open source license (MIT license) applicable
+# to the current version of the project delivered to anyone in the future.
+#
 import logging
 import os
+import subprocess
 from dataclasses import dataclass
 from pathlib import Path
-from subprocess import check_output
 from typing import List
 
 from apigateway.apps.support.api_sdk.models import Packager
@@ -36,7 +35,7 @@ def pack(self, output_dir: str) -> List[str]:
         try:
             # 切换到 Maven 项目目录
             os.chdir(output_dir)
-            output = check_output(
+            result = subprocess.run(
                 [
                     "mvn",
                     "-s",
@@ -46,10 +45,14 @@ def pack(self, output_dir: str) -> List[str]:
                 ],
                 env={"HOME": output_dir},
                 cwd=output_dir,
+                capture_output=True,
+                text=True,
+                check=True,
+                timeout=120,
             )
 
-            logger.info(output.decode("utf-8"))
-
+            logger.info("stdout %s", result.stdout)
+            logger.info("stderr %s", result.stderr)
             # 查找生成的 JAR 文件
             target_dir = os.path.join(output_dir, "target")
             jar_files = [os.path.join(target_dir, f) for f in os.listdir(target_dir) if f.endswith(".jar")]
@@ -59,6 +62,9 @@ def pack(self, output_dir: str) -> List[str]:
 
             return [os.path.join(output_dir + "/target", os.path.basename(jar_file)) for jar_file in jar_files]
 
+        except subprocess.CalledProcessError as e:
+            logger.exception("Command failed with return code %s", e.returncode)
+            raise
         finally:
             # 切换回原来的工作目录
             os.chdir(original_dir)
diff --git a/src/dashboard/apigateway/apigateway/tests/apps/support/api_sdk/distributors/test_pypi.py b/src/dashboard/apigateway/apigateway/tests/apps/support/api_sdk/distributors/test_pypi.py
@@ -15,6 +15,10 @@
 # We undertake not to change the open source license (MIT license) applicable
 # to the current version of the project delivered to anyone in the future.
 #
+import os
+import subprocess
+from unittest.mock import patch
+
 import pytest
 
 from apigateway.apps.support.api_sdk.distributors.pypi import PypiSourceDistributor
@@ -57,9 +61,37 @@ def sdist(tmpdir, sdk_context):
     dist = tmpdir.join("dist")
     dist.mkdir()
 
-    source_tar = dist.join(f"{sdk_context.name}.tar.gz")
-    source_tar.write("")
-    return source_tar
+    # Create a simple package structure
+    package_dir = tmpdir.join(sdk_context.name)
+    package_dir.mkdir()
+
+    init_file = package_dir.join("__init__.py")
+    init_file.write("# Sample package")
+
+    # Create a setup.py file with necessary metadata
+    setup_py = tmpdir.join("setup.py")
+    setup_py.write(f"""\
+from setuptools import setup, find_packages
+
+setup(
+    name='{sdk_context.name}',
+    version='1.0.0',
+    packages=find_packages(),
+    description='A test Python package',
+    author='Your Name',
+    author_email='[email protected]',
+    url='https://example.com',
+)
+""")
+
+    # Build the source distribution
+    current_dir = os.getcwd()
+    os.chdir(tmpdir)
+    try:
+        subprocess.check_call([os.sys.executable, "setup.py", "sdist"])
+    finally:
+        os.chdir(current_dir)
+    return dist.join(f"{sdk_context.name}-1.0.0.tar.gz")
 
 
 @pytest.fixture
@@ -80,16 +112,14 @@ def test_pypirc(
 def test_distribute(
     output_dir,
     sdk_context,
-    python_setup_script,
     sdist,
-    python_setup_history,
     distributor: PypiSourceDistributor,
     package_searcher_result,
 ):
-    result = distributor.distribute(output_dir, [sdist])
-
-    python_setup_history = python_setup_history.read()
-    assert f"setup.py sdist upload -r {distributor.repository}" in python_setup_history
+    # Mock the check_call to prevent actual call to `twine upload`
+    with patch("subprocess.check_call") as mock_check_call:
+        mock_check_call.return_value = 0  # Simulate successful execution
+        result = distributor.distribute(output_dir, [sdist])
 
     assert sdk_context.config["python"]["is_uploaded_to_pypi"]
     assert sdk_context.config["python"]["repository"] == distributor.repository

diff --git a/src/dashboard/apigateway/requirements.txt b/src/dashboard/apigateway/requirements.txt
@@ -5,6 +5,7 @@ arrow==1.3.0 ; python_version >= "3.10" and python_version < "3.11"
 asgiref==3.8.1 ; python_version >= "3.10" and python_version < "3.11"
 async-timeout==4.0.3 ; python_version >= "3.10" and python_version < "3.11"
 attrs==24.2.0 ; python_version >= "3.10" and python_version < "3.11"
+backports-tarfile==1.2.0 ; python_version >= "3.10" and python_version < "3.11"
 beautifulsoup4==4.12.3 ; python_version >= "3.10" and python_version < "3.11"
 billiard==4.2.1 ; python_version >= "3.10" and python_version < "3.11"
 bk-crypto-python-sdk==2.0.0 ; python_version >= "3.10" and python_version < "3.11"
@@ -58,14 +59,23 @@ gunicorn==23.0.0 ; python_version >= "3.10" and python_version < "3.11"
 idna==3.10 ; python_version >= "3.10" and python_version < "3.11"
 importlib-metadata==8.4.0 ; python_version >= "3.10" and python_version < "3.11"
 inflection==0.5.1 ; python_version >= "3.10" and python_version < "3.11"
+jaraco-classes==3.4.0 ; python_version >= "3.10" and python_version < "3.11"
+jaraco-context==6.0.1 ; python_version >= "3.10" and python_version < "3.11"
+jaraco-functools==4.1.0 ; python_version >= "3.10" and python_version < "3.11"
+jeepney==0.8.0 ; python_version >= "3.10" and python_version < "3.11" and sys_platform == "linux"
 jinja2==3.1.4 ; python_version >= "3.10" and python_version < "3.11"
 jsonfield==3.1.0 ; python_version >= "3.10" and python_version < "3.11"
 jsonschema-path==0.3.3 ; python_version >= "3.10" and python_version < "3.11"
 jsonschema-specifications==2023.12.1 ; python_version >= "3.10" and python_version < "3.11"
 jsonschema==4.23.0 ; python_version >= "3.10" and python_version < "3.11"
+keyring==25.5.0 ; python_version >= "3.10" and python_version < "3.11"
 kombu==5.4.2 ; python_version >= "3.10" and python_version < "3.11"
 lazy-object-proxy==1.10.0 ; python_version >= "3.10" and python_version < "3.11"
+markdown-it-py==3.0.0 ; python_version >= "3.10" and python_version < "3.11"
 markupsafe==3.0.2 ; python_version >= "3.10" and python_version < "3.11"
+mdurl==0.1.2 ; python_version >= "3.10" and python_version < "3.11"
+more-itertools==10.5.0 ; python_version >= "3.10" and python_version < "3.11"
+nh3==0.2.18 ; python_version >= "3.10" and python_version < "3.11"
 openapi-schema-validator==0.6.2 ; python_version >= "3.10" and python_version < "3.11"
 openapi-spec-validator==0.7.1 ; python_version >= "3.10" and python_version < "3.11"
 opentelemetry-api==1.27.0 ; python_version >= "3.10" and python_version < "3.11"
@@ -91,6 +101,7 @@ opentelemetry-util-http==0.48b0 ; python_version >= "3.10" and python_version <
 packaging==24.1 ; python_version >= "3.10" and python_version < "3.11"
 pathable==0.4.3 ; python_version >= "3.10" and python_version < "3.11"
 pillow==11.0.0 ; python_version >= "3.10" and python_version < "3.11"
+pkginfo==1.10.0 ; python_version >= "3.10" and python_version < "3.11"
 portalocker==2.10.1 ; python_version >= "3.10" and python_version < "3.11"
 prance==23.6.21.0 ; python_version >= "3.10" and python_version < "3.11"
 prometheus-client==0.21.0 ; python_version >= "3.10" and python_version < "3.11"
@@ -111,16 +122,22 @@ python-editor==1.0.4 ; python_version >= "3.10" and python_version < "3.11"
 python-json-logger==2.0.7 ; python_version >= "3.10" and python_version < "3.11"
 python-redis-lock==4.0.0 ; python_version >= "3.10" and python_version < "3.11"
 pytz==2024.2 ; python_version >= "3.10" and python_version < "3.11"
+pywin32-ctypes==0.2.3 ; python_version >= "3.10" and python_version < "3.11" and sys_platform == "win32"
 pywin32==308 ; python_version >= "3.10" and python_version < "3.11" and platform_system == "Windows"
 pyyaml==6.0.2 ; python_version >= "3.10" and python_version < "3.11"
 raven==6.10.0 ; python_version >= "3.10" and python_version < "3.11"
+readme-renderer==44.0 ; python_version >= "3.10" and python_version < "3.11"
 redis==5.2.0 ; python_version >= "3.10" and python_version < "3.11"
 referencing==0.35.1 ; python_version >= "3.10" and python_version < "3.11"
+requests-toolbelt==1.0.0 ; python_version >= "3.10" and python_version < "3.11"
 requests==2.32.3 ; python_version >= "3.10" and python_version < "3.11"
 rfc3339-validator==0.1.4 ; python_version >= "3.10" and python_version < "3.11"
+rfc3986==2.0.0 ; python_version >= "3.10" and python_version < "3.11"
+rich==13.9.3 ; python_version >= "3.10" and python_version < "3.11"
 rpds-py==0.20.0 ; python_version >= "3.10" and python_version < "3.11"
 ruamel-yaml-clib==0.2.12 ; platform_python_implementation == "CPython" and python_version < "3.11" and python_version >= "3.10"
 ruamel-yaml==0.18.6 ; python_version >= "3.10" and python_version < "3.11"
+secretstorage==3.3.3 ; python_version >= "3.10" and python_version < "3.11" and sys_platform == "linux"
 setuptools==75.3.0 ; python_version >= "3.10" and python_version < "3.11"
 six==1.16.0 ; python_version >= "3.10" and python_version < "3.11"
 soupsieve==2.6 ; python_version >= "3.10" and python_version < "3.11"
@@ -129,6 +146,7 @@ tenacity==9.0.0 ; python_version >= "3.10" and python_version < "3.11"
 thrift==0.21.0 ; python_version >= "3.10" and python_version < "3.11"
 toml==0.10.2 ; python_version >= "3.10" and python_version < "3.11"
 tongsuopy-crayon==1.0.2b6 ; python_version >= "3.10" and python_version < "3.11"
+twine==5.1.1 ; python_version >= "3.10" and python_version < "3.11"
 types-python-dateutil==2.9.0.20241003 ; python_version >= "3.10" and python_version < "3.11"
 typing-extensions==4.12.2 ; python_version >= "3.10" and python_version < "3.11"
 tzdata==2024.2 ; python_version >= "3.10" and python_version < "3.11"

diff --git a/src/dashboard/apigateway/requirements_dev.txt b/src/dashboard/apigateway/requirements_dev.txt
@@ -6,6 +6,7 @@ asgiref==3.8.1 ; python_version >= "3.10" and python_version < "3.11"
 asttokens==2.4.1 ; python_version >= "3.10" and python_version < "3.11"
 async-timeout==4.0.3 ; python_version >= "3.10" and python_version < "3.11"
 attrs==24.2.0 ; python_version >= "3.10" and python_version < "3.11"
+backports-tarfile==1.2.0 ; python_version >= "3.10" and python_version < "3.11"
 beautifulsoup4==4.12.3 ; python_version >= "3.10" and python_version < "3.11"
 billiard==4.2.1 ; python_version >= "3.10" and python_version < "3.11"
 bk-crypto-python-sdk==2.0.0 ; python_version >= "3.10" and python_version < "3.11"
@@ -76,20 +77,27 @@ importlib-metadata==8.4.0 ; python_version >= "3.10" and python_version < "3.11"
 inflection==0.5.1 ; python_version >= "3.10" and python_version < "3.11"
 iniconfig==2.0.0 ; python_version >= "3.10" and python_version < "3.11"
 ipython==8.29.0 ; python_version >= "3.10" and python_version < "3.11"
+jaraco-classes==3.4.0 ; python_version >= "3.10" and python_version < "3.11"
+jaraco-context==6.0.1 ; python_version >= "3.10" and python_version < "3.11"
+jaraco-functools==4.1.0 ; python_version >= "3.10" and python_version < "3.11"
 jedi==0.19.1 ; python_version >= "3.10" and python_version < "3.11"
+jeepney==0.8.0 ; python_version >= "3.10" and python_version < "3.11" and sys_platform == "linux"
 jinja2==3.1.4 ; python_version >= "3.10" and python_version < "3.11"
 jsonfield==3.1.0 ; python_version >= "3.10" and python_version < "3.11"
 jsonschema-path==0.3.3 ; python_version >= "3.10" and python_version < "3.11"
 jsonschema-specifications==2023.12.1 ; python_version >= "3.10" and python_version < "3.11"
 jsonschema==4.23.0 ; python_version >= "3.10" and python_version < "3.11"
+keyring==25.5.0 ; python_version >= "3.10" and python_version < "3.11"
 kombu==5.4.2 ; python_version >= "3.10" and python_version < "3.11"
 lazy-object-proxy==1.10.0 ; python_version >= "3.10" and python_version < "3.11"
 markdown-it-py==3.0.0 ; python_version >= "3.10" and python_version < "3.11"
 markupsafe==3.0.2 ; python_version >= "3.10" and python_version < "3.11"
 matplotlib-inline==0.1.7 ; python_version >= "3.10" and python_version < "3.11"
 mdurl==0.1.2 ; python_version >= "3.10" and python_version < "3.11"
+more-itertools==10.5.0 ; python_version >= "3.10" and python_version < "3.11"
 mypy-extensions==1.0.0 ; python_version >= "3.10" and python_version < "3.11"
 mypy==1.13.0 ; python_version >= "3.10" and python_version < "3.11"
+nh3==0.2.18 ; python_version >= "3.10" and python_version < "3.11"
 nodeenv==1.9.1 ; python_version >= "3.10" and python_version < "3.11"
 nose==1.3.7 ; python_version >= "3.10" and python_version < "3.11"
 openapi-schema-validator==0.6.2 ; python_version >= "3.10" and python_version < "3.11"
@@ -119,6 +127,7 @@ parso==0.8.4 ; python_version >= "3.10" and python_version < "3.11"
 pathable==0.4.3 ; python_version >= "3.10" and python_version < "3.11"
 pexpect==4.9.0 ; python_version >= "3.10" and python_version < "3.11" and (sys_platform != "win32" and sys_platform != "emscripten")
 pillow==11.0.0 ; python_version >= "3.10" and python_version < "3.11"
+pkginfo==1.10.0 ; python_version >= "3.10" and python_version < "3.11"
 platformdirs==4.3.6 ; python_version >= "3.10" and python_version < "3.11"
 pluggy==1.5.0 ; python_version >= "3.10" and python_version < "3.11"
 portalocker==2.10.1 ; python_version >= "3.10" and python_version < "3.11"
@@ -154,20 +163,25 @@ python-json-logger==2.0.7 ; python_version >= "3.10" and python_version < "3.11"
 python-redis-lock==4.0.0 ; python_version >= "3.10" and python_version < "3.11"
 pytoolconfig[global]==1.3.1 ; python_version >= "3.10" and python_version < "3.11"
 pytz==2024.2 ; python_version >= "3.10" and python_version < "3.11"
+pywin32-ctypes==0.2.3 ; python_version >= "3.10" and python_version < "3.11" and sys_platform == "win32"
 pywin32==308 ; python_version >= "3.10" and python_version < "3.11" and platform_system == "Windows"
 pyyaml==6.0.2 ; python_version >= "3.10" and python_version < "3.11"
 raven==6.10.0 ; python_version >= "3.10" and python_version < "3.11"
+readme-renderer==44.0 ; python_version >= "3.10" and python_version < "3.11"
 redis==5.2.0 ; python_version >= "3.10" and python_version < "3.11"
 referencing==0.35.1 ; python_version >= "3.10" and python_version < "3.11"
+requests-toolbelt==1.0.0 ; python_version >= "3.10" and python_version < "3.11"
 requests==2.32.3 ; python_version >= "3.10" and python_version < "3.11"
 responses==0.25.3 ; python_version >= "3.10" and python_version < "3.11"
 rfc3339-validator==0.1.4 ; python_version >= "3.10" and python_version < "3.11"
+rfc3986==2.0.0 ; python_version >= "3.10" and python_version < "3.11"
 rich==13.9.3 ; python_version >= "3.10" and python_version < "3.11"
 rope==1.13.0 ; python_version >= "3.10" and python_version < "3.11"
 rpds-py==0.20.0 ; python_version >= "3.10" and python_version < "3.11"
 ruamel-yaml-clib==0.2.12 ; platform_python_implementation == "CPython" and python_version < "3.11" and python_version >= "3.10"
 ruamel-yaml==0.18.6 ; python_version >= "3.10" and python_version < "3.11"
 ruff==0.7.1 ; python_version >= "3.10" and python_version < "3.11"
+secretstorage==3.3.3 ; python_version >= "3.10" and python_version < "3.11" and sys_platform == "linux"
 setuptools==75.3.0 ; python_version >= "3.10" and python_version < "3.11"
 six==1.16.0 ; python_version >= "3.10" and python_version < "3.11"
 sortedcontainers==2.4.0 ; python_version >= "3.10" and python_version < "3.11"
@@ -181,6 +195,7 @@ tomli==2.0.2 ; python_version >= "3.10" and python_version < "3.11"
 tongsuopy-crayon==1.0.2b6 ; python_version >= "3.10" and python_version < "3.11"
 tox==4.23.2 ; python_version >= "3.10" and python_version < "3.11"
 traitlets==5.14.3 ; python_version >= "3.10" and python_version < "3.11"
+twine==5.1.1 ; python_version >= "3.10" and python_version < "3.11"
 types-cachetools==5.5.0.20240820 ; python_version >= "3.10" and python_version < "3.11"
 types-cffi==1.16.0.20240331 ; python_version >= "3.10" and python_version < "3.11"
 types-docutils==0.21.0.20241005 ; python_version >= "3.10" and python_version < "3.11"