From 201afa2100bf460945b281a8a8be2232cd421444 Mon Sep 17 00:00:00 2001 From: chalice-1831 <844589474@qq.com> Date: Tue, 24 Dec 2024 21:10:01 +0800 Subject: [PATCH] =?UTF-8?q?feat:=20=E4=BE=9D=E8=B5=96=E5=8C=85=E5=8D=87?= =?UTF-8?q?=E7=BA=A7(closed=20#2160)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- apps/adapters/api/gse/v1.py | 2 +- apps/adapters/api/gse/v2.py | 2 +- apps/backend/agent/artifact_builder/base.py | 2 +- apps/backend/agent/artifact_builder/proxy.py | 2 +- apps/backend/agent/manager.py | 2 +- apps/backend/agent/solution_maker.py | 2 +- apps/backend/api/constants.py | 2 +- apps/backend/celery.py | 4 +- .../agent_new/add_or_update_hosts.py | 2 +- .../components/collections/agent_new/base.py | 2 +- .../collections/agent_new/bind_host_agent.py | 2 +- .../agent_new/check_agent_status.py | 2 +- .../agent_new/check_policy_gse_to_proxy.py | 2 +- .../agent_new/choose_access_point.py | 2 +- .../collections/agent_new/components.py | 2 +- .../collections/agent_new/configure_policy.py | 2 +- .../collections/agent_new/get_agent_status.py | 2 +- .../collections/agent_new/install.py | 2 +- .../agent_new/install_other_agent.py | 2 +- .../agent_new/push_host_identifier.py | 2 +- .../collections/agent_new/query_password.py | 2 +- .../collections/agent_new/register_host.py | 2 +- .../agent_new/unbind_host_agent.py | 2 +- .../agent_new/update_process_status.py | 2 +- .../agent_new/upgrade_to_agent_id.py | 2 +- .../components/collections/agent_new/wait.py | 2 +- apps/backend/components/collections/base.py | 2 +- .../components/collections/common/remote.py | 2 +- apps/backend/components/collections/core.py | 2 +- apps/backend/components/collections/job.py | 2 +- apps/backend/components/collections/plugin.py | 2 +- .../components/collections/subsubscription.py | 2 +- apps/backend/components/task_service.py | 4 +- apps/backend/constants.py | 2 +- apps/backend/exceptions.py | 2 +- .../backend/healthz/checker/celery_checker.py | 2 +- apps/backend/healthz/checker/checker.py | 14 +-- .../healthz/checker/supervisor_checker.py | 3 +- apps/backend/healthz/checker/utils.py | 7 +- apps/backend/healthz/define.py | 2 +- .../periodic_tasks/cache_scope_instances.py | 6 +- .../periodic_tasks/calculate_statistics.py | 4 +- .../check_zombie_sub_inst_record.py | 6 +- apps/backend/periodic_tasks/clean_sub_data.py | 4 +- .../periodic_tasks/clean_subscription_data.py | 4 +- .../collect_auto_trigger_job.py | 4 +- .../schedule_running_subscription_task.py | 8 +- .../update_subscription_instances.py | 4 +- apps/backend/plugin/handler.py | 2 +- apps/backend/plugin/manager.py | 2 +- apps/backend/plugin/serializers.py | 2 +- apps/backend/plugin/tasks.py | 2 +- apps/backend/plugin/tools.py | 2 +- apps/backend/plugin/views.py | 2 +- apps/backend/serializers/response.py | 2 +- apps/backend/serializers/views.py | 2 +- apps/backend/subscription/handler.py | 2 +- apps/backend/subscription/steps/adapter.py | 2 +- apps/backend/subscription/steps/agent.py | 2 +- apps/backend/subscription/steps/plugin.py | 2 +- apps/backend/subscription/tasks.py | 2 +- apps/backend/sync_task/constants.py | 6 +- apps/backend/sync_task/serializers.py | 4 +- .../test_check_policy_gse_to_proxy.py | 2 +- .../collections/agent_new/test_install.py | 2 +- .../agent_new/test_query_password.py | 2 +- .../tests/subscription/test_performance.py | 1 - apps/backend/urls.py | 20 ++-- apps/backend/utils/ssh.py | 2 +- apps/backend/views.py | 2 +- apps/core/concurrent/controller.py | 2 +- apps/core/concurrent/serializers.py | 2 +- apps/core/encrypt/constants.py | 2 +- apps/core/encrypt/models.py | 2 +- apps/core/encrypt/serializers.py | 2 +- apps/core/files/base.py | 2 +- apps/core/files/constants.py | 2 +- apps/core/files/exceptions.py | 2 +- apps/core/files/file_source.py | 2 +- apps/core/files/migrations/0001_initial.py | 3 +- apps/core/files/models.py | 5 +- apps/core/files/storage.py | 26 +++++ apps/core/gray/constants.py | 2 +- apps/core/gray/handlers.py | 2 +- apps/core/gray/permission.py | 2 +- apps/core/gray/serializers.py | 2 +- apps/core/gray/tools.py | 2 +- apps/core/ipchooser/constants.py | 2 +- apps/core/ipchooser/exceptions.py | 2 +- apps/core/ipchooser/handlers/host_handler.py | 2 +- apps/core/ipchooser/permission.py | 2 +- apps/core/ipchooser/query/resource.py | 2 +- apps/core/ipchooser/serializers/base.py | 2 +- apps/core/ipchooser/serializers/host_sers.py | 2 +- apps/core/ipchooser/serializers/topo_sers.py | 2 +- apps/core/ipchooser/views.py | 2 +- apps/core/remote/conns/asyncssh_impl.py | 2 +- apps/core/remote/conns/paramiko_impl.py | 2 +- apps/core/remote/exceptions.py | 2 +- apps/core/script_manage/data.py | 2 +- apps/core/tag/constants.py | 2 +- apps/core/tag/exceptions.py | 2 +- apps/core/tag/models.py | 2 +- apps/core/tag/permission.py | 2 +- apps/core/tag/serializers.py | 2 +- apps/core/tag/targets/plugin.py | 2 +- apps/core/urls.py | 8 +- apps/exceptions.py | 2 +- apps/generic.py | 2 +- apps/iam/exceptions.py | 4 +- apps/iam/handlers/actions.py | 2 +- apps/iam/handlers/permission.py | 2 +- apps/iam/handlers/resources.py | 4 +- apps/iam/serializers.py | 2 +- apps/iam/urls.py | 4 +- apps/middlewares.py | 2 +- apps/node_man/constants.py | 2 +- apps/node_man/exceptions.py | 2 +- apps/node_man/handlers/cloud.py | 2 +- apps/node_man/handlers/cmdb.py | 2 +- .../node_man/handlers/healthz/saas_healthz.py | 2 +- apps/node_man/handlers/host.py | 2 +- apps/node_man/handlers/job.py | 2 +- apps/node_man/handlers/meta.py | 2 +- apps/node_man/handlers/permission.py | 2 +- apps/node_man/handlers/plugin_v2.py | 2 +- apps/node_man/handlers/policy.py | 8 +- apps/node_man/handlers/validator.py | 2 +- .../management/commands/load_env_info.py | 2 +- apps/node_man/migrations/0001_initial.py | 49 ++++---- .../migrations/0005_auto_20200518_1209.py | 7 +- .../migrations/0006_auto_20200602_1104.py | 5 +- .../migrations/0008_accesspoint_creator.py | 5 +- .../migrations/0009_resourcewatchevent.py | 3 +- .../0015_accesspoint_port_config.py | 5 +- .../migrations/0020_auto_20200917_1132.py | 3 +- .../0022_accesspoint_proxy_package.py | 5 +- .../0024_accesspoint_bscp_config.py | 5 +- .../migrations/0027_auto_20210804_1259.py | 5 +- .../migrations/0028_auto_20210208_1752.py | 3 +- .../migrations/0029_auto_20210309_0916.py | 3 +- .../0072_subscription_instance_selector.py | 11 +- .../0083_subscription_operate_info.py | 5 +- apps/node_man/models.py | 65 ++++++----- .../add_biz_to_gse2_gray_scope.py | 4 +- .../periodic_tasks/clean_expired_info.py | 4 +- .../clean_resource_watch_event.py | 4 +- .../clean_subscription_record_info.py | 4 +- .../periodic_tasks/configuration_policy.py | 4 +- .../periodic_tasks/gse_svr_discovery.py | 4 +- .../periodic_tasks/sync_agent_status_task.py | 6 +- .../periodic_tasks/sync_all_isp_to_cmdb.py | 4 +- .../periodic_tasks/sync_cmdb_biz_topo_task.py | 8 +- .../periodic_tasks/sync_cmdb_cloud_area.py | 4 +- .../node_man/periodic_tasks/sync_cmdb_host.py | 6 +- .../periodic_tasks/sync_proc_status_task.py | 6 +- .../periodic_tasks/update_proxy_file.py | 4 +- apps/node_man/serializers/ap.py | 6 +- apps/node_man/serializers/base.py | 2 +- apps/node_man/serializers/cloud.py | 2 +- apps/node_man/serializers/cmdb.py | 2 +- apps/node_man/serializers/debug.py | 2 +- apps/node_man/serializers/host.py | 2 +- apps/node_man/serializers/host_v2.py | 2 +- apps/node_man/serializers/iam.py | 2 +- apps/node_man/serializers/install_channel.py | 2 +- apps/node_man/serializers/job.py | 2 +- apps/node_man/serializers/meta.py | 2 +- apps/node_man/serializers/plugin.py | 2 +- apps/node_man/serializers/plugin_v2.py | 2 +- apps/node_man/serializers/policy.py | 2 +- apps/node_man/serializers/response.py | 2 +- apps/node_man/serializers/sync_task.py | 6 +- apps/node_man/tests/test_handlers/test_job.py | 2 +- .../node_man/tests/test_handlers/test_meta.py | 2 +- apps/node_man/tools/host.py | 2 +- apps/node_man/tools/job.py | 2 +- apps/node_man/tools/policy.py | 2 +- apps/node_man/urls.py | 27 +++-- apps/node_man/views/__init__.py | 2 +- apps/node_man/views/ap.py | 2 +- apps/node_man/views/meta.py | 2 +- apps/node_man/views/permission.py | 2 +- apps/node_man/views/plugin_v2.py | 2 +- apps/utils/drf.py | 2 +- apps/utils/encrypt/key.py | 2 +- apps/utils/orm.py | 2 +- bin/hooks/start_cmds/celery/start-baworker | 2 +- bin/hooks/start_cmds/celery/start-bworker | 2 +- .../celery/start-common-pipeline-worker | 2 +- .../start_cmds/celery/start-common-worker | 2 +- bin/hooks/start_cmds/celery/start-dworker | 2 +- bin/hooks/start_cmds/celery/start-paworker | 2 +- bin/hooks/start_cmds/celery/start-psworker | 2 +- bin/hooks/start_cmds/celery/start-pworker | 2 +- common/api/base.py | 2 +- common/api/exception.py | 4 +- common/api/models.py | 2 +- common/api/modules/bk_node.py | 2 +- common/api/modules/cc.py | 2 +- common/api/modules/esb.py | 2 +- common/api/modules/gse.py | 2 +- common/api/modules/gse_v2.py | 2 +- common/api/modules/job.py | 2 +- common/api/modules/sops.py | 2 +- common/api/modules/yunti.py | 2 +- common/api/tasks.py | 4 +- common/context_processors.py | 2 +- config/default.py | 12 +- impacket/smbserver.py | 3 +- pipeline/apps.py | 6 +- pipeline/component_framework/models.py | 2 +- pipeline/contrib/celery_tools/___init__.py | 12 ++ pipeline/contrib/celery_tools/periodic.py | 46 ++++++++ .../contrib/external_plugins/models/base.py | 2 +- .../contrib/external_plugins/models/source.py | 16 ++- .../periodic_task/djcelery/tzcrontab.py | 14 +-- pipeline/contrib/periodic_task/models.py | 105 +++++++++++++++--- .../contrib/periodic_task/signals/__init__.py | 6 +- pipeline/contrib/periodic_task/tasks.py | 4 +- pipeline/contrib/statistics/models.py | 2 +- .../core/flow/activity/service_activity.py | 13 ++- pipeline/core/flow/io.py | 2 +- pipeline/core/flow/signals.py | 2 +- pipeline/core/signals/__init__.py | 2 +- pipeline/django_signal_valve/models.py | 2 +- pipeline/engine/admin.py | 2 +- pipeline/engine/conf/function_switch.py | 2 +- pipeline/engine/models/core.py | 16 ++- pipeline/engine/models/data.py | 2 +- pipeline/engine/models/function.py | 2 +- pipeline/engine/signals/__init__.py | 32 +++--- pipeline/engine/tasks.py | 31 +++--- pipeline/log/models.py | 2 +- pipeline/log/tasks.py | 2 +- pipeline/models.py | 2 +- pipeline/validators/connection.py | 2 +- pipeline/validators/gateway.py | 2 +- pipeline/variable_framework/models.py | 2 +- requirements.txt | 95 ++++++++-------- .../images/family_bucket/Dockerfile | 10 +- support-files/supervisord.conf.bak | 2 +- .../#etc#supervisor-bknodeman-nodeman.conf | 4 +- urls.py | 18 ++- 244 files changed, 693 insertions(+), 539 deletions(-) create mode 100644 pipeline/contrib/celery_tools/___init__.py create mode 100644 pipeline/contrib/celery_tools/periodic.py diff --git a/apps/adapters/api/gse/v1.py b/apps/adapters/api/gse/v1.py index 9e46e761ac..c2904c072d 100644 --- a/apps/adapters/api/gse/v1.py +++ b/apps/adapters/api/gse/v1.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ import typing -from collections import Mapping +from collections.abc import Mapping from apps.node_man import constants, models from apps.utils import basic diff --git a/apps/adapters/api/gse/v2.py b/apps/adapters/api/gse/v2.py index 48292d012e..9766d37c9c 100644 --- a/apps/adapters/api/gse/v2.py +++ b/apps/adapters/api/gse/v2.py @@ -10,7 +10,7 @@ """ import logging import typing -from collections import Mapping +from collections.abc import Mapping from django.conf import settings diff --git a/apps/backend/agent/artifact_builder/base.py b/apps/backend/agent/artifact_builder/base.py index 8bcb8e8c72..72e8197e27 100644 --- a/apps/backend/agent/artifact_builder/base.py +++ b/apps/backend/agent/artifact_builder/base.py @@ -17,7 +17,7 @@ import typing from django.conf import settings -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.backend import exceptions from apps.backend.agent.config_parser import GseConfigParser diff --git a/apps/backend/agent/artifact_builder/proxy.py b/apps/backend/agent/artifact_builder/proxy.py index 5223ca675a..a1fc20d5e0 100644 --- a/apps/backend/agent/artifact_builder/proxy.py +++ b/apps/backend/agent/artifact_builder/proxy.py @@ -14,7 +14,7 @@ import tarfile import typing -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.node_man import constants diff --git a/apps/backend/agent/manager.py b/apps/backend/agent/manager.py index 1fa8393d3a..46b87e7021 100644 --- a/apps/backend/agent/manager.py +++ b/apps/backend/agent/manager.py @@ -12,7 +12,7 @@ from typing import Any, Dict, List from django.conf import settings -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from apps.backend.components.collections.agent_new import components from apps.node_man.constants import NodeType diff --git a/apps/backend/agent/solution_maker.py b/apps/backend/agent/solution_maker.py index ef7d6d3eb8..97d1082888 100644 --- a/apps/backend/agent/solution_maker.py +++ b/apps/backend/agent/solution_maker.py @@ -22,7 +22,7 @@ symmetric_cipher_manager, ) from django.conf import settings -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.backend.api import constants as backend_api_constants from apps.backend.subscription.steps.agent_adapter.base import AgentSetupInfo diff --git a/apps/backend/api/constants.py b/apps/backend/api/constants.py index 267f80ad55..af936cb8d3 100644 --- a/apps/backend/api/constants.py +++ b/apps/backend/api/constants.py @@ -11,7 +11,7 @@ from django.conf import settings -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ class OS(object): diff --git a/apps/backend/celery.py b/apps/backend/celery.py index a2ccfe1e58..8834d26317 100644 --- a/apps/backend/celery.py +++ b/apps/backend/celery.py @@ -11,13 +11,13 @@ import os -from celery import Celery, platforms +from celery import Celery from django.conf import settings # http://docs.celeryproject.org/en/latest/userguide/daemonizing.html#running-the-worker-with-superuser-privileges-root # for root start celery -platforms.C_FORCE_ROOT = True +os.environ.setdefault("C_FORCE_ROOT", "true") # set the default Django settings module for the 'celery' program. os.environ.setdefault("DJANGO_SETTINGS_MODULE", "settings") diff --git a/apps/backend/components/collections/agent_new/add_or_update_hosts.py b/apps/backend/components/collections/agent_new/add_or_update_hosts.py index 6e29390be8..57c89a462b 100644 --- a/apps/backend/components/collections/agent_new/add_or_update_hosts.py +++ b/apps/backend/components/collections/agent_new/add_or_update_hosts.py @@ -16,7 +16,7 @@ from django.conf import settings from django.db import transaction from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.core.concurrent import controller from apps.node_man import constants, models, tools diff --git a/apps/backend/components/collections/agent_new/base.py b/apps/backend/components/collections/agent_new/base.py index 3c8fad0cd1..3a8711b26c 100644 --- a/apps/backend/components/collections/agent_new/base.py +++ b/apps/backend/components/collections/agent_new/base.py @@ -17,7 +17,7 @@ from typing import Any, Dict, List, Match, Optional, Set, Tuple, Union from django.conf import settings -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.backend.agent.tools import ( InstallationTools, diff --git a/apps/backend/components/collections/agent_new/bind_host_agent.py b/apps/backend/components/collections/agent_new/bind_host_agent.py index 7a05dca18e..f8841369aa 100644 --- a/apps/backend/components/collections/agent_new/bind_host_agent.py +++ b/apps/backend/components/collections/agent_new/bind_host_agent.py @@ -12,7 +12,7 @@ from typing import Any, Callable, Dict, List, Optional, Set, Tuple, Union from django.db.models import Q -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.core.concurrent import controller from apps.node_man import models diff --git a/apps/backend/components/collections/agent_new/check_agent_status.py b/apps/backend/components/collections/agent_new/check_agent_status.py index 98db68dc86..018228e45f 100644 --- a/apps/backend/components/collections/agent_new/check_agent_status.py +++ b/apps/backend/components/collections/agent_new/check_agent_status.py @@ -11,7 +11,7 @@ from typing import Set -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.node_man import constants, models diff --git a/apps/backend/components/collections/agent_new/check_policy_gse_to_proxy.py b/apps/backend/components/collections/agent_new/check_policy_gse_to_proxy.py index 38eaa44de6..6763c87870 100644 --- a/apps/backend/components/collections/agent_new/check_policy_gse_to_proxy.py +++ b/apps/backend/components/collections/agent_new/check_policy_gse_to_proxy.py @@ -11,7 +11,7 @@ from typing import Any, Dict, List, Optional, Union from django.db.models import Q -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.node_man import constants, models from env.constants import GseVersion diff --git a/apps/backend/components/collections/agent_new/choose_access_point.py b/apps/backend/components/collections/agent_new/choose_access_point.py index 12a5f81a9b..ba30a10bbc 100644 --- a/apps/backend/components/collections/agent_new/choose_access_point.py +++ b/apps/backend/components/collections/agent_new/choose_access_point.py @@ -12,7 +12,7 @@ from collections import defaultdict from typing import Any, Dict, List, Optional, Set -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.backend import constants as backend_constants from apps.backend.agent.tools import get_cloud_id__proxies_map diff --git a/apps/backend/components/collections/agent_new/components.py b/apps/backend/components/collections/agent_new/components.py index 9fb74ef433..0a55d44ea8 100644 --- a/apps/backend/components/collections/agent_new/components.py +++ b/apps/backend/components/collections/agent_new/components.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from pipeline.component_framework.component import Component diff --git a/apps/backend/components/collections/agent_new/configure_policy.py b/apps/backend/components/collections/agent_new/configure_policy.py index 70ea7e929c..1b762c658f 100644 --- a/apps/backend/components/collections/agent_new/configure_policy.py +++ b/apps/backend/components/collections/agent_new/configure_policy.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.backend.api.constants import POLLING_INTERVAL from apps.backend.components.collections.agent_new.base import AgentBaseService diff --git a/apps/backend/components/collections/agent_new/get_agent_status.py b/apps/backend/components/collections/agent_new/get_agent_status.py index 7eb4a6f960..b96794f76f 100644 --- a/apps/backend/components/collections/agent_new/get_agent_status.py +++ b/apps/backend/components/collections/agent_new/get_agent_status.py @@ -11,7 +11,7 @@ from collections import defaultdict from typing import Dict, List, Optional, Set, Union -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.backend.api.constants import POLLING_INTERVAL from apps.node_man import constants, models diff --git a/apps/backend/components/collections/agent_new/install.py b/apps/backend/components/collections/agent_new/install.py index c9314e30b7..4df224f8d3 100644 --- a/apps/backend/components/collections/agent_new/install.py +++ b/apps/backend/components/collections/agent_new/install.py @@ -22,7 +22,7 @@ from django.conf import settings from django.utils import timezone, translation -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from redis.client import Pipeline from apps.backend.agent import solution_maker diff --git a/apps/backend/components/collections/agent_new/install_other_agent.py b/apps/backend/components/collections/agent_new/install_other_agent.py index eb0a2226b6..bc14e364c4 100644 --- a/apps/backend/components/collections/agent_new/install_other_agent.py +++ b/apps/backend/components/collections/agent_new/install_other_agent.py @@ -12,7 +12,7 @@ from typing import Any, Dict, List from django.conf import settings -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.backend.api.constants import POLLING_INTERVAL from apps.core.concurrent.retry import RetryHandler diff --git a/apps/backend/components/collections/agent_new/push_host_identifier.py b/apps/backend/components/collections/agent_new/push_host_identifier.py index df227c1f34..e1c49b27db 100644 --- a/apps/backend/components/collections/agent_new/push_host_identifier.py +++ b/apps/backend/components/collections/agent_new/push_host_identifier.py @@ -10,7 +10,7 @@ """ from typing import Any, Dict, List, Optional -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.backend.api.constants import POLLING_INTERVAL from apps.core.concurrent import controller diff --git a/apps/backend/components/collections/agent_new/query_password.py b/apps/backend/components/collections/agent_new/query_password.py index 1c39d1490c..f71b544257 100644 --- a/apps/backend/components/collections/agent_new/query_password.py +++ b/apps/backend/components/collections/agent_new/query_password.py @@ -13,7 +13,7 @@ from typing import Any, Dict, List, Optional, Union from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.core.concurrent import controller from apps.node_man import constants, models diff --git a/apps/backend/components/collections/agent_new/register_host.py b/apps/backend/components/collections/agent_new/register_host.py index 8dc6ca6f2b..fae668b341 100644 --- a/apps/backend/components/collections/agent_new/register_host.py +++ b/apps/backend/components/collections/agent_new/register_host.py @@ -16,7 +16,7 @@ from django.conf import settings from django.db import transaction from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.backend.subscription import tools from apps.component.esbclient import client_v2 diff --git a/apps/backend/components/collections/agent_new/unbind_host_agent.py b/apps/backend/components/collections/agent_new/unbind_host_agent.py index 1edbb3b2d3..4feb563498 100644 --- a/apps/backend/components/collections/agent_new/unbind_host_agent.py +++ b/apps/backend/components/collections/agent_new/unbind_host_agent.py @@ -12,7 +12,7 @@ from typing import Dict, List, Optional, Set, Union from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.core.concurrent import controller from apps.node_man import models diff --git a/apps/backend/components/collections/agent_new/update_process_status.py b/apps/backend/components/collections/agent_new/update_process_status.py index d44b3faf28..b2148db8b0 100644 --- a/apps/backend/components/collections/agent_new/update_process_status.py +++ b/apps/backend/components/collections/agent_new/update_process_status.py @@ -11,7 +11,7 @@ from typing import List -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.node_man import constants from apps.node_man.models import Host, ProcessStatus diff --git a/apps/backend/components/collections/agent_new/upgrade_to_agent_id.py b/apps/backend/components/collections/agent_new/upgrade_to_agent_id.py index e1d77c795b..8da22d9bf4 100644 --- a/apps/backend/components/collections/agent_new/upgrade_to_agent_id.py +++ b/apps/backend/components/collections/agent_new/upgrade_to_agent_id.py @@ -11,7 +11,7 @@ from typing import Dict, List, Optional, Union -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from .base import AgentBaseService, AgentCommonData diff --git a/apps/backend/components/collections/agent_new/wait.py b/apps/backend/components/collections/agent_new/wait.py index 7b03be5062..4293e9b41d 100644 --- a/apps/backend/components/collections/agent_new/wait.py +++ b/apps/backend/components/collections/agent_new/wait.py @@ -11,7 +11,7 @@ import time -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from pipeline.core.flow import Service, StaticIntervalGenerator diff --git a/apps/backend/components/collections/base.py b/apps/backend/components/collections/base.py index 8cb1abf86c..b4fcf0228d 100644 --- a/apps/backend/components/collections/base.py +++ b/apps/backend/components/collections/base.py @@ -31,7 +31,7 @@ from django.db.models import Value from django.db.models.functions import Concat from django.utils import timezone -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from apps.adapters.api.gse import GseApiBaseHelper, get_gse_api_helper from apps.backend.api.constants import POLLING_TIMEOUT diff --git a/apps/backend/components/collections/common/remote.py b/apps/backend/components/collections/common/remote.py index 94c5899a23..d369ecbfc3 100644 --- a/apps/backend/components/collections/common/remote.py +++ b/apps/backend/components/collections/common/remote.py @@ -14,7 +14,7 @@ from collections import ChainMap, defaultdict from enum import Enum -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.backend import constants as backend_constants from apps.core.concurrent import controller diff --git a/apps/backend/components/collections/core.py b/apps/backend/components/collections/core.py index 67cebbda4d..3044fd3d76 100644 --- a/apps/backend/components/collections/core.py +++ b/apps/backend/components/collections/core.py @@ -14,7 +14,7 @@ from enum import Enum from typing import Any, Callable, Dict, Iterable, List, Optional, Set, Tuple, Union -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.core.concurrent import core_concurrent_constants from apps.node_man import constants, models diff --git a/apps/backend/components/collections/job.py b/apps/backend/components/collections/job.py index 5fee80cd0d..4f3d776014 100644 --- a/apps/backend/components/collections/job.py +++ b/apps/backend/components/collections/job.py @@ -19,7 +19,7 @@ import six from django.conf import settings -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.backend.api.constants import POLLING_INTERVAL from apps.backend.api.job import process_parms diff --git a/apps/backend/components/collections/plugin.py b/apps/backend/components/collections/plugin.py index db4aa1b525..8feebd0bbf 100644 --- a/apps/backend/components/collections/plugin.py +++ b/apps/backend/components/collections/plugin.py @@ -22,7 +22,7 @@ from django.db import IntegrityError from django.db.models import F, Q from django.utils import timezone -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from apps.backend.api.constants import ( GSE_RUNNING_TASK_CODE, diff --git a/apps/backend/components/collections/subsubscription.py b/apps/backend/components/collections/subsubscription.py index 54132866f0..9c5b975ab7 100644 --- a/apps/backend/components/collections/subsubscription.py +++ b/apps/backend/components/collections/subsubscription.py @@ -13,7 +13,7 @@ from collections import defaultdict from typing import Any, Dict, List, Optional, Tuple -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from apps.backend.api.constants import ( POLLING_INTERVAL, diff --git a/apps/backend/components/task_service.py b/apps/backend/components/task_service.py index 6b9298cd64..8a7ccc5336 100644 --- a/apps/backend/components/task_service.py +++ b/apps/backend/components/task_service.py @@ -8,11 +8,11 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from celery.task import task +from celery import current_app from pipeline.service import task_service -@task(ignore_result=True) +@current_app.task(ignore_result=True) def callback(*args, **kwargs): task_service.callback(*args, **kwargs) diff --git a/apps/backend/constants.py b/apps/backend/constants.py index 81e37b4439..2ea661fdb4 100644 --- a/apps/backend/constants.py +++ b/apps/backend/constants.py @@ -15,7 +15,7 @@ from typing import Dict from django.conf import settings -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.utils import basic, enum diff --git a/apps/backend/exceptions.py b/apps/backend/exceptions.py index a2e810d9f8..39cb2c93b1 100644 --- a/apps/backend/exceptions.py +++ b/apps/backend/exceptions.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.exceptions import AppBaseException diff --git a/apps/backend/healthz/checker/celery_checker.py b/apps/backend/healthz/checker/celery_checker.py index c775340fff..f2aa4a9051 100644 --- a/apps/backend/healthz/checker/celery_checker.py +++ b/apps/backend/healthz/checker/celery_checker.py @@ -14,7 +14,7 @@ import random from django.conf import settings -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from common.log import logger diff --git a/apps/backend/healthz/checker/checker.py b/apps/backend/healthz/checker/checker.py index a3952edad0..d79b1b6b97 100644 --- a/apps/backend/healthz/checker/checker.py +++ b/apps/backend/healthz/checker/checker.py @@ -18,7 +18,6 @@ from django.core import signals from django.db import connections -from six.moves import zip from apps.backend.healthz.constants import CheckerStatus from common.log import logger @@ -38,12 +37,13 @@ def __init__(self, name, checker, safe=False): def _get_func_args_decl(self, func, skip=0): args = [] kwargs = {} - argspec = inspect.getargspec(func) - iargs = iter(argspec.args[::-1]) - if argspec.defaults: - defaults = argspec.defaults[::-1] - kwargs.update((k, v) for v, k in zip(defaults, iargs)) - args.extend(iargs) + sig = inspect.signature(func) + params = sig.parameters + for name, param in params.items(): + if param.default == param.empty: + args.append(name) + else: + kwargs[name] = param.default return args[:-skip], kwargs def get_argspec(self): diff --git a/apps/backend/healthz/checker/supervisor_checker.py b/apps/backend/healthz/checker/supervisor_checker.py index 63e32be902..d6cbd847b8 100644 --- a/apps/backend/healthz/checker/supervisor_checker.py +++ b/apps/backend/healthz/checker/supervisor_checker.py @@ -13,8 +13,7 @@ from subprocess import check_output from django.conf import settings -from django.utils.translation import ugettext as _ -from six.moves import zip +from django.utils.translation import gettext as _ from common.log import logger diff --git a/apps/backend/healthz/checker/utils.py b/apps/backend/healthz/checker/utils.py index bc3ebcb1d9..b13faba21a 100644 --- a/apps/backend/healthz/checker/utils.py +++ b/apps/backend/healthz/checker/utils.py @@ -10,11 +10,8 @@ """ import re +import xmlrpc.client -# six.moves是一个虚拟命名空间，提供Py2 / 3 之间重命名模块的访问，此处 xmlrpc_client IDE报红是正常的，无需安装依赖 -# 通过 six.moves.xmlrpc_client 导入是无需考虑是是位于 py2-xmlrpclib 或者 py3-xmlrpc.client -# 参考：https://stackoverflow.com/questions/27668081/how-to-install-six-moves-xmlrpc-client -import six.moves.xmlrpc_client from django.conf import settings from supervisor.xmlrpc import SupervisorTransport @@ -41,7 +38,7 @@ def get_supervisor_client(): else: url = settings.SUPERVISOR_SERVER - return six.moves.xmlrpc_client.ServerProxy( + return xmlrpc.client.ServerProxy( "http://127.0.0.1", transport=SupervisorTransport( settings.SUPERVISOR_USERNAME, diff --git a/apps/backend/healthz/define.py b/apps/backend/healthz/define.py index a436ebc934..35f5355597 100644 --- a/apps/backend/healthz/define.py +++ b/apps/backend/healthz/define.py @@ -14,7 +14,7 @@ import json from django.conf import settings -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.backend.healthz.constants import HEALTHZ_FIELD_NAMES from config.default import CELERY_QUEUES diff --git a/apps/backend/periodic_tasks/cache_scope_instances.py b/apps/backend/periodic_tasks/cache_scope_instances.py index d4d534de65..798572dd18 100644 --- a/apps/backend/periodic_tasks/cache_scope_instances.py +++ b/apps/backend/periodic_tasks/cache_scope_instances.py @@ -10,7 +10,7 @@ """ import logging -from celery.task import periodic_task, task +from celery import current_app from django.core.cache import caches from apps.backend.subscription import constants, tools @@ -22,7 +22,7 @@ cache = caches["db"] -@task(queue="default", ignore_result=True) +@current_app.task(queue="default", ignore_result=True) def get_instances_by_scope_task(subscription_id): subscription = models.Subscription.objects.get(id=subscription_id) scope_md5 = count_md5(subscription.scope) @@ -37,7 +37,7 @@ def get_instances_by_scope_task(subscription_id): logger.info(f"[cache_subscription_scope_instances] (subscription: {subscription_id}) end.") -@periodic_task( +@current_app.task( run_every=constants.SUBSCRIPTION_UPDATE_INTERVAL, queue="backend", options={"queue": "backend"}, diff --git a/apps/backend/periodic_tasks/calculate_statistics.py b/apps/backend/periodic_tasks/calculate_statistics.py index 908252517b..f4a4213aa7 100644 --- a/apps/backend/periodic_tasks/calculate_statistics.py +++ b/apps/backend/periodic_tasks/calculate_statistics.py @@ -14,7 +14,7 @@ import logging from collections import Counter, defaultdict -from celery.task import periodic_task +from celery import current_app from django.utils import timezone from apps.backend.subscription import task_tools @@ -23,7 +23,7 @@ logger = logging.getLogger("celery") -@periodic_task(run_every=30, queue="backend", options={"queue": "backend"}, ignore_result=True) +@current_app.task(run_every=30, queue="backend", options={"queue": "backend"}, ignore_result=True) def calculate_statistics(): """ 统计任务汇总状态 diff --git a/apps/backend/periodic_tasks/check_zombie_sub_inst_record.py b/apps/backend/periodic_tasks/check_zombie_sub_inst_record.py index 8fe643186b..43698c4bec 100644 --- a/apps/backend/periodic_tasks/check_zombie_sub_inst_record.py +++ b/apps/backend/periodic_tasks/check_zombie_sub_inst_record.py @@ -12,11 +12,11 @@ import logging from datetime import timedelta -from celery.task import periodic_task +from celery import current_app from django.db.models import Value from django.db.models.functions import Concat from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.backend.subscription.constants import ( CHECK_ZOMBIE_SUB_INST_RECORD_INTERVAL, @@ -31,7 +31,7 @@ MAX_RUNNING_TIME_OF_TASK = 30 * 60 -@periodic_task( +@current_app.task( run_every=CHECK_ZOMBIE_SUB_INST_RECORD_INTERVAL, queue="backend", # 这个是用来在代码调用中指定队列的，例如： update_subscription_instances.delay() options={"queue": "backend"}, # 这个是用来celery beat调度指定队列的 diff --git a/apps/backend/periodic_tasks/clean_sub_data.py b/apps/backend/periodic_tasks/clean_sub_data.py index 86746310b6..354a30a53a 100644 --- a/apps/backend/periodic_tasks/clean_sub_data.py +++ b/apps/backend/periodic_tasks/clean_sub_data.py @@ -14,8 +14,8 @@ from dataclasses import asdict, dataclass from datetime import timedelta +from celery import current_app from celery.schedules import crontab -from celery.task import periodic_task from dacite import from_dict from django.db import connection from django.db.models import Value @@ -272,7 +272,7 @@ def clean_sub_data(config: CleanConfig): handle_sub_delete(normal_task_ids, task_id__info_map) -@periodic_task(queue="default", options={"queue": "default"}, run_every=crontab(minute="*/1")) +@current_app.task(queue="default", options={"queue": "default"}, run_every=crontab(minute="*/1")) def clean_sub_data_task(): config: CleanConfig = get_config() if not config.enable: diff --git a/apps/backend/periodic_tasks/clean_subscription_data.py b/apps/backend/periodic_tasks/clean_subscription_data.py index 6bb7e8ac62..e28c9bf3a9 100644 --- a/apps/backend/periodic_tasks/clean_subscription_data.py +++ b/apps/backend/periodic_tasks/clean_subscription_data.py @@ -10,8 +10,8 @@ """ from typing import Dict, List, Optional, Union +from celery import current_app from celery.schedules import crontab -from celery.task import periodic_task from django.db import connection from django.utils import timezone @@ -24,7 +24,7 @@ JOB_SUB_INSTANCE_MAP_TABLE = "node_man_jobsubscriptioninstancemap" -@periodic_task( +@current_app.task( queue="default", options={"queue": "default"}, run_every=crontab(minute="*/5"), diff --git a/apps/backend/periodic_tasks/collect_auto_trigger_job.py b/apps/backend/periodic_tasks/collect_auto_trigger_job.py index f90a3b3f46..24684700d3 100644 --- a/apps/backend/periodic_tasks/collect_auto_trigger_job.py +++ b/apps/backend/periodic_tasks/collect_auto_trigger_job.py @@ -12,7 +12,7 @@ import logging from collections import defaultdict -from celery.task import periodic_task +from celery import current_app from django.conf import settings from django.db import transaction from django.db.models import Q @@ -22,7 +22,7 @@ logger = logging.getLogger("celery") -@periodic_task( +@current_app.task( run_every=constants.COLLECT_AUTO_TRIGGER_JOB_INTERVAL, queue="backend", # 这个是用来在代码调用中指定队列的，例如： update_subscription_instances.delay() options={"queue": "backend"}, # 这个是用来celery beat调度指定队列的 diff --git a/apps/backend/periodic_tasks/schedule_running_subscription_task.py b/apps/backend/periodic_tasks/schedule_running_subscription_task.py index b7da48c1bc..3b6ec18c42 100644 --- a/apps/backend/periodic_tasks/schedule_running_subscription_task.py +++ b/apps/backend/periodic_tasks/schedule_running_subscription_task.py @@ -12,7 +12,7 @@ from datetime import timedelta from typing import Any, Dict, List, Set -from celery.task import periodic_task +from celery import current_app from django.db.models import QuerySet from django.utils import timezone @@ -34,7 +34,7 @@ def get_need_clean_subscription_app_code(): return app_codes -@periodic_task(run_every=constants.UPDATE_SUBSCRIPTION_TASK_INTERVAL, queue="backend", options={"queue": "backend"}) +@current_app.task(run_every=constants.UPDATE_SUBSCRIPTION_TASK_INTERVAL, queue="backend", options={"queue": "backend"}) def schedule_update_subscription(): name: str = constants.UPDATE_SUBSCRIPTION_REDIS_KEY_TPL # 取出该hashset中所有的参数 @@ -57,7 +57,7 @@ def schedule_update_subscription(): logger.info(f"update subscription with results: {results}, length -> {len(results)} ") -@periodic_task(run_every=constants.UPDATE_SUBSCRIPTION_TASK_INTERVAL, queue="backend", options={"queue": "backend"}) +@current_app.task(run_every=constants.UPDATE_SUBSCRIPTION_TASK_INTERVAL, queue="backend", options={"queue": "backend"}) def schedule_run_subscription(): name: str = constants.RUN_SUBSCRIPTION_REDIS_KEY_TPL length: int = min(REDIS_INST.llen(name), constants.MAX_RUN_SUBSCRIPTION_TASK_COUNT) @@ -82,7 +82,7 @@ def schedule_run_subscription(): logger.info(f"run subscription with results: {results}, length -> {len(results)}") -@periodic_task( +@current_app.task( run_every=constants.HANDLE_UNINSTALL_REST_SUBSCRIPTION_TASK_INTERVAL, queue="default", options={"queue": "default"}, diff --git a/apps/backend/periodic_tasks/update_subscription_instances.py b/apps/backend/periodic_tasks/update_subscription_instances.py index f1302ec96a..03f4843c1a 100644 --- a/apps/backend/periodic_tasks/update_subscription_instances.py +++ b/apps/backend/periodic_tasks/update_subscription_instances.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import logging -from celery.task import periodic_task +from celery import current_app from django.db.models import Value from apps.backend.subscription.constants import SUBSCRIPTION_UPDATE_INTERVAL @@ -16,7 +16,7 @@ logger = logging.getLogger("celery") -@periodic_task( +@current_app.task( run_every=SUBSCRIPTION_UPDATE_INTERVAL, queue="backend", # 这个是用来在代码调用中指定队列的，例如： update_subscription_instances.delay() options={"queue": "backend"}, # 这个是用来celery beat调度指定队列的 diff --git a/apps/backend/plugin/handler.py b/apps/backend/plugin/handler.py index 8e85bf9737..53d3f88706 100644 --- a/apps/backend/plugin/handler.py +++ b/apps/backend/plugin/handler.py @@ -15,7 +15,7 @@ from typing import Any, Dict, List, Optional from django.conf import settings -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from apps.backend import exceptions from apps.backend.plugin import tools diff --git a/apps/backend/plugin/manager.py b/apps/backend/plugin/manager.py index ae4e072de8..3c7e490a60 100644 --- a/apps/backend/plugin/manager.py +++ b/apps/backend/plugin/manager.py @@ -14,7 +14,7 @@ import logging from typing import List -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from apps.backend.components.collections import plugin from apps.backend.utils.pipeline_parser import PipelineParser as CustomPipelineParser diff --git a/apps/backend/plugin/serializers.py b/apps/backend/plugin/serializers.py index 0770206328..11a005bf29 100644 --- a/apps/backend/plugin/serializers.py +++ b/apps/backend/plugin/serializers.py @@ -13,7 +13,7 @@ import base64 import hashlib -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from apps.exceptions import BackendValidationError, ValidationError diff --git a/apps/backend/plugin/tasks.py b/apps/backend/plugin/tasks.py index 5c6dcb1ce1..dec6e45eca 100644 --- a/apps/backend/plugin/tasks.py +++ b/apps/backend/plugin/tasks.py @@ -15,7 +15,7 @@ import traceback from django.utils import timezone -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from apps.backend.celery import app from apps.backend.plugin import tools diff --git a/apps/backend/plugin/tools.py b/apps/backend/plugin/tools.py index d04f78da2c..c689536c87 100644 --- a/apps/backend/plugin/tools.py +++ b/apps/backend/plugin/tools.py @@ -20,7 +20,7 @@ from django.conf import settings from django.db import transaction from django.utils import timezone, translation -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from packaging import version from rest_framework import serializers from rest_framework.exceptions import ValidationError diff --git a/apps/backend/plugin/views.py b/apps/backend/plugin/views.py index a4ec175f35..66ac382607 100644 --- a/apps/backend/plugin/views.py +++ b/apps/backend/plugin/views.py @@ -29,7 +29,7 @@ from django.db.models import Q from django.http import HttpResponseForbidden, JsonResponse from django.utils.translation import get_language -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from django.views.decorators.csrf import csrf_exempt from drf_yasg.utils import swagger_auto_schema from rest_framework import mixins diff --git a/apps/backend/serializers/response.py b/apps/backend/serializers/response.py index 1ded5cbea9..e6777cb7e7 100644 --- a/apps/backend/serializers/response.py +++ b/apps/backend/serializers/response.py @@ -10,7 +10,7 @@ """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from apps.mock_data.backend_mkd.subscription import api diff --git a/apps/backend/serializers/views.py b/apps/backend/serializers/views.py index 003e9d590d..6f55faba70 100644 --- a/apps/backend/serializers/views.py +++ b/apps/backend/serializers/views.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from apps.node_man import constants diff --git a/apps/backend/subscription/handler.py b/apps/backend/subscription/handler.py index 021d1ddfbe..5a37d00164 100644 --- a/apps/backend/subscription/handler.py +++ b/apps/backend/subscription/handler.py @@ -22,7 +22,7 @@ from django.db import transaction from django.db.models import Max, Q, QuerySet, Value from django.utils.translation import get_language -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from apps.backend import constants as backend_constants from apps.backend.subscription import errors, task_tools, tasks, tools diff --git a/apps/backend/subscription/steps/adapter.py b/apps/backend/subscription/steps/adapter.py index 56e3975788..295a3514a7 100644 --- a/apps/backend/subscription/steps/adapter.py +++ b/apps/backend/subscription/steps/adapter.py @@ -15,7 +15,7 @@ from typing import Any, Dict, List, Union from django.db.models import Max, Subquery, Value -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import exceptions, serializers from apps.backend.subscription import errors diff --git a/apps/backend/subscription/steps/agent.py b/apps/backend/subscription/steps/agent.py index ec6bb466bc..7ebd90bf58 100644 --- a/apps/backend/subscription/steps/agent.py +++ b/apps/backend/subscription/steps/agent.py @@ -13,7 +13,7 @@ from typing import Any, Dict, List, Optional, Tuple, Union from django.conf import settings -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.backend import constants as backend_const from apps.backend.agent.manager import AgentManager diff --git a/apps/backend/subscription/steps/plugin.py b/apps/backend/subscription/steps/plugin.py index c7d5f6a911..10c32f231f 100644 --- a/apps/backend/subscription/steps/plugin.py +++ b/apps/backend/subscription/steps/plugin.py @@ -16,7 +16,7 @@ import six from django.db.models import QuerySet -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.adapters.api.gse import get_gse_api_helper from apps.backend import constants as backend_const diff --git a/apps/backend/subscription/tasks.py b/apps/backend/subscription/tasks.py index 8165cc4b8e..496f689e8d 100644 --- a/apps/backend/subscription/tasks.py +++ b/apps/backend/subscription/tasks.py @@ -19,7 +19,7 @@ from typing import Any, Dict, List, Optional, Set, Union from django.db.models import Value -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from apps.backend.celery import app from apps.backend.components.collections.base import ActivityType diff --git a/apps/backend/sync_task/constants.py b/apps/backend/sync_task/constants.py index f9be617d69..db73e932f3 100644 --- a/apps/backend/sync_task/constants.py +++ b/apps/backend/sync_task/constants.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ from django.conf import settings -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.utils.enum import EnhanceEnum @@ -33,7 +33,5 @@ def get_member__cache_key_map(cls): @classmethod def get_member__import_path_map(cls): - import_path_map = { - cls.SYNC_CMDB_HOST: "apps.node_man.periodic_tasks.sync_cmdb_host.sync_cmdb_host_task" - } + import_path_map = {cls.SYNC_CMDB_HOST: "apps.node_man.periodic_tasks.sync_cmdb_host.sync_cmdb_host_task"} return import_path_map diff --git a/apps/backend/sync_task/serializers.py b/apps/backend/sync_task/serializers.py index b8330ed486..2d124e72d3 100644 --- a/apps/backend/sync_task/serializers.py +++ b/apps/backend/sync_task/serializers.py @@ -8,8 +8,8 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers -from django.utils.translation import ugettext_lazy as _ from apps.backend.sync_task.handler import AsyncTaskHandler from apps.exceptions import ValidationError @@ -26,7 +26,7 @@ class CreateSyncTaskSerializer(GatewaySerializer): def validate(self, attrs): if not hasattr(AsyncTaskHandler, attrs["task_name"]): - raise ValidationError(_("不支持的任务名称: {}".format(attrs['task_name']))) + raise ValidationError(_("不支持的任务名称: {}".format(attrs["task_name"]))) return attrs diff --git a/apps/backend/tests/components/collections/agent_new/test_check_policy_gse_to_proxy.py b/apps/backend/tests/components/collections/agent_new/test_check_policy_gse_to_proxy.py index 7ce0d8d09b..56e00489a7 100644 --- a/apps/backend/tests/components/collections/agent_new/test_check_policy_gse_to_proxy.py +++ b/apps/backend/tests/components/collections/agent_new/test_check_policy_gse_to_proxy.py @@ -13,7 +13,7 @@ from typing import Any, Dict from django.conf import settings -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from mock.mock import patch from apps.backend.components.collections.agent_new import ( diff --git a/apps/backend/tests/components/collections/agent_new/test_install.py b/apps/backend/tests/components/collections/agent_new/test_install.py index bfd7b131b7..550a49042a 100644 --- a/apps/backend/tests/components/collections/agent_new/test_install.py +++ b/apps/backend/tests/components/collections/agent_new/test_install.py @@ -19,7 +19,7 @@ import mock from django.conf import settings from django.test import override_settings -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from apps.backend.agent.solution_maker import ExecutionSolution from apps.backend.agent.tools import InstallationTools, gen_commands diff --git a/apps/backend/tests/components/collections/agent_new/test_query_password.py b/apps/backend/tests/components/collections/agent_new/test_query_password.py index 69f66ffa51..3c7dc125a9 100644 --- a/apps/backend/tests/components/collections/agent_new/test_query_password.py +++ b/apps/backend/tests/components/collections/agent_new/test_query_password.py @@ -14,7 +14,7 @@ import mock from django.db.models import F -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from apps.backend.components.collections.agent_new import query_password from apps.backend.components.collections.agent_new.components import ( diff --git a/apps/backend/tests/subscription/test_performance.py b/apps/backend/tests/subscription/test_performance.py index 67ffbc442a..56fff8bf5c 100644 --- a/apps/backend/tests/subscription/test_performance.py +++ b/apps/backend/tests/subscription/test_performance.py @@ -16,7 +16,6 @@ import mock from django.test import TestCase -from six.moves import range from apps.backend.subscription.tasks import ( run_subscription_task, diff --git a/apps/backend/urls.py b/apps/backend/urls.py index 997d9e80eb..fa2ea28a3f 100644 --- a/apps/backend/urls.py +++ b/apps/backend/urls.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ from django.conf import settings -from django.conf.urls import include, url +from django.urls import include, re_path from rest_framework import routers as drf_routers from apps.backend import views @@ -19,7 +19,7 @@ from apps.backend.sync_task.views import SyncTaskViewSet urlpatterns = [ - url(r"^version/?$", views.version), + re_path(r"^version/?$", views.version), ] if settings.BK_BACKEND_CONFIG or settings.IN_TEST or settings.DEBUG: @@ -31,13 +31,13 @@ export_routers = drf_routers.DefaultRouter(trailing_slash=True) urlpatterns.extend( [ - url(r"api/", include(routers.urls)), - url(r"^package/upload/$", upload_package), - url(r"^export/download/$", export_download, name="export_download"), - url(r"^export/", include(export_routers.urls)), - url(r"^get_gse_config/", views.get_gse_config), - url(r"^report_log/", views.report_log), - url(r"^api/job_callback/", views.job_callback), - url(r"tools/download/", views.tools_download), + re_path(r"api/", include(routers.urls)), + re_path(r"^package/upload/$", upload_package), + re_path(r"^export/download/$", export_download, name="export_download"), + re_path(r"^export/", include(export_routers.urls)), + re_path(r"^get_gse_config/", views.get_gse_config), + re_path(r"^report_log/", views.report_log), + re_path(r"^api/job_callback/", views.job_callback), + re_path(r"tools/download/", views.tools_download), ] ) diff --git a/apps/backend/utils/ssh.py b/apps/backend/utils/ssh.py index 86de053edc..a8169a3bf8 100644 --- a/apps/backend/utils/ssh.py +++ b/apps/backend/utils/ssh.py @@ -17,7 +17,7 @@ import paramiko import wrapt -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from six import StringIO from apps.backend.constants import ( diff --git a/apps/backend/views.py b/apps/backend/views.py index 36c42f2bcb..92207bafde 100644 --- a/apps/backend/views.py +++ b/apps/backend/views.py @@ -18,7 +18,7 @@ from blueapps.account.decorators import login_exempt from django.conf import settings from django.http import HttpResponse, JsonResponse, StreamingHttpResponse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.views.decorators.csrf import csrf_exempt from apps.backend.constants import ( diff --git a/apps/core/concurrent/controller.py b/apps/core/concurrent/controller.py index 6311e93b17..1d26bcc65a 100644 --- a/apps/core/concurrent/controller.py +++ b/apps/core/concurrent/controller.py @@ -12,7 +12,7 @@ from typing import Any, Callable, Dict, List, Optional, Tuple, Type import wrapt -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import exceptions from apps.exceptions import ValidationError diff --git a/apps/core/concurrent/serializers.py b/apps/core/concurrent/serializers.py index 3f1c27f2bb..83f2c48e2c 100644 --- a/apps/core/concurrent/serializers.py +++ b/apps/core/concurrent/serializers.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from . import constants diff --git a/apps/core/encrypt/constants.py b/apps/core/encrypt/constants.py index f9e2dca7ea..4d6eb10d15 100644 --- a/apps/core/encrypt/constants.py +++ b/apps/core/encrypt/constants.py @@ -11,7 +11,7 @@ from enum import Enum from typing import Dict -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.utils.enum import EnhanceEnum diff --git a/apps/core/encrypt/models.py b/apps/core/encrypt/models.py index 8ae2a63890..563cf49485 100644 --- a/apps/core/encrypt/models.py +++ b/apps/core/encrypt/models.py @@ -11,7 +11,7 @@ from bkcrypto.constants import AsymmetricCipherType from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from . import constants diff --git a/apps/core/encrypt/serializers.py b/apps/core/encrypt/serializers.py index 1a9a752adb..4e0fdafad5 100644 --- a/apps/core/encrypt/serializers.py +++ b/apps/core/encrypt/serializers.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers diff --git a/apps/core/files/base.py b/apps/core/files/base.py index 7833bf423d..b596a8b97d 100644 --- a/apps/core/files/base.py +++ b/apps/core/files/base.py @@ -17,7 +17,7 @@ from django.core.exceptions import SuspiciousFileOperation from django.core.files.storage import Storage from django.utils.crypto import get_random_string -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from common.api import JobApi diff --git a/apps/core/files/constants.py b/apps/core/files/constants.py index f0f6457133..0499c32e9e 100644 --- a/apps/core/files/constants.py +++ b/apps/core/files/constants.py @@ -11,7 +11,7 @@ from enum import Enum from typing import Dict, List -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.utils.cache import class_member_cache from apps.utils.enum import EnhanceEnum diff --git a/apps/core/files/exceptions.py b/apps/core/files/exceptions.py index a2b72f5eef..863c619968 100644 --- a/apps/core/files/exceptions.py +++ b/apps/core/files/exceptions.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.core.exceptions import CoreBaseException diff --git a/apps/core/files/file_source.py b/apps/core/files/file_source.py index fd445bf450..836f15a6ed 100644 --- a/apps/core/files/file_source.py +++ b/apps/core/files/file_source.py @@ -10,7 +10,7 @@ """ from typing import Any, Dict, Optional -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.utils.md5 import count_md5 from common.api import JobApi diff --git a/apps/core/files/migrations/0001_initial.py b/apps/core/files/migrations/0001_initial.py index b5afef44dc..0832cad73e 100644 --- a/apps/core/files/migrations/0001_initial.py +++ b/apps/core/files/migrations/0001_initial.py @@ -1,6 +1,5 @@ # Generated by Django 2.2.6 on 2021-08-27 11:07 -import django_mysql.models from django.db import migrations, models @@ -21,7 +20,7 @@ class Migration(migrations.Migration): ("file_source_id", models.IntegerField(db_index=True, verbose_name="文件源ID")), ("code", models.CharField(max_length=128, verbose_name="文件源标识")), ("alias", models.CharField(max_length=128, verbose_name="文件源别名")), - ("access_params", django_mysql.models.JSONField(default=dict, verbose_name="文件源接入参数")), + ("access_params", models.JSONField(default=dict, verbose_name="文件源接入参数")), ("file_prefix", models.CharField(default="", max_length=128, verbose_name="文件源标识")), ("update_time", models.DateTimeField(auto_now=True, db_index=True, verbose_name="更新时间")), ("create_time", models.DateTimeField(auto_now_add=True, db_index=True, verbose_name="创建时间")), diff --git a/apps/core/files/models.py b/apps/core/files/models.py index 89ee686818..e262b0e3c2 100644 --- a/apps/core/files/models.py +++ b/apps/core/files/models.py @@ -9,8 +9,7 @@ specific language governing permissions and limitations under the License. """ from django.db import models -from django.utils.translation import ugettext_lazy as _ -from django_mysql.models import JSONField +from django.utils.translation import gettext_lazy as _ from apps.utils.cache import class_member_cache @@ -40,7 +39,7 @@ class BKJobFileSource(models.Model): code = models.CharField(_("文件源标识"), max_length=128) alias = models.CharField(_("文件源别名"), max_length=128) - access_params = JSONField(_("文件源接入参数"), default=dict) + access_params = models.JSONField(_("文件源接入参数"), default=dict) file_prefix = models.CharField(_("文件源标识"), max_length=128, default="") update_time = models.DateTimeField(_("更新时间"), auto_now=True, db_index=True) diff --git a/apps/core/files/storage.py b/apps/core/files/storage.py index 9c5fb57a36..70aa73bac6 100644 --- a/apps/core/files/storage.py +++ b/apps/core/files/storage.py @@ -13,6 +13,7 @@ from bkstorages.backends import bkrepo from django.conf import settings +from django.core.files import File from django.core.files.storage import FileSystemStorage, Storage, get_storage_class from django.utils.deconstruct import deconstructible from django.utils.functional import cached_property @@ -80,6 +81,21 @@ def get_file_md5(self, file_name: str) -> str: file_md5 = file_metadata["X-Checksum-Md5"] return file_md5 + def save(self, name, content, max_length=None): + # django3.2 之后存储文件名不能为绝对路径 此处采用兼容方式 + if name is None: + name = content.name + if not hasattr(content, "chunks"): + content = File(content, name) + name = self.get_available_name(name, max_length=max_length) + name = self._save(name, content) + return name + + def _save(self, name, content): + storage_path = super()._save(name, content) + # bkstorage == 2.0.0 后如果 name 以 / 开头 _save会将其抛弃，需要手动拼接 + return f"/{storage_path}" if name.startswith("/") else storage_path + def _handle_file_source_list( self, file_source_list: List[Dict[str, Any]], extra_transfer_file_params: Dict[str, Any] ) -> List[Dict[str, Any]]: @@ -158,6 +174,16 @@ def location(self): """路径指向 / ，重写前路径指向「项目根目录」""" return self.base_location + def save(self, name, content, max_length=None): + # django3.2 之后存储文件名不能为绝对路径 此处采用兼容方式 + if name is None: + name = content.name + if not hasattr(content, "chunks"): + content = File(content, name) + name = self.get_available_name(name, max_length=max_length) + name = self._save(name, content) + return name + def _save(self, name, content): # 如果允许覆盖，保存前删除文件 if self.file_overwrite: diff --git a/apps/core/gray/constants.py b/apps/core/gray/constants.py index add78a4fa0..a6d4e44427 100644 --- a/apps/core/gray/constants.py +++ b/apps/core/gray/constants.py @@ -11,7 +11,7 @@ # from enum import Enum # from typing import Dict # -# from django.utils.translation import ugettext_lazy as _ +# from django.utils.translation import gettext_lazy as _ # # from apps.utils.enum import EnhanceEnum diff --git a/apps/core/gray/handlers.py b/apps/core/gray/handlers.py index cbbfdd30c7..10165b40b3 100644 --- a/apps/core/gray/handlers.py +++ b/apps/core/gray/handlers.py @@ -15,7 +15,7 @@ from django.db.models import F, QuerySet from django.db.transaction import atomic from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.adapters.api.gse import get_gse_api_helper from apps.core.ipchooser.constants import CommonEnum diff --git a/apps/core/gray/permission.py b/apps/core/gray/permission.py index b50ce92eff..0f0798a5ab 100644 --- a/apps/core/gray/permission.py +++ b/apps/core/gray/permission.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import permissions from apps.node_man.handlers.iam import IamHandler diff --git a/apps/core/gray/serializers.py b/apps/core/gray/serializers.py index 97f48663d1..bab8d6ebca 100644 --- a/apps/core/gray/serializers.py +++ b/apps/core/gray/serializers.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers diff --git a/apps/core/gray/tools.py b/apps/core/gray/tools.py index 095cd29257..20d03a1106 100644 --- a/apps/core/gray/tools.py +++ b/apps/core/gray/tools.py @@ -10,7 +10,7 @@ """ import typing -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.core.concurrent.cache import FuncCacheDecorator from apps.exceptions import ApiError diff --git a/apps/core/ipchooser/constants.py b/apps/core/ipchooser/constants.py index 1d2d85b192..b843f987d4 100644 --- a/apps/core/ipchooser/constants.py +++ b/apps/core/ipchooser/constants.py @@ -11,7 +11,7 @@ from enum import Enum from typing import Dict -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.utils.enum import EnhanceEnum diff --git a/apps/core/ipchooser/exceptions.py b/apps/core/ipchooser/exceptions.py index 4057808136..648732701c 100644 --- a/apps/core/ipchooser/exceptions.py +++ b/apps/core/ipchooser/exceptions.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.core.exceptions import CoreBaseException diff --git a/apps/core/ipchooser/handlers/host_handler.py b/apps/core/ipchooser/handlers/host_handler.py index a371a569f2..798193f9c9 100644 --- a/apps/core/ipchooser/handlers/host_handler.py +++ b/apps/core/ipchooser/handlers/host_handler.py @@ -11,7 +11,7 @@ import typing from collections import defaultdict -from django_mysql.models import QuerySet +from django.db.models import QuerySet from apps.core.concurrent import controller from apps.core.ipchooser.tools.host_tool import HostTool diff --git a/apps/core/ipchooser/permission.py b/apps/core/ipchooser/permission.py index faefd0b90d..7bdce3fba6 100644 --- a/apps/core/ipchooser/permission.py +++ b/apps/core/ipchooser/permission.py @@ -11,7 +11,7 @@ import typing from django.http import QueryDict -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.iam import Permission from apps.iam.exceptions import PermissionDeniedError diff --git a/apps/core/ipchooser/query/resource.py b/apps/core/ipchooser/query/resource.py index 84e1b517f0..2c2ad3b326 100644 --- a/apps/core/ipchooser/query/resource.py +++ b/apps/core/ipchooser/query/resource.py @@ -12,7 +12,7 @@ import typing from django.conf import settings -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.core.concurrent import controller from apps.node_man import constants as node_man_constants diff --git a/apps/core/ipchooser/serializers/base.py b/apps/core/ipchooser/serializers/base.py index f3c75791b9..ead5dbfe3b 100644 --- a/apps/core/ipchooser/serializers/base.py +++ b/apps/core/ipchooser/serializers/base.py @@ -10,7 +10,7 @@ """ import typing -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from apps.node_man import constants as node_man_constants diff --git a/apps/core/ipchooser/serializers/host_sers.py b/apps/core/ipchooser/serializers/host_sers.py index 4f83b6afc0..e5d49f6b96 100644 --- a/apps/core/ipchooser/serializers/host_sers.py +++ b/apps/core/ipchooser/serializers/host_sers.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from ..tests import mock_data diff --git a/apps/core/ipchooser/serializers/topo_sers.py b/apps/core/ipchooser/serializers/topo_sers.py index e1c02f13fe..26a23519d4 100644 --- a/apps/core/ipchooser/serializers/topo_sers.py +++ b/apps/core/ipchooser/serializers/topo_sers.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from ..tests import mock_data diff --git a/apps/core/ipchooser/views.py b/apps/core/ipchooser/views.py index 0e08558e08..d172953eef 100644 --- a/apps/core/ipchooser/views.py +++ b/apps/core/ipchooser/views.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from drf_yasg.utils import swagger_auto_schema from rest_framework import status from rest_framework.decorators import action diff --git a/apps/core/remote/conns/asyncssh_impl.py b/apps/core/remote/conns/asyncssh_impl.py index 6874f1a5fe..58c6b8c697 100644 --- a/apps/core/remote/conns/asyncssh_impl.py +++ b/apps/core/remote/conns/asyncssh_impl.py @@ -13,7 +13,7 @@ from concurrent import futures import asyncssh -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from .. import constants, exceptions from ..clients import file diff --git a/apps/core/remote/conns/paramiko_impl.py b/apps/core/remote/conns/paramiko_impl.py index 8ec2b9a057..7cb0f4cb8f 100644 --- a/apps/core/remote/conns/paramiko_impl.py +++ b/apps/core/remote/conns/paramiko_impl.py @@ -13,7 +13,7 @@ import typing import paramiko -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from six import StringIO from .. import exceptions diff --git a/apps/core/remote/exceptions.py b/apps/core/remote/exceptions.py index 872ef8b7f4..05ebcb25ac 100644 --- a/apps/core/remote/exceptions.py +++ b/apps/core/remote/exceptions.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.core.exceptions import CoreBaseException diff --git a/apps/core/script_manage/data.py b/apps/core/script_manage/data.py index d788ff6d58..61246b8caf 100644 --- a/apps/core/script_manage/data.py +++ b/apps/core/script_manage/data.py @@ -11,7 +11,7 @@ import typing -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ import env from apps.node_man import constants as node_man_constants diff --git a/apps/core/tag/constants.py b/apps/core/tag/constants.py index 8b0c27614a..32bcdca226 100644 --- a/apps/core/tag/constants.py +++ b/apps/core/tag/constants.py @@ -11,7 +11,7 @@ from enum import Enum from typing import Dict -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.node_man.constants import GsePackageCode from apps.utils.enum import EnhanceEnum diff --git a/apps/core/tag/exceptions.py b/apps/core/tag/exceptions.py index 26227c56f7..e9f5d38abd 100644 --- a/apps/core/tag/exceptions.py +++ b/apps/core/tag/exceptions.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.core.exceptions import CoreBaseException diff --git a/apps/core/tag/models.py b/apps/core/tag/models.py index 1e61e7b423..a046f221e4 100644 --- a/apps/core/tag/models.py +++ b/apps/core/tag/models.py @@ -10,7 +10,7 @@ """ from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.utils import orm diff --git a/apps/core/tag/permission.py b/apps/core/tag/permission.py index 991e49a277..9464e51d7b 100644 --- a/apps/core/tag/permission.py +++ b/apps/core/tag/permission.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ from django.conf import settings -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import permissions from apps.node_man.constants import IamActionType diff --git a/apps/core/tag/serializers.py b/apps/core/tag/serializers.py index c2464efb36..3818555dd8 100644 --- a/apps/core/tag/serializers.py +++ b/apps/core/tag/serializers.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from . import constants, exceptions, models diff --git a/apps/core/tag/targets/plugin.py b/apps/core/tag/targets/plugin.py index 03e445df4e..23b1810df2 100644 --- a/apps/core/tag/targets/plugin.py +++ b/apps/core/tag/targets/plugin.py @@ -15,7 +15,7 @@ from collections import defaultdict from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from packaging import version from apps.node_man import models as node_man_models diff --git a/apps/core/urls.py b/apps/core/urls.py index a20958f5da..85a31a86b5 100644 --- a/apps/core/urls.py +++ b/apps/core/urls.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from django.conf.urls import include, url +from django.urls import include, re_path from rest_framework import routers from .encrypt import views as encrypt_views @@ -20,9 +20,7 @@ router.register( encrypt_views.RSAViewSet.URL_BASE_NAME, encrypt_views.RSAViewSet, basename=encrypt_views.RSAViewSet.URL_BASE_NAME ) -router.register( - tag_views.TagViewSet.URL_BASE_NAME, tag_views.TagViewSet, basename=encrypt_views.RSAViewSet.URL_BASE_NAME -) +router.register(tag_views.TagViewSet.URL_BASE_NAME, tag_views.TagViewSet, basename=tag_views.TagViewSet.URL_BASE_NAME) router.register( tag_views.TagChangeRecordViewSet.URL_BASE_NAME, tag_views.TagChangeRecordViewSet, @@ -49,5 +47,5 @@ urlpatterns = [ - url(r"api/", include(router.urls)), + re_path(r"api/", include(router.urls)), ] diff --git a/apps/exceptions.py b/apps/exceptions.py index 1be22bd4b1..65b5ac428d 100644 --- a/apps/exceptions.py +++ b/apps/exceptions.py @@ -10,7 +10,7 @@ """ import typing -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ class ErrorCode(object): diff --git a/apps/generic.py b/apps/generic.py index d8e3181d27..89723af573 100644 --- a/apps/generic.py +++ b/apps/generic.py @@ -12,7 +12,7 @@ from django.conf import settings from django.http import Http404, JsonResponse -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from rest_framework import exceptions, filters, status from rest_framework.authentication import BasicAuthentication from rest_framework.response import Response diff --git a/apps/iam/exceptions.py b/apps/iam/exceptions.py index 75f98b2187..25c9a4258f 100644 --- a/apps/iam/exceptions.py +++ b/apps/iam/exceptions.py @@ -9,8 +9,8 @@ See the License for the specific language governing permissions and limitations under the License. """ from django.conf import settings -from django.utils.translation import ugettext as _ -from django.utils.translation import ugettext_lazy as _lazy +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy as _lazy from apps.exceptions import AppBaseException, ErrorCode from apps.node_man import constants diff --git a/apps/iam/handlers/actions.py b/apps/iam/handlers/actions.py index d3d43ea01f..43d58c3fe6 100644 --- a/apps/iam/handlers/actions.py +++ b/apps/iam/handlers/actions.py @@ -11,7 +11,7 @@ import json from typing import Dict, List, Union -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from iam import Action from apps.iam.exceptions import ActionNotExistError diff --git a/apps/iam/handlers/permission.py b/apps/iam/handlers/permission.py index 3be4815cd0..83830aea60 100644 --- a/apps/iam/handlers/permission.py +++ b/apps/iam/handlers/permission.py @@ -11,7 +11,7 @@ from typing import Dict, List, Union from django.conf import settings -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from iam import IAM, DummyIAM, MultiActionRequest, Request, Resource, Subject from iam.apply.models import ( ActionWithoutResources, diff --git a/apps/iam/handlers/resources.py b/apps/iam/handlers/resources.py index 67a20e8cff..e99828a586 100644 --- a/apps/iam/handlers/resources.py +++ b/apps/iam/handlers/resources.py @@ -14,8 +14,8 @@ from typing import Dict, List, Set, Union from django.conf import settings -from django.utils.translation import ugettext as _ -from django.utils.translation import ugettext_lazy as _lazy +from django.utils.translation import gettext as _ +from django.utils.translation import gettext_lazy as _lazy from iam import Resource from apps.iam.exceptions import ResourceNotExistError diff --git a/apps/iam/serializers.py b/apps/iam/serializers.py index 77a67a46bb..eac8bb0154 100644 --- a/apps/iam/serializers.py +++ b/apps/iam/serializers.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers diff --git a/apps/iam/urls.py b/apps/iam/urls.py index f0938c73ca..e1ce26b95b 100644 --- a/apps/iam/urls.py +++ b/apps/iam/urls.py @@ -10,7 +10,7 @@ """ from blueapps.account.decorators import login_exempt from django.conf import settings -from django.conf.urls import include, url +from django.urls import include, re_path from iam.contrib.django.dispatcher import DjangoBasicResourceApiDispatcher from rest_framework import routers @@ -20,4 +20,4 @@ router = routers.DefaultRouter(trailing_slash=True) -urlpatterns = [url(r"^", include(router.urls)), url(r"^resource/$", dispatcher.as_view([login_exempt]))] +urlpatterns = [re_path(r"^", include(router.urls)), re_path(r"^resource/$", dispatcher.as_view([login_exempt]))] diff --git a/apps/middlewares.py b/apps/middlewares.py index 426e3dd6b9..4d868c1074 100644 --- a/apps/middlewares.py +++ b/apps/middlewares.py @@ -36,7 +36,7 @@ from django.middleware.locale import LocaleMiddleware as GenericLocaleMiddleware from django.utils import translation from django.utils.deprecation import MiddlewareMixin -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from apps.exceptions import AppBaseException, BkJwtVerifyFailException from apps.utils.local import activate_request diff --git a/apps/node_man/constants.py b/apps/node_man/constants.py index f15d11c3c9..0131fb7fa0 100644 --- a/apps/node_man/constants.py +++ b/apps/node_man/constants.py @@ -19,7 +19,7 @@ from typing import Any, Dict, List, Union from django.conf import settings -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.backend import constants as backend_const from apps.utils.basic import ( diff --git a/apps/node_man/exceptions.py b/apps/node_man/exceptions.py index d34d20aad3..a9efe62276 100644 --- a/apps/node_man/exceptions.py +++ b/apps/node_man/exceptions.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.exceptions import AppBaseException diff --git a/apps/node_man/handlers/cloud.py b/apps/node_man/handlers/cloud.py index 3fcc82f7a2..420e4f402d 100644 --- a/apps/node_man/handlers/cloud.py +++ b/apps/node_man/handlers/cloud.py @@ -11,7 +11,7 @@ from django.conf import settings from django.db.models.aggregates import Count from django.db.transaction import atomic -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.exceptions import ValidationError from apps.node_man import constants as const diff --git a/apps/node_man/handlers/cmdb.py b/apps/node_man/handlers/cmdb.py index 3da5b5e31b..59b487900d 100644 --- a/apps/node_man/handlers/cmdb.py +++ b/apps/node_man/handlers/cmdb.py @@ -16,7 +16,7 @@ from blueapps.account.models import User from django.conf import settings from django.core.cache import cache -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from apps.component.esbclient import client_v2 from apps.exceptions import ComponentCallError diff --git a/apps/node_man/handlers/healthz/saas_healthz.py b/apps/node_man/handlers/healthz/saas_healthz.py index 0e9e479b00..4a9b92301b 100644 --- a/apps/node_man/handlers/healthz/saas_healthz.py +++ b/apps/node_man/handlers/healthz/saas_healthz.py @@ -15,7 +15,7 @@ import time import six -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from apps.component.esbclient import client_v2 from apps.node_man.handlers.healthz.constants import CheckerStatus diff --git a/apps/node_man/handlers/host.py b/apps/node_man/handlers/host.py index 8f357fec5d..33fbb7713a 100644 --- a/apps/node_man/handlers/host.py +++ b/apps/node_man/handlers/host.py @@ -13,7 +13,7 @@ from django.db.models import Count, Q from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.core.ipchooser import core_ipchooser_constants from apps.core.ipchooser.tools.base import HostQuerySqlHelper diff --git a/apps/node_man/handlers/job.py b/apps/node_man/handlers/job.py index 92b297603d..5d147b2e5d 100644 --- a/apps/node_man/handlers/job.py +++ b/apps/node_man/handlers/job.py @@ -20,7 +20,7 @@ from django.db.models import Q from django.utils import timezone from django.utils.translation import get_language -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.backend.subscription.errors import SubscriptionTaskNotReadyError from apps.core.concurrent.retry import RetryHandler diff --git a/apps/node_man/handlers/meta.py b/apps/node_man/handlers/meta.py index d6447ee877..3d726d3040 100644 --- a/apps/node_man/handlers/meta.py +++ b/apps/node_man/handlers/meta.py @@ -14,7 +14,7 @@ from django.conf import settings from django.db import connection -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from apps.core.concurrent.cache import FuncCacheDecorator from apps.node_man import constants, models, tools diff --git a/apps/node_man/handlers/permission.py b/apps/node_man/handlers/permission.py index f4d98341e9..58e895dfc7 100644 --- a/apps/node_man/handlers/permission.py +++ b/apps/node_man/handlers/permission.py @@ -11,7 +11,7 @@ from typing import Dict, List, Optional, Set, Union from django.conf import settings -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import permissions from apps.backend.sync_task.constants import SyncTaskType diff --git a/apps/node_man/handlers/plugin_v2.py b/apps/node_man/handlers/plugin_v2.py index 983136b839..7bddd6eec1 100644 --- a/apps/node_man/handlers/plugin_v2.py +++ b/apps/node_man/handlers/plugin_v2.py @@ -19,7 +19,7 @@ from django.core.cache import cache from django.core.files.uploadedfile import InMemoryUploadedFile from django.db.models import Count -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.component.esbclient import client_v2 from apps.core.files import core_files_constants diff --git a/apps/node_man/handlers/policy.py b/apps/node_man/handlers/policy.py index 73217e6353..fb90a65c9d 100644 --- a/apps/node_man/handlers/policy.py +++ b/apps/node_man/handlers/policy.py @@ -20,7 +20,7 @@ from django.conf import settings from django.db.models import Q from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from packaging import version from apps.backend.constants import PluginMigrateType @@ -136,9 +136,9 @@ def search_deploy_policy(query_params: Dict[str, Any]) -> Dict[str, Any]: all_policy_ids = [policy["id"] for policy in all_policies] # 查询每个策略下最新的任务 - sub_task_infos = models.SubscriptionTask.objects.filter( - subscription_id__in=all_policy_ids - ).values("id", "subscription_id") + sub_task_infos = models.SubscriptionTask.objects.filter(subscription_id__in=all_policy_ids).values( + "id", "subscription_id" + ) max_sub_task_id_list, task_ids_gby_sub_id = [], defaultdict(list) for sub_task_dict in sub_task_infos: diff --git a/apps/node_man/handlers/validator.py b/apps/node_man/handlers/validator.py index 88d4ac39ad..219e48e5b2 100644 --- a/apps/node_man/handlers/validator.py +++ b/apps/node_man/handlers/validator.py @@ -13,7 +13,7 @@ from django.conf import settings from django.db.models.aggregates import Count -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.adapters.api.gse import get_gse_api_helper from apps.backend.components.collections.base import DBHelperMixin diff --git a/apps/node_man/management/commands/load_env_info.py b/apps/node_man/management/commands/load_env_info.py index 0cc1fc70e2..b117590dad 100644 --- a/apps/node_man/management/commands/load_env_info.py +++ b/apps/node_man/management/commands/load_env_info.py @@ -17,7 +17,7 @@ from django.core.management.base import BaseCommand from django.db import transaction -from MySQLdb.connections import Connection +from pymysql import Connection from apps.node_man import constants, models from common.log import logger diff --git a/apps/node_man/migrations/0001_initial.py b/apps/node_man/migrations/0001_initial.py index a6e6ce9e12..63a8a17d85 100644 --- a/apps/node_man/migrations/0001_initial.py +++ b/apps/node_man/migrations/0001_initial.py @@ -9,7 +9,6 @@ specific language governing permissions and limitations under the License. """ import bkcrypto.contrib.django.fields -import django_mysql.models from django.db import migrations, models import apps.node_man.models @@ -32,8 +31,8 @@ class Migration(migrations.Migration): ("ap_type", models.CharField(default="user", max_length=255, verbose_name="接入点类型")), ("region_id", models.CharField(default="", max_length=255, verbose_name="区域id")), ("city_id", models.CharField(default="", max_length=255, verbose_name="城市id")), - ("servers", django_mysql.models.JSONField(default=dict, verbose_name="服务器列表")), - ("zk_hosts", django_mysql.models.JSONField(default=dict, verbose_name="ZK服务器列表")), + ("servers", models.JSONField(default=dict, verbose_name="服务器列表")), + ("zk_hosts", models.JSONField(default=dict, verbose_name="ZK服务器列表")), ("zk_account", models.CharField(default="", max_length=255, verbose_name="ZK账号")), ( "zk_password", @@ -43,7 +42,7 @@ class Migration(migrations.Migration): ), ("package_inner_url", models.TextField(verbose_name="安装包内网地址")), ("package_outer_url", models.TextField(verbose_name="安装包外网地址")), - ("agent_config", django_mysql.models.JSONField(default=dict, verbose_name="Agent配置信息")), + ("agent_config", models.JSONField(default=dict, verbose_name="Agent配置信息")), ("status", models.CharField(default="", max_length=255, verbose_name="接入点状态")), ("description", models.TextField(verbose_name="接入点描述")), ("is_enabled", models.BooleanField(default=True, verbose_name="是否启用")), @@ -61,7 +60,7 @@ class Migration(migrations.Migration): ("bk_cloud_name", models.CharField(max_length=45)), ("isp", models.CharField(blank=True, max_length=45, null=True, verbose_name="云服务商")), ("ap_id", models.IntegerField(null=True, verbose_name="接入点ID")), - ("creator", django_mysql.models.JSONField(default=dict, verbose_name="管控区域创建者")), + ("creator", models.JSONField(default=dict, verbose_name="管控区域创建者")), ("is_visible", models.BooleanField(default=True, verbose_name="是否可见")), ("is_deleted", models.BooleanField(default=False, verbose_name="是否删除")), ], @@ -79,7 +78,7 @@ class Migration(migrations.Migration): ("event_type", models.CharField(max_length=20, verbose_name="事件类型")), ("action", models.CharField(max_length=20, verbose_name="动作")), ("obj_type", models.CharField(max_length=32, verbose_name="对象类型")), - ("data", django_mysql.models.JSONField(default=dict, verbose_name="实例信息")), + ("data", models.JSONField(default=dict, verbose_name="实例信息")), ("create_time", models.DateTimeField(auto_now_add=True, verbose_name="创建时间")), ], ), @@ -112,7 +111,7 @@ class Migration(migrations.Migration): db_index=True, max_length=255, primary_key=True, serialize=False, verbose_name="键" ), ), - ("v_json", django_mysql.models.JSONField(default=dict, verbose_name="值")), + ("v_json", models.JSONField(default=dict, verbose_name="值")), ], options={ "verbose_name": "配置表", @@ -203,7 +202,7 @@ class Migration(migrations.Migration): ), ), ("ap_id", models.IntegerField(null=True, verbose_name="接入点ID")), - ("upstream_nodes", django_mysql.models.JSONField(default=list, verbose_name="上游节点")), + ("upstream_nodes", models.JSONField(default=list, verbose_name="上游节点")), ("created_at", models.DateTimeField(auto_now_add=True, verbose_name="创建时间")), ("updated_at", models.DateTimeField(null=True, verbose_name="更新时间")), ], @@ -242,7 +241,7 @@ class Migration(migrations.Migration): ), ( "extra_data", - django_mysql.models.JSONField(blank=True, default=dict, null=True, verbose_name="额外认证资料"), + models.JSONField(blank=True, default=dict, null=True, verbose_name="额外认证资料"), ), ("retention", models.IntegerField(default=1, verbose_name="保留天数")), ("updated_at", models.DateTimeField(null=True, verbose_name="更新时间")), @@ -293,7 +292,7 @@ class Migration(migrations.Migration): ), ), ("subscription_id", models.IntegerField(db_index=True, verbose_name="订阅ID")), - ("task_id_list", django_mysql.models.JSONField(default=list, verbose_name="任务ID列表")), + ("task_id_list", models.JSONField(default=list, verbose_name="任务ID列表")), ("start_time", models.DateTimeField(auto_now_add=True, verbose_name="创建任务时间")), ("end_time", models.DateTimeField(blank=True, null=True, verbose_name="任务结束时间")), ( @@ -314,14 +313,14 @@ class Migration(migrations.Migration): ), ( "global_params", - django_mysql.models.JSONField(blank=True, default=dict, null=True, verbose_name="全局运行参数"), + models.JSONField(blank=True, default=dict, null=True, verbose_name="全局运行参数"), ), ( "statistics", - django_mysql.models.JSONField(blank=True, default=dict, null=True, verbose_name="任务统计信息"), + models.JSONField(blank=True, default=dict, null=True, verbose_name="任务统计信息"), ), - ("bk_biz_scope", django_mysql.models.JSONField(default=dict, verbose_name="业务范围")), - ("error_hosts", django_mysql.models.JSONField(default=dict, verbose_name="发生错误的主机")), + ("bk_biz_scope", models.JSONField(default=dict, verbose_name="业务范围")), + ("error_hosts", models.JSONField(default=dict, verbose_name="发生错误的主机")), ], options={ "verbose_name": "任务信息", @@ -406,7 +405,7 @@ class Migration(migrations.Migration): name="PipelineTree", fields=[ ("id", models.CharField(max_length=32, primary_key=True, serialize=False, verbose_name="PipelineID")), - ("tree", django_mysql.models.JSONField(default=dict, verbose_name="Pipeline拓扑树")), + ("tree", models.JSONField(default=dict, verbose_name="Pipeline拓扑树")), ], ), migrations.CreateModel( @@ -500,7 +499,7 @@ class Migration(migrations.Migration): verbose_name="进程类型", ), ), - ("configs", django_mysql.models.JSONField(default=list, verbose_name="配置文件")), + ("configs", models.JSONField(default=list, verbose_name="配置文件")), ("listen_ip", models.CharField(max_length=45, null=True, verbose_name="监听IP")), ("listen_port", models.IntegerField(null=True, verbose_name="监听端口")), ("setup_path", models.TextField(default="", verbose_name="二进制文件所在路径")), @@ -529,7 +528,7 @@ class Migration(migrations.Migration): fields=[ ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), ("bk_username", models.CharField(max_length=45, verbose_name="用户名")), - ("favorite", django_mysql.models.JSONField(default=dict, verbose_name="用户收藏信息,比如管控区域等")), + ("favorite", models.JSONField(default=dict, verbose_name="用户收藏信息,比如管控区域等")), ("update_time", models.DateTimeField(auto_now_add=True)), ], options={ @@ -554,8 +553,8 @@ class Migration(migrations.Migration): verbose_name="节点类型", ), ), - ("nodes", django_mysql.models.JSONField(default=list, verbose_name="节点")), - ("target_hosts", django_mysql.models.JSONField(default=None, null=True, verbose_name="下发的目标机器")), + ("nodes", models.JSONField(default=list, verbose_name="节点")), + ("target_hosts", models.JSONField(default=None, null=True, verbose_name="下发的目标机器")), ("from_system", models.CharField(max_length=30, verbose_name="所属系统")), ("update_time", models.DateTimeField(auto_now=True, verbose_name="更新时间")), ("create_time", models.DateTimeField(auto_now_add=True, verbose_name="创建时间")), @@ -571,8 +570,8 @@ class Migration(migrations.Migration): ("task_id", models.IntegerField(db_index=True, verbose_name="任务ID")), ("subscription_id", models.IntegerField(db_index=True, verbose_name="订阅ID")), ("instance_id", models.CharField(db_index=True, max_length=50, verbose_name="实例ID")), - ("instance_info", django_mysql.models.JSONField(default=dict, verbose_name="实例信息")), - ("steps", django_mysql.models.JSONField(default=dict, verbose_name="步骤信息")), + ("instance_info", models.JSONField(default=dict, verbose_name="实例信息")), + ("steps", models.JSONField(default=dict, verbose_name="步骤信息")), ("pipeline_id", models.CharField(blank=True, default="", max_length=50, verbose_name="Pipeline ID")), ("update_time", models.DateTimeField(auto_now=True, verbose_name="更新时间")), ("create_time", models.DateTimeField(auto_now_add=True, verbose_name="创建时间")), @@ -585,8 +584,8 @@ class Migration(migrations.Migration): fields=[ ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), ("subscription_id", models.IntegerField(db_index=True, verbose_name="订阅ID")), - ("scope", django_mysql.models.JSONField(default=dict, verbose_name="执行范围")), - ("actions", django_mysql.models.JSONField(default=dict, verbose_name="不同step执行的动作名称。键值对")), + ("scope", models.JSONField(default=dict, verbose_name="执行范围")), + ("actions", models.JSONField(default=dict, verbose_name="不同step执行的动作名称。键值对")), ("create_time", models.DateTimeField(auto_now_add=True, verbose_name="创建时间")), ("is_auto_trigger", models.BooleanField(default=False, verbose_name="是否为自动触发")), ], @@ -620,8 +619,8 @@ class Migration(migrations.Migration): ("index", models.IntegerField(default=0, verbose_name="顺序")), ("step_id", models.CharField(max_length=64, verbose_name="步骤ID")), ("type", models.CharField(max_length=20, verbose_name="步骤类型")), - ("config", django_mysql.models.JSONField(default=dict, verbose_name="配置")), - ("params", django_mysql.models.JSONField(default=dict, verbose_name="参数")), + ("config", models.JSONField(default=dict, verbose_name="配置")), + ("params", models.JSONField(default=dict, verbose_name="参数")), ], options={ "ordering": ["index"], diff --git a/apps/node_man/migrations/0005_auto_20200518_1209.py b/apps/node_man/migrations/0005_auto_20200518_1209.py index 087c8e0883..dd049d8190 100644 --- a/apps/node_man/migrations/0005_auto_20200518_1209.py +++ b/apps/node_man/migrations/0005_auto_20200518_1209.py @@ -10,7 +10,6 @@ """ # Generated by Django 2.2.8 on 2020-05-18 04:09 -import django_mysql.models from django.db import migrations, models @@ -37,17 +36,17 @@ class Migration(migrations.Migration): migrations.AlterField( model_name="accesspoint", name="taskserver", - field=django_mysql.models.JSONField(default=dict, verbose_name="GSE 任务服务器列表"), + field=models.JSONField(default=dict, verbose_name="GSE 任务服务器列表"), ), migrations.AddField( model_name="accesspoint", name="btfileserver", - field=django_mysql.models.JSONField(default=dict, verbose_name="GSE BT文件服务器列表"), + field=models.JSONField(default=dict, verbose_name="GSE BT文件服务器列表"), ), migrations.AddField( model_name="accesspoint", name="dataserver", - field=django_mysql.models.JSONField(default=dict, verbose_name="GSE 数据服务器列表"), + field=models.JSONField(default=dict, verbose_name="GSE 数据服务器列表"), ), migrations.AddField( model_name="accesspoint", diff --git a/apps/node_man/migrations/0006_auto_20200602_1104.py b/apps/node_man/migrations/0006_auto_20200602_1104.py index c43d74ddeb..b075a60973 100644 --- a/apps/node_man/migrations/0006_auto_20200602_1104.py +++ b/apps/node_man/migrations/0006_auto_20200602_1104.py @@ -10,8 +10,7 @@ """ # Generated by Django 2.2.8 on 2020-06-02 03:04 -import django_mysql.models -from django.db import migrations +from django.db import migrations, models from django.db.backends.mysql import schema import apps.node_man.models @@ -48,7 +47,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name="host", name="extra_data", - field=django_mysql.models.JSONField(blank=True, default=dict, null=True, verbose_name="额外数据"), + field=models.JSONField(blank=True, default=dict, null=True, verbose_name="额外数据"), ), migrations.AlterField( model_name="pipelinetree", diff --git a/apps/node_man/migrations/0008_accesspoint_creator.py b/apps/node_man/migrations/0008_accesspoint_creator.py index 8040eda53a..451b350558 100644 --- a/apps/node_man/migrations/0008_accesspoint_creator.py +++ b/apps/node_man/migrations/0008_accesspoint_creator.py @@ -8,10 +8,9 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -import django_mysql.models # Generated by Django 2.2.8 on 2020-06-05 11:22 -from django.db import migrations +from django.db import migrations, models class Migration(migrations.Migration): @@ -24,6 +23,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name="accesspoint", name="creator", - field=django_mysql.models.JSONField(default=("admin",), verbose_name="接入点创建者"), + field=models.JSONField(default=("admin",), verbose_name="接入点创建者"), ), ] diff --git a/apps/node_man/migrations/0009_resourcewatchevent.py b/apps/node_man/migrations/0009_resourcewatchevent.py index 1ea4d97deb..e9c4a60f20 100644 --- a/apps/node_man/migrations/0009_resourcewatchevent.py +++ b/apps/node_man/migrations/0009_resourcewatchevent.py @@ -10,7 +10,6 @@ """ # Generated by Django 2.2.8 on 2020-06-19 08:23 -import django_mysql.models from django.db import migrations, models @@ -34,7 +33,7 @@ class Migration(migrations.Migration): ), ), ("bk_resource", models.CharField(max_length=32, verbose_name="资源")), - ("bk_detail", django_mysql.models.JSONField(default=dict, verbose_name="事件详情")), + ("bk_detail", models.JSONField(default=dict, verbose_name="事件详情")), ( "create_time", models.DateTimeField( diff --git a/apps/node_man/migrations/0015_accesspoint_port_config.py b/apps/node_man/migrations/0015_accesspoint_port_config.py index f72215e7a3..b9c784ea2a 100644 --- a/apps/node_man/migrations/0015_accesspoint_port_config.py +++ b/apps/node_man/migrations/0015_accesspoint_port_config.py @@ -10,8 +10,7 @@ """ # Generated by Django 2.2.8 on 2020-09-23 06:20 -import django_mysql.models -from django.db import migrations +from django.db import migrations, models class Migration(migrations.Migration): @@ -24,6 +23,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name="accesspoint", name="port_config", - field=django_mysql.models.JSONField(default=dict, verbose_name="GSE端口配置"), + field=models.JSONField(default=dict, verbose_name="GSE端口配置"), ), ] diff --git a/apps/node_man/migrations/0020_auto_20200917_1132.py b/apps/node_man/migrations/0020_auto_20200917_1132.py index c3218acb0d..c545c07661 100644 --- a/apps/node_man/migrations/0020_auto_20200917_1132.py +++ b/apps/node_man/migrations/0020_auto_20200917_1132.py @@ -10,7 +10,6 @@ """ # Generated by Django 2.2.8 on 2020-09-17 03:32 -import django_mysql.models from django.db import migrations, models @@ -24,7 +23,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name="subscription", name="bk_biz_scope", - field=django_mysql.models.JSONField(default=list, verbose_name="业务范围"), + field=models.JSONField(default=list, verbose_name="业务范围"), ), migrations.AddField( model_name="subscription", diff --git a/apps/node_man/migrations/0022_accesspoint_proxy_package.py b/apps/node_man/migrations/0022_accesspoint_proxy_package.py index 972b016388..0d2d2a607a 100644 --- a/apps/node_man/migrations/0022_accesspoint_proxy_package.py +++ b/apps/node_man/migrations/0022_accesspoint_proxy_package.py @@ -10,8 +10,7 @@ """ # Generated by Django 2.2.8 on 2020-11-02 06:33 -import django_mysql.models -from django.db import migrations +from django.db import migrations, models class Migration(migrations.Migration): @@ -24,6 +23,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name="accesspoint", name="proxy_package", - field=django_mysql.models.JSONField(default=list, verbose_name="Proxy上的安装包"), + field=models.JSONField(default=list, verbose_name="Proxy上的安装包"), ), ] diff --git a/apps/node_man/migrations/0024_accesspoint_bscp_config.py b/apps/node_man/migrations/0024_accesspoint_bscp_config.py index deb8902625..d48656233e 100644 --- a/apps/node_man/migrations/0024_accesspoint_bscp_config.py +++ b/apps/node_man/migrations/0024_accesspoint_bscp_config.py @@ -10,8 +10,7 @@ """ # Generated by Django 2.2.8 on 2020-12-10 12:34 -import django_mysql.models -from django.db import migrations +from django.db import migrations, models class Migration(migrations.Migration): @@ -24,6 +23,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name="accesspoint", name="bscp_config", - field=django_mysql.models.JSONField(default=dict, verbose_name="BSCP配置"), + field=models.JSONField(default=dict, verbose_name="BSCP配置"), ), ] diff --git a/apps/node_man/migrations/0027_auto_20210804_1259.py b/apps/node_man/migrations/0027_auto_20210804_1259.py index f3cf21212e..026570f9d0 100644 --- a/apps/node_man/migrations/0027_auto_20210804_1259.py +++ b/apps/node_man/migrations/0027_auto_20210804_1259.py @@ -1,6 +1,5 @@ # Generated by Django 2.2.8 on 2021-08-04 04:59 -import django_mysql.models from django.db import migrations, models @@ -17,8 +16,8 @@ class Migration(migrations.Migration): ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), ("name", models.CharField(max_length=45, verbose_name="名称")), ("bk_cloud_id", models.IntegerField(verbose_name="管控区域ID")), - ("jump_servers", django_mysql.models.JSONField(default=dict, verbose_name="安装通道跳板机")), - ("upstream_servers", django_mysql.models.JSONField(default=dict, verbose_name="上游节点")), + ("jump_servers", models.JSONField(default=dict, verbose_name="安装通道跳板机")), + ("upstream_servers", models.JSONField(default=dict, verbose_name="上游节点")), ], options={ "verbose_name": "安装通道", diff --git a/apps/node_man/migrations/0028_auto_20210208_1752.py b/apps/node_man/migrations/0028_auto_20210208_1752.py index f9736384dc..6d688fe723 100644 --- a/apps/node_man/migrations/0028_auto_20210208_1752.py +++ b/apps/node_man/migrations/0028_auto_20210208_1752.py @@ -10,7 +10,6 @@ """ # Generated by Django 2.2.8 on 2021-02-08 09:52 -import django_mysql.models from django.db import migrations, models @@ -26,7 +25,7 @@ class Migration(migrations.Migration): fields=[ ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), ("job_instance_id", models.BigIntegerField(db_index=True, verbose_name="作业实例ID")), - ("subscription_instance_ids", django_mysql.models.JSONField(default=list, verbose_name="订阅实例ID列表")), + ("subscription_instance_ids", models.JSONField(default=list, verbose_name="订阅实例ID列表")), ("node_id", models.CharField(db_index=True, max_length=32, verbose_name="节点ID")), ( "status", diff --git a/apps/node_man/migrations/0029_auto_20210309_0916.py b/apps/node_man/migrations/0029_auto_20210309_0916.py index 7e7419ffc6..61882120d7 100644 --- a/apps/node_man/migrations/0029_auto_20210309_0916.py +++ b/apps/node_man/migrations/0029_auto_20210309_0916.py @@ -10,7 +10,6 @@ """ # Generated by Django 2.2.8 on 2021-03-09 01:16 -import django_mysql.models from django.db import migrations, models @@ -24,7 +23,7 @@ class Migration(migrations.Migration): migrations.AddField( model_name="gseplugindesc", name="node_manage_control", - field=django_mysql.models.JSONField(blank=True, default=dict, null=True, verbose_name="节点管理管控插件信息"), + field=models.JSONField(blank=True, default=dict, null=True, verbose_name="节点管理管控插件信息"), ), migrations.AlterField( model_name="job", diff --git a/apps/node_man/migrations/0072_subscription_instance_selector.py b/apps/node_man/migrations/0072_subscription_instance_selector.py index c66939f3a6..5f134e1cac 100644 --- a/apps/node_man/migrations/0072_subscription_instance_selector.py +++ b/apps/node_man/migrations/0072_subscription_instance_selector.py @@ -1,19 +1,18 @@ # Generated by Django 3.2.4 on 2023-08-04 02:19 -from django.db import migrations -import django_mysql.models +from django.db import migrations, models class Migration(migrations.Migration): dependencies = [ - ('node_man', '0071_update_ap_gse_version_to_v2'), + ("node_man", "0071_update_ap_gse_version_to_v2"), ] operations = [ migrations.AddField( - model_name='subscription', - name='instance_selector', - field=django_mysql.models.JSONField(blank=True, default=dict, null=True, verbose_name='订阅任务范围主机属性筛选'), + model_name="subscription", + name="instance_selector", + field=models.JSONField(blank=True, default=dict, null=True, verbose_name="订阅任务范围主机属性筛选"), ), ] diff --git a/apps/node_man/migrations/0083_subscription_operate_info.py b/apps/node_man/migrations/0083_subscription_operate_info.py index 1dfc4719f0..526d84545f 100644 --- a/apps/node_man/migrations/0083_subscription_operate_info.py +++ b/apps/node_man/migrations/0083_subscription_operate_info.py @@ -9,8 +9,7 @@ specific language governing permissions and limitations under the License. """ -import django_mysql.models -from django.db import migrations +from django.db import migrations, models class Migration(migrations.Migration): @@ -23,6 +22,6 @@ class Migration(migrations.Migration): migrations.AddField( model_name="subscription", name="operate_info", - field=django_mysql.models.JSONField(default=None, null=True, verbose_name="操作信息"), + field=models.JSONField(default=None, null=True, verbose_name="操作信息"), ), ] diff --git a/apps/node_man/models.py b/apps/node_man/models.py index aed50d23cb..65d186d07c 100644 --- a/apps/node_man/models.py +++ b/apps/node_man/models.py @@ -33,13 +33,12 @@ from django.conf import settings from django.core.cache import cache from django.db import models -from django.db.models import Q, QuerySet +from django.db.models import JSONField, Q, QuerySet from django.utils import timezone -from django.utils.encoding import force_text +from django.utils.encoding import force_str from django.utils.functional import Promise from django.utils.translation import get_language -from django.utils.translation import ugettext_lazy as _ -from django_mysql.models import JSONField +from django.utils.translation import gettext_lazy as _ from jinja2 import Template from apps.backend.subscription.errors import PipelineExecuteFailed, SubscriptionNotExist @@ -73,12 +72,14 @@ class LazyJSONEncoder(json.JSONEncoder): def default(self, obj): if isinstance(obj, Promise): - return force_text(obj) + return force_str(obj) return super(LazyJSONEncoder, self).default(obj) class LazyJSONField(JSONField): - _default_json_encoder = LazyJSONEncoder(allow_nan=False) + def __init__(self, *args, **kwargs): + kwargs["encoder"] = LazyJSONEncoder + super().__init__(*args, **kwargs) class GlobalSettings(models.Model): @@ -179,7 +180,7 @@ class KeyEnum(Enum): CMDB_INTERNAL_CLOUD_IDS = "CMDB_INTERNAL_CLOUD_IDS" key = models.CharField(_("键"), max_length=255, db_index=True, primary_key=True) - v_json = JSONField(_("值")) + v_json = JSONField(_("值"), default=dict) def map_values(self, objs, source, target): """ @@ -193,9 +194,7 @@ def map_values(self, objs, source, target): def fetch_isp(self): isps = dict(GlobalSettings.objects.filter(key="isp").values_list("key", "v_json")).get("isp", []) - result = self.map_values( - isps, lambda isp: isp["isp"], lambda isp: {"isp_name": isp["isp_name"]} - ) + result = self.map_values(isps, lambda isp: isp["isp"], lambda isp: {"isp_name": isp["isp_name"]}) return result @@ -228,7 +227,7 @@ class IdentityData(models.Model): password = SymmetricTextField(_("密码"), blank=True, null=True) port = models.IntegerField(_("端口"), null=True, default=22) key = SymmetricTextField(_("密钥"), blank=True, null=True) - extra_data = JSONField(_("额外认证资料"), blank=True, null=True) + extra_data = JSONField(_("额外认证资料"), blank=True, null=True, default=dict) retention = models.IntegerField(_("保留天数"), default=1) updated_at = models.DateTimeField(_("更新时间"), null=True, auto_now=False) @@ -279,7 +278,7 @@ class Host(models.Model): updated_at = models.DateTimeField(_("更新时间"), null=True, auto_now=False, db_index=True) dept_name = models.CharField(_("运维部门"), max_length=128, db_index=True, blank=True, null=True, default="") - extra_data = JSONField(_("额外数据"), blank=True, null=True) + extra_data = JSONField(_("额外数据"), blank=True, null=True, default=dict) @classmethod def get_by_host_info(cls, host_info): @@ -553,17 +552,17 @@ class AccessPoint(models.Model): gse_version = models.CharField( _("GSE 版本"), max_length=32, default=GseVersion.V1.value, choices=GseVersion.list_choices() ) - btfileserver = JSONField(_("GSE BT文件服务器列表")) - dataserver = JSONField(_("GSE 数据服务器列表")) - taskserver = JSONField(_("GSE 任务服务器列表")) - zk_hosts = JSONField(_("ZK服务器列表")) + btfileserver = JSONField(_("GSE BT文件服务器列表"), default=dict) + dataserver = JSONField(_("GSE 数据服务器列表"), default=dict) + taskserver = JSONField(_("GSE 任务服务器列表"), default=dict) + zk_hosts = JSONField(_("ZK服务器列表"), default=dict) zk_account = models.CharField(_("ZK账号"), max_length=255, default="", blank=True, null=True) zk_password = SymmetricTextField(_("密码"), blank=True, null=True) package_inner_url = models.TextField(_("安装包内网地址")) package_outer_url = models.TextField(_("安装包外网地址")) # 历史遗留命名，现在该路径表示存储源的存储目录路径 nginx_path = models.TextField(_("Nginx路径"), blank=True, null=True) - agent_config = JSONField(_("Agent配置信息")) + agent_config = JSONField(_("Agent配置信息"), default=dict) status = models.CharField(_("接入点状态"), max_length=255, default="", blank=True, null=True) description = models.TextField(_("接入点描述")) is_enabled = models.BooleanField(_("是否启用"), default=True) @@ -751,7 +750,7 @@ class Cloud(models.Model): isp = models.CharField(_("云服务商"), max_length=45, null=True, blank=True) ap_id = models.IntegerField(_("接入点ID"), null=True) gse_v1_ap_id = models.IntegerField(_("GSE1.0接入点ID"), null=True, blank=True) - creator = JSONField(_("管控区域创建者")) + creator = JSONField(_("管控区域创建者"), default=dict) is_visible = models.BooleanField(_("是否可见"), default=True) is_deleted = models.BooleanField(_("是否删除"), default=False) @@ -789,8 +788,8 @@ class InstallChannel(models.Model): name = models.CharField(_("名称"), max_length=45) bk_cloud_id = models.IntegerField(_("管控区域ID")) - jump_servers = JSONField(_("安装通道跳板机")) - upstream_servers = JSONField(_("上游节点")) + jump_servers = JSONField(_("安装通道跳板机"), default=dict) + upstream_servers = JSONField(_("上游节点"), default=dict) hidden = models.BooleanField(_("是否隐藏"), default=False) def get_upstream_servers_by_ap_id(self, ap_id): @@ -875,10 +874,10 @@ class Job(export_job_prometheus_mixin(), models.Model): choices=constants.JobStatusType.get_choices(), default=constants.JobStatusType.PENDING, ) - global_params = JSONField(_("全局运行参数"), blank=True, null=True) + global_params = JSONField(_("全局运行参数"), blank=True, null=True, default=dict) statistics = JSONField(_("任务统计信息"), blank=True, null=True, default=dict) - bk_biz_scope = JSONField(_("业务范围")) - error_hosts = JSONField(_("发生错误的主机")) + bk_biz_scope = JSONField(_("业务范围"), default=dict) + error_hosts = JSONField(_("发生错误的主机"), default=dict) is_auto_trigger = models.BooleanField(_("是否为自动触发"), default=False) class Meta: @@ -944,7 +943,7 @@ class GsePluginDesc(models.Model): ) source_app_code = models.CharField(_("来源系统APP CODE"), max_length=64, null=True, blank=True) - node_manage_control = JSONField(_("节点管理管控插件信息"), null=True, blank=True) + node_manage_control = JSONField(_("节点管理管控插件信息"), null=True, blank=True, default=dict) class Meta: verbose_name = _("插件信息（GsePluginDesc）") @@ -1784,8 +1783,8 @@ class SubscriptionStep(models.Model): index = models.IntegerField(_("顺序"), default=0) step_id = models.CharField(_("步骤ID"), max_length=64, db_index=True) type = models.CharField(_("步骤类型"), max_length=20) - config = JSONField(_("配置")) - params = JSONField(_("参数")) + config = JSONField(_("配置"), default=dict) + params = JSONField(_("参数"), default=dict) @property def subscription(self): @@ -1857,7 +1856,7 @@ class CategoryType(object): object_type = models.CharField(_("对象类型"), max_length=20, choices=OBJECT_TYPE_CHOICES, db_index=True) node_type = models.CharField(_("节点类型"), max_length=20, choices=NODE_TYPE_CHOICES, db_index=True) nodes = JSONField(_("节点"), default=list) - instance_selector = JSONField(_("订阅任务范围主机属性筛选"), null=True, blank=True) + instance_selector = JSONField(_("订阅任务范围主机属性筛选"), null=True, blank=True, default=dict) target_hosts = JSONField(_("下发的目标机器"), default=None, null=True) from_system = models.CharField(_("所属系统"), max_length=30) update_time = models.DateTimeField(_("更新时间"), auto_now=True, db_index=True) @@ -2188,8 +2187,8 @@ class SubscriptionTask(models.Model): """订阅执行任务""" subscription_id = models.IntegerField(_("订阅ID"), db_index=True) - scope = JSONField(_("执行范围")) - actions = JSONField(_("不同step执行的动作名称。键值对")) + scope = JSONField(_("执行范围"), default=dict) + actions = JSONField(_("不同step执行的动作名称。键值对"), default=dict) create_time = models.DateTimeField(_("创建时间"), auto_now_add=True, db_index=True) err_msg = models.TextField(_("错误信息"), blank=True, null=True) is_ready = models.BooleanField(_("是否准备就绪"), default=False) @@ -2233,8 +2232,8 @@ class SubscriptionInstanceRecord(models.Model): task_id = models.IntegerField(_("任务ID"), db_index=True) subscription_id = models.IntegerField(_("订阅ID"), db_index=True) instance_id = models.CharField(_("实例ID"), max_length=128, db_index=True) - instance_info = JSONField(_("实例信息")) - steps = JSONField(_("步骤信息")) + instance_info = JSONField(_("实例信息"), default=dict) + steps = JSONField(_("步骤信息"), default=dict) pipeline_id = models.CharField(_("Pipeline ID"), max_length=50, default="", blank=True, db_index=True) start_pipeline_id = models.CharField(_("Start Pipeline ID"), max_length=50, default="", blank=True, db_index=True) update_time = models.DateTimeField(_("更新时间"), default=timezone.now, db_index=True) @@ -2414,7 +2413,7 @@ class CmdbEventRecord(models.Model): event_type = models.CharField(_("事件类型"), max_length=20) action = models.CharField(_("动作"), max_length=20) obj_type = models.CharField(_("对象类型"), max_length=32) - data = JSONField(_("实例信息")) + data = JSONField(_("实例信息"), default=dict) create_time = models.DateTimeField(_("创建时间"), auto_now_add=True) class Meta: @@ -2428,7 +2427,7 @@ class PipelineTree(models.Model): """ id = models.CharField(_("PipelineID"), primary_key=True, max_length=32) - tree = LazyJSONField(_("Pipeline拓扑树")) + tree = LazyJSONField(_("Pipeline拓扑树"), default=dict) def run(self, priority=None): # 根据流程描述结构创建流程对象 diff --git a/apps/node_man/periodic_tasks/add_biz_to_gse2_gray_scope.py b/apps/node_man/periodic_tasks/add_biz_to_gse2_gray_scope.py index 4b955cebd3..5ef98d8cf6 100644 --- a/apps/node_man/periodic_tasks/add_biz_to_gse2_gray_scope.py +++ b/apps/node_man/periodic_tasks/add_biz_to_gse2_gray_scope.py @@ -10,7 +10,7 @@ """ from typing import Dict, List -from celery.task import periodic_task +from celery import current_app from django.db import transaction from apps.backend.subscription.tools import fetch_biz_info_map @@ -20,7 +20,7 @@ from common.log import logger -@periodic_task( +@current_app.task( queue="default", options={"queue": "default"}, run_every=SYNC_BIZ_TO_GRAY_SCOPE_LIST_INTERVAL, diff --git a/apps/node_man/periodic_tasks/clean_expired_info.py b/apps/node_man/periodic_tasks/clean_expired_info.py index 53840d7dc8..79b886afdb 100644 --- a/apps/node_man/periodic_tasks/clean_expired_info.py +++ b/apps/node_man/periodic_tasks/clean_expired_info.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from celery.task import periodic_task +from celery import current_app from django.utils import timezone from apps.node_man import constants @@ -36,7 +36,7 @@ def clean_identity_data(task_id, start, end): clean_identity_data(task_id, end, end + constants.QUERY_EXPIRED_INFO_LENS) -@periodic_task( +@current_app.task( queue="default", options={"queue": "default"}, run_every=constants.CLEAN_EXPIRED_INFO_INTERVAL, diff --git a/apps/node_man/periodic_tasks/clean_resource_watch_event.py b/apps/node_man/periodic_tasks/clean_resource_watch_event.py index 71e4ffc3ce..75a39de3ff 100644 --- a/apps/node_man/periodic_tasks/clean_resource_watch_event.py +++ b/apps/node_man/periodic_tasks/clean_resource_watch_event.py @@ -9,14 +9,14 @@ specific language governing permissions and limitations under the License. """ +from celery import current_app from celery.schedules import crontab -from celery.task import periodic_task from apps.node_man.models import ResourceWatchEvent from common.log import logger -@periodic_task( +@current_app.task( queue="default", options={"queue": "default"}, run_every=crontab(hour="2", minute="0", day_of_week="*", day_of_month="*", month_of_year="*"), diff --git a/apps/node_man/periodic_tasks/clean_subscription_record_info.py b/apps/node_man/periodic_tasks/clean_subscription_record_info.py index 24247795ba..fa36a3e5d0 100644 --- a/apps/node_man/periodic_tasks/clean_subscription_record_info.py +++ b/apps/node_man/periodic_tasks/clean_subscription_record_info.py @@ -8,8 +8,8 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +from celery import current_app from celery.schedules import crontab -from celery.task import periodic_task from django.utils import timezone from apps.node_man import models @@ -61,7 +61,7 @@ def update_subscription_instance_record(task_id): models.GlobalSettings.update_config(KEY, last_sub_task_id) -@periodic_task( +@current_app.task( queue="default", options={"queue": "default"}, run_every=crontab(hour="0", minute="0", day_of_week="*", day_of_month="*", month_of_year="*"), diff --git a/apps/node_man/periodic_tasks/configuration_policy.py b/apps/node_man/periodic_tasks/configuration_policy.py index 653ec3ad93..604ce54db7 100644 --- a/apps/node_man/periodic_tasks/configuration_policy.py +++ b/apps/node_man/periodic_tasks/configuration_policy.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from celery.task import periodic_task +from celery import current_app from apps.node_man import constants from apps.node_man.constants import NodeType @@ -18,7 +18,7 @@ from common.log import logger -@periodic_task( +@current_app.task( queue="default", options={"queue": "default"}, run_every=constants.CONFIGURATION_POLICY_INTERVAL, diff --git a/apps/node_man/periodic_tasks/gse_svr_discovery.py b/apps/node_man/periodic_tasks/gse_svr_discovery.py index 5a437ab921..275fae6666 100644 --- a/apps/node_man/periodic_tasks/gse_svr_discovery.py +++ b/apps/node_man/periodic_tasks/gse_svr_discovery.py @@ -11,7 +11,7 @@ from telnetlib import Telnet from typing import Any, Dict, List, Optional, Tuple -from celery.task import periodic_task +from celery import current_app from django.conf import settings from kazoo.client import KazooClient from kazoo.exceptions import NoAuthError, NoNodeError @@ -47,7 +47,7 @@ def __exit__(self, exc_type, exc_val, exc_tb): self.zk_client.stop() -@periodic_task( +@current_app.task( queue="default", options={"queue": "default"}, run_every=constants.GSE_SVR_DISCOVERY_INTERVAL, diff --git a/apps/node_man/periodic_tasks/sync_agent_status_task.py b/apps/node_man/periodic_tasks/sync_agent_status_task.py index ca5c0fd1f9..6a26771120 100644 --- a/apps/node_man/periodic_tasks/sync_agent_status_task.py +++ b/apps/node_man/periodic_tasks/sync_agent_status_task.py @@ -11,7 +11,7 @@ import typing from collections import defaultdict -from celery.task import periodic_task, task +from celery import current_app from django.conf import settings from django.db.models import QuerySet from django.db.transaction import atomic @@ -25,7 +25,7 @@ from common.log import logger -@task(queue="default", ignore_result=True) +@current_app.task(queue="default", ignore_result=True) def update_or_create_host_agent_status(task_id: int, host_queryset: QuerySet): """ 更新 Agent 状态 @@ -169,7 +169,7 @@ def update_or_create_host_agent_status(task_id: int, host_queryset: QuerySet): return host_id__agent_state_info -@periodic_task( +@current_app.task( queue="default", options={"queue": "default"}, run_every=constants.SYNC_AGENT_STATUS_TASK_INTERVAL, diff --git a/apps/node_man/periodic_tasks/sync_all_isp_to_cmdb.py b/apps/node_man/periodic_tasks/sync_all_isp_to_cmdb.py index 42f7674a61..585b22b57c 100644 --- a/apps/node_man/periodic_tasks/sync_all_isp_to_cmdb.py +++ b/apps/node_man/periodic_tasks/sync_all_isp_to_cmdb.py @@ -11,7 +11,7 @@ import time from typing import Any, Dict, List -from celery.task import periodic_task +from celery import current_app from apps.component.esbclient import client_v2 from apps.exceptions import ComponentCallError @@ -47,7 +47,7 @@ def sync_all_isp_to_cmdb(task_id): logger.info(f"{task_id} | Sync cloud isp info task complete.") -@periodic_task( +@current_app.task( queue="default", options={"queue": "default"}, run_every=constants.SYNC_ISP_TO_CMDB_INTERVAL, diff --git a/apps/node_man/periodic_tasks/sync_cmdb_biz_topo_task.py b/apps/node_man/periodic_tasks/sync_cmdb_biz_topo_task.py index 7c1202cd73..2c28dfe846 100644 --- a/apps/node_man/periodic_tasks/sync_cmdb_biz_topo_task.py +++ b/apps/node_man/periodic_tasks/sync_cmdb_biz_topo_task.py @@ -11,7 +11,7 @@ from concurrent.futures import ThreadPoolExecutor, as_completed from copy import deepcopy -from celery.task import periodic_task, task +from celery import current_app from django.conf import settings from django.core.cache import cache @@ -85,7 +85,7 @@ def format_biz_topo(biz_topo: dict) -> dict: return {"biz_format_topo": biz_topo_copy, "biz_nodes": biz_nodes} -@task(queue="default", ignore_result=True) +@current_app.task(queue="default", ignore_result=True) def get_and_cache_format_biz_topo(bk_biz_id: int) -> dict: """ 获取格式化业务拓扑并缓存 @@ -156,7 +156,7 @@ def cache_all_biz_topo(): as_completed(tasks) -@task(queue="default", ignore_result=True) +@current_app.task(queue="default", ignore_result=True) def cache_all_biz_topo_delay_task(): task_id = sync_cmdb_biz_topo_periodic_task.request.id logger.warning(f"{task_id} | cache_all_biz_topo_delay_task: Sync cmdb biz topo task' cache expired") @@ -164,7 +164,7 @@ def cache_all_biz_topo_delay_task(): logger.warning(f"{task_id} | cache_all_biz_topo_delay_task: Re-cache finished") -@periodic_task( +@current_app.task( queue="default", options={"queue": "default"}, run_every=constants.SYNC_CMDB_BIZ_TOPO_TASK_INTERVAL, diff --git a/apps/node_man/periodic_tasks/sync_cmdb_cloud_area.py b/apps/node_man/periodic_tasks/sync_cmdb_cloud_area.py index 7af8022699..cdf6e3b47e 100644 --- a/apps/node_man/periodic_tasks/sync_cmdb_cloud_area.py +++ b/apps/node_man/periodic_tasks/sync_cmdb_cloud_area.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from celery.task import periodic_task +from celery import current_app from apps.component.esbclient import client_v2 from apps.exceptions import ComponentCallError @@ -71,7 +71,7 @@ def update_or_create_cloud_area(task_id, start): update_or_create_cloud_area(task_id, start + constants.QUERY_CLOUD_LIMIT) -@periodic_task( +@current_app.task( queue="default", options={"queue": "default"}, run_every=constants.SYNC_CMDB_CLOUD_AREA_INTERVAL, diff --git a/apps/node_man/periodic_tasks/sync_cmdb_host.py b/apps/node_man/periodic_tasks/sync_cmdb_host.py index 96adf513db..bad28ebed8 100644 --- a/apps/node_man/periodic_tasks/sync_cmdb_host.py +++ b/apps/node_man/periodic_tasks/sync_cmdb_host.py @@ -12,8 +12,8 @@ import math import typing +from celery import current_app from celery.schedules import crontab -from celery.task import periodic_task from django.conf import settings from django.db import transaction from django.db.models import Q @@ -603,7 +603,7 @@ def query_cmdb_and_handle_need_delete_host_ids(host_ids: typing.List[int], task_ return [] -@periodic_task( +@current_app.task( queue="default", options={"queue": "default"}, run_every=crontab(hour="0", minute="0", day_of_week="*", day_of_month="*", month_of_year="*"), @@ -625,7 +625,7 @@ def sync_cmdb_host_task(bk_biz_id=None): sync_cmdb_host(bk_biz_id, task_id) -@periodic_task( +@current_app.task( queue="default", options={"queue": "default"}, run_every=constants.CLEAR_NEED_DELETE_HOST_IDS_INTERVAL, diff --git a/apps/node_man/periodic_tasks/sync_proc_status_task.py b/apps/node_man/periodic_tasks/sync_proc_status_task.py index 4273c7cfe5..49695173ae 100644 --- a/apps/node_man/periodic_tasks/sync_proc_status_task.py +++ b/apps/node_man/periodic_tasks/sync_proc_status_task.py @@ -12,7 +12,7 @@ from collections import defaultdict from blueapps.conf import settings -from celery.task import periodic_task, task +from celery import current_app from django.db.models import QuerySet from django.db.transaction import atomic @@ -26,7 +26,7 @@ from common.log import logger -@task(queue="default", ignore_result=True) +@current_app.task(queue="default", ignore_result=True) def update_or_create_proc_status( task_id: int, host_queryset: QuerySet, @@ -190,7 +190,7 @@ def update_or_create_proc_status( ) -@periodic_task( +@current_app.task( queue="default", options={"queue": "default"}, run_every=constants.SYNC_PROC_STATUS_TASK_INTERVAL, diff --git a/apps/node_man/periodic_tasks/update_proxy_file.py b/apps/node_man/periodic_tasks/update_proxy_file.py index b8ebe0ec0b..107b59c502 100644 --- a/apps/node_man/periodic_tasks/update_proxy_file.py +++ b/apps/node_man/periodic_tasks/update_proxy_file.py @@ -16,8 +16,8 @@ from json import JSONDecodeError from typing import Dict, List, Union +from celery import current_app from celery.schedules import crontab -from celery.task import periodic_task from django.conf import settings from django.db.models import Q @@ -32,7 +32,7 @@ from common.log import logger -@periodic_task( +@current_app.task( queue="default", options={"queue": "default"}, run_every=crontab(hour="1", minute="0", day_of_week="*", day_of_month="*", month_of_year="*"), diff --git a/apps/node_man/serializers/ap.py b/apps/node_man/serializers/ap.py index 9adf3fd8ae..732eb914f9 100644 --- a/apps/node_man/serializers/ap.py +++ b/apps/node_man/serializers/ap.py @@ -10,7 +10,7 @@ """ from typing import List -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from apps.backend.subscription.steps.agent_adapter.base import AgentSetupTools @@ -111,10 +111,10 @@ class ZKSerializer(serializers.Serializer): zk_password = serializers.CharField(label=_("ZK密码"), required=False, allow_blank=True) agent_config = serializers.DictField(label=_("Agent配置")) description = serializers.CharField(label=_("接入点描述"), allow_blank=True) - creator = serializers.JSONField(_("接入点创建者"), required=False) + creator = serializers.JSONField(label=_("接入点创建者"), required=False) port_config = serializers.DictField(default=GSE_PORT_DEFAULT_VALUE) proxy_package = serializers.ListField() - bscp_config = serializers.DictField(_("BSCP配置"), required=False) + bscp_config = serializers.DictField(label=_("BSCP配置"), required=False) outer_callback_url = serializers.CharField(label=_("节点管理外网回调地址"), required=False, allow_blank=True) callback_url = serializers.CharField(label=_("节点管理内网回调地址"), required=False, allow_blank=True) diff --git a/apps/node_man/serializers/base.py b/apps/node_man/serializers/base.py index 433868587e..302711a461 100644 --- a/apps/node_man/serializers/base.py +++ b/apps/node_man/serializers/base.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from apps.exceptions import ValidationError diff --git a/apps/node_man/serializers/cloud.py b/apps/node_man/serializers/cloud.py index 85272a22c5..a73c046cfa 100644 --- a/apps/node_man/serializers/cloud.py +++ b/apps/node_man/serializers/cloud.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers diff --git a/apps/node_man/serializers/cmdb.py b/apps/node_man/serializers/cmdb.py index 25ce4e4b66..efeef47f82 100644 --- a/apps/node_man/serializers/cmdb.py +++ b/apps/node_man/serializers/cmdb.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from apps.node_man.constants import IamActionType diff --git a/apps/node_man/serializers/debug.py b/apps/node_man/serializers/debug.py index 3ee2969e63..d8e5eb074c 100644 --- a/apps/node_man/serializers/debug.py +++ b/apps/node_man/serializers/debug.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers diff --git a/apps/node_man/serializers/host.py b/apps/node_man/serializers/host.py index 2d2b07ae1e..2cfb6df1bf 100644 --- a/apps/node_man/serializers/host.py +++ b/apps/node_man/serializers/host.py @@ -10,7 +10,7 @@ """ from ipaddress import IPv4Address -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from apps.exceptions import ValidationError diff --git a/apps/node_man/serializers/host_v2.py b/apps/node_man/serializers/host_v2.py index 3ee6af4fab..6344ffd635 100644 --- a/apps/node_man/serializers/host_v2.py +++ b/apps/node_man/serializers/host_v2.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from apps.exceptions import ValidationError diff --git a/apps/node_man/serializers/iam.py b/apps/node_man/serializers/iam.py index 8b56f8b26a..b13d764eab 100644 --- a/apps/node_man/serializers/iam.py +++ b/apps/node_man/serializers/iam.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from apps.node_man.constants import IAM_ACTION_DICT diff --git a/apps/node_man/serializers/install_channel.py b/apps/node_man/serializers/install_channel.py index 53ba664b4f..a29efdc6a9 100644 --- a/apps/node_man/serializers/install_channel.py +++ b/apps/node_man/serializers/install_channel.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from apps.utils import basic diff --git a/apps/node_man/serializers/job.py b/apps/node_man/serializers/job.py index 8d5fc28165..1c22201cf4 100644 --- a/apps/node_man/serializers/job.py +++ b/apps/node_man/serializers/job.py @@ -14,7 +14,7 @@ from django.conf import settings from django.db.models import Q -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from apps.backend.subscription.steps.agent_adapter.adapter import LEGACY diff --git a/apps/node_man/serializers/meta.py b/apps/node_man/serializers/meta.py index 2ceb0068ac..fe9d7a15b8 100644 --- a/apps/node_man/serializers/meta.py +++ b/apps/node_man/serializers/meta.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from apps.node_man.constants import JOB_MAX_VALUE, NODE_MAN_LOG_LEVEL diff --git a/apps/node_man/serializers/plugin.py b/apps/node_man/serializers/plugin.py index f7f6ab8b1f..52a9ac79b4 100644 --- a/apps/node_man/serializers/plugin.py +++ b/apps/node_man/serializers/plugin.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ from django.utils.translation import get_language -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from apps.core.ipchooser.handlers.host_handler import HostHandler diff --git a/apps/node_man/serializers/plugin_v2.py b/apps/node_man/serializers/plugin_v2.py index 80a5bcd2b2..cd1f2eff7e 100644 --- a/apps/node_man/serializers/plugin_v2.py +++ b/apps/node_man/serializers/plugin_v2.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from rest_framework.exceptions import ValidationError diff --git a/apps/node_man/serializers/policy.py b/apps/node_man/serializers/policy.py index 51401b3a0d..e79d36e32f 100644 --- a/apps/node_man/serializers/policy.py +++ b/apps/node_man/serializers/policy.py @@ -10,7 +10,7 @@ """ import typing -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from apps.exceptions import ValidationError diff --git a/apps/node_man/serializers/response.py b/apps/node_man/serializers/response.py index f0fadc5438..f079235d97 100644 --- a/apps/node_man/serializers/response.py +++ b/apps/node_man/serializers/response.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from apps.mock_data.node_man_mkd.cloud.api import CLOUD_LIST_RESPONSE diff --git a/apps/node_man/serializers/sync_task.py b/apps/node_man/serializers/sync_task.py index a5c801fce3..7ba3a23ae3 100644 --- a/apps/node_man/serializers/sync_task.py +++ b/apps/node_man/serializers/sync_task.py @@ -8,8 +8,8 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers -from django.utils.translation import ugettext_lazy as _ from apps.backend.sync_task.constants import SyncTaskType from apps.exceptions import ValidationError @@ -20,8 +20,8 @@ class CreateSyncTaskSerializer(serializers.Serializer): task_params = serializers.DictField(label="任务调用参数", required=False) def validate(self, attrs): - if attrs['task_name'] not in SyncTaskType.list_member_values(): - raise ValidationError(_("不支持的任务名称: {}".format(attrs['task_name']))) + if attrs["task_name"] not in SyncTaskType.list_member_values(): + raise ValidationError(_("不支持的任务名称: {}".format(attrs["task_name"]))) return attrs diff --git a/apps/node_man/tests/test_handlers/test_job.py b/apps/node_man/tests/test_handlers/test_job.py index 562c269d21..446d2a1cbd 100644 --- a/apps/node_man/tests/test_handlers/test_job.py +++ b/apps/node_man/tests/test_handlers/test_job.py @@ -14,7 +14,7 @@ from django.test import TestCase from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.mock_data import common_unit from apps.node_man import constants, tools diff --git a/apps/node_man/tests/test_handlers/test_meta.py b/apps/node_man/tests/test_handlers/test_meta.py index 1c7e23e086..146121ea27 100644 --- a/apps/node_man/tests/test_handlers/test_meta.py +++ b/apps/node_man/tests/test_handlers/test_meta.py @@ -14,7 +14,7 @@ from django.conf import settings from django.test import override_settings -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from apps.node_man import constants as const from apps.node_man import tools diff --git a/apps/node_man/tools/host.py b/apps/node_man/tools/host.py index 42b9ab7c9e..42daa8bb8f 100644 --- a/apps/node_man/tools/host.py +++ b/apps/node_man/tools/host.py @@ -13,7 +13,7 @@ from bkcrypto.asymmetric.ciphers import BaseAsymmetricCipher from django.db.models import QuerySet -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.core.encrypt import constants as core_encrypt_constants from apps.core.encrypt import handlers as core_encrypt_handlers diff --git a/apps/node_man/tools/job.py b/apps/node_man/tools/job.py index d69e64ebeb..289cf8f8ca 100644 --- a/apps/node_man/tools/job.py +++ b/apps/node_man/tools/job.py @@ -17,7 +17,7 @@ from django.conf import settings from django.db.models import Q from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.backend.subscription import tools from apps.node_man import constants, models diff --git a/apps/node_man/tools/policy.py b/apps/node_man/tools/policy.py index 3ef4096f2c..7e6b610d29 100644 --- a/apps/node_man/tools/policy.py +++ b/apps/node_man/tools/policy.py @@ -14,7 +14,7 @@ from django.utils import timezone from django.utils.translation import get_language -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.node_man import exceptions, models from apps.node_man.handlers.cmdb import CmdbHandler diff --git a/apps/node_man/urls.py b/apps/node_man/urls.py index faa90fc485..8c75e8286c 100644 --- a/apps/node_man/urls.py +++ b/apps/node_man/urls.py @@ -10,8 +10,7 @@ """ from blueapps.account.decorators import login_exempt from django.conf import settings -from django.conf.urls import url -from django.urls import include +from django.urls import include, re_path from iam import IAM from iam.contrib.django.dispatcher import DjangoBasicResourceApiDispatcher from rest_framework import routers @@ -80,16 +79,16 @@ package_dispatcher.register("package", PackageResourceProvider()) urlpatterns = [ - url(r"^$", views.index), - url(r"^ping/?$", views.ping), - url(r"^version/?$", views.version), - url(r"^metrics/?$", views.metrics), - url(r"^logout/?$", views.user_exit), - url(r"^tools/download/$", views.tools_download), - url(r"api/", include(router.urls)), - url(r"api/iam/v1/biz", biz_dispatcher.as_view([login_exempt])), - url(r"api/iam/v1/cloud", cloud_dispatcher.as_view([login_exempt])), - url(r"api/iam/v1/ap", ap_dispatcher.as_view([login_exempt])), - url(r"api/iam/v1/strategy", strategy_dispatcher.as_view([login_exempt])), - url(r"api/iam/v1/package", package_dispatcher.as_view([login_exempt])), + re_path(r"^$", views.index), + re_path(r"^ping/?$", views.ping), + re_path(r"^version/?$", views.version), + re_path(r"^metrics/?$", views.metrics), + re_path(r"^logout/?$", views.user_exit), + re_path(r"^tools/download/$", views.tools_download), + re_path(r"api/", include(router.urls)), + re_path(r"api/iam/v1/biz", biz_dispatcher.as_view([login_exempt])), + re_path(r"api/iam/v1/cloud", cloud_dispatcher.as_view([login_exempt])), + re_path(r"api/iam/v1/ap", ap_dispatcher.as_view([login_exempt])), + re_path(r"api/iam/v1/strategy", strategy_dispatcher.as_view([login_exempt])), + re_path(r"api/iam/v1/package", package_dispatcher.as_view([login_exempt])), ] diff --git a/apps/node_man/views/__init__.py b/apps/node_man/views/__init__.py index fb7c0321f0..b697f4164c 100644 --- a/apps/node_man/views/__init__.py +++ b/apps/node_man/views/__init__.py @@ -16,7 +16,7 @@ from django.contrib.auth import logout from django.http import HttpResponse, JsonResponse, StreamingHttpResponse from django.shortcuts import render -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from django.views.decorators.cache import never_cache from django_prometheus.exports import ExportToDjangoView diff --git a/apps/node_man/views/ap.py b/apps/node_man/views/ap.py index dca9ca9f4a..0138657f9e 100644 --- a/apps/node_man/views/ap.py +++ b/apps/node_man/views/ap.py @@ -10,7 +10,7 @@ """ from django.conf import settings from django.db.transaction import atomic -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from drf_yasg.utils import swagger_auto_schema from rest_framework.decorators import action from rest_framework.response import Response diff --git a/apps/node_man/views/meta.py b/apps/node_man/views/meta.py index 0a9f0c75b6..f5f74aaf16 100644 --- a/apps/node_man/views/meta.py +++ b/apps/node_man/views/meta.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from drf_yasg.utils import swagger_auto_schema from rest_framework.decorators import action from rest_framework.response import Response diff --git a/apps/node_man/views/permission.py b/apps/node_man/views/permission.py index 8917a8e255..c901332d39 100644 --- a/apps/node_man/views/permission.py +++ b/apps/node_man/views/permission.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from drf_yasg.utils import swagger_auto_schema from rest_framework.decorators import action from rest_framework.response import Response diff --git a/apps/node_man/views/plugin_v2.py b/apps/node_man/views/plugin_v2.py index 7df98ba818..3a00b20e44 100644 --- a/apps/node_man/views/plugin_v2.py +++ b/apps/node_man/views/plugin_v2.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ from django.http import JsonResponse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from drf_yasg.utils import swagger_auto_schema from rest_framework.decorators import action from rest_framework.response import Response diff --git a/apps/utils/drf.py b/apps/utils/drf.py index d18b0ee254..85e9e40a9f 100644 --- a/apps/utils/drf.py +++ b/apps/utils/drf.py @@ -14,7 +14,7 @@ import six from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from rest_framework import serializers from rest_framework.authentication import SessionAuthentication from rest_framework.fields import DateTimeField, empty diff --git a/apps/utils/encrypt/key.py b/apps/utils/encrypt/key.py index 6930097ecf..63fa36ec9f 100644 --- a/apps/utils/encrypt/key.py +++ b/apps/utils/encrypt/key.py @@ -15,7 +15,7 @@ from bkcrypto.asymmetric.configs import KeyConfig as AsymmetricKeyConfig from bkcrypto.symmetric.configs import KeyConfig as SymmetricKeyConfig from django.conf import settings -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.node_man.exceptions import QueryGlobalSettingsException from apps.node_man.models import GlobalSettings diff --git a/apps/utils/orm.py b/apps/utils/orm.py index 3081df4840..de42b3ed58 100644 --- a/apps/utils/orm.py +++ b/apps/utils/orm.py @@ -13,7 +13,7 @@ from django.db import models from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from apps.utils.local import ( get_request_username, diff --git a/bin/hooks/start_cmds/celery/start-baworker b/bin/hooks/start_cmds/celery/start-baworker index c7ed78aefd..30df559761 100755 --- a/bin/hooks/start_cmds/celery/start-baworker +++ b/bin/hooks/start_cmds/celery/start-baworker @@ -1,2 +1,2 @@ #!/bin/bash -celery -A apps.backend worker -Q backend_additional_task -n baworker@%h -P threads -c 64 -O fair --time-limit=1800 --maxtasksperchild=50 +celery -A apps.backend worker -Q backend_additional_task -n baworker@%h -P threads -c 64 -O fair --time-limit=1800 --max-tasks-per-child=50 diff --git a/bin/hooks/start_cmds/celery/start-bworker b/bin/hooks/start_cmds/celery/start-bworker index a06bd197b6..8845c230ff 100755 --- a/bin/hooks/start_cmds/celery/start-bworker +++ b/bin/hooks/start_cmds/celery/start-bworker @@ -1,2 +1,2 @@ #!/bin/bash -celery -A apps.backend worker -Q backend -n bworker@%h -P threads -c 64 --maxtasksperchild=50 -O fair --time-limit=1800 +celery -A apps.backend worker -Q backend -n bworker@%h -P threads -c 64 --max-tasks-per-child=50 -O fair --time-limit=1800 diff --git a/bin/hooks/start_cmds/celery/start-common-pipeline-worker b/bin/hooks/start_cmds/celery/start-common-pipeline-worker index e32092bde3..db3622b839 100755 --- a/bin/hooks/start_cmds/celery/start-common-pipeline-worker +++ b/bin/hooks/start_cmds/celery/start-common-pipeline-worker @@ -1,2 +1,2 @@ #!/bin/bash -celery -A apps.backend worker -Q backend_additional_task,pipeline,pipeline_priority,service_schedule,service_schedule_priority,pipeline_additional_task,pipeline_additional_task_priority -n pcworker@%h -P threads -c 128 --maxtasksperchild=50 -O fair --time-limit=1800 +celery -A apps.backend worker -Q backend_additional_task,pipeline,pipeline_priority,service_schedule,service_schedule_priority,pipeline_additional_task,pipeline_additional_task_priority -n pcworker@%h -P threads -c 128 --max-tasks-per-child=50 -O fair --time-limit=1800 diff --git a/bin/hooks/start_cmds/celery/start-common-worker b/bin/hooks/start_cmds/celery/start-common-worker index 3a8018afae..526b5e12f4 100755 --- a/bin/hooks/start_cmds/celery/start-common-worker +++ b/bin/hooks/start_cmds/celery/start-common-worker @@ -1,2 +1,2 @@ #!/bin/bash -celery -A apps.backend worker -Q default,backend -n cworker@%h -P threads -c 128 --maxtasksperchild=50 -O fair --time-limit=1800 +celery -A apps.backend worker -Q default,backend -n cworker@%h -P threads -c 128 --max-tasks-per-child=50 -O fair --time-limit=1800 diff --git a/bin/hooks/start_cmds/celery/start-dworker b/bin/hooks/start_cmds/celery/start-dworker index 3d47e966d2..043d9366ce 100755 --- a/bin/hooks/start_cmds/celery/start-dworker +++ b/bin/hooks/start_cmds/celery/start-dworker @@ -1,2 +1,2 @@ #!/bin/bash -celery -A apps.backend worker -Q default -n default@%h -P threads -c 64 --maxtasksperchild=50 -O fair --time-limit=1800 +celery -A apps.backend worker -Q default -n default@%h -P threads -c 64 --max-tasks-per-child=50 -O fair --time-limit=1800 diff --git a/bin/hooks/start_cmds/celery/start-paworker b/bin/hooks/start_cmds/celery/start-paworker index 37a8d7c4ac..9a0d66d6f5 100755 --- a/bin/hooks/start_cmds/celery/start-paworker +++ b/bin/hooks/start_cmds/celery/start-paworker @@ -1,2 +1,2 @@ #!/bin/bash -celery -A apps.backend worker -Q pipeline_additional_task,pipeline_additional_task_priority -n common_worker@%h -l info -P threads -c 32 --maxtasksperchild=50 -O fair --time-limit=1800 +celery -A apps.backend worker -Q pipeline_additional_task,pipeline_additional_task_priority -n common_worker@%h -l info -P threads -c 32 --max-tasks-per-child=50 -O fair --time-limit=1800 diff --git a/bin/hooks/start_cmds/celery/start-psworker b/bin/hooks/start_cmds/celery/start-psworker index af0977dd2b..cfa15c3b42 100755 --- a/bin/hooks/start_cmds/celery/start-psworker +++ b/bin/hooks/start_cmds/celery/start-psworker @@ -4,4 +4,4 @@ # this problem solution is not use eventlet # refer to -> https://github.com/celery/celery/issues/5924 # refer to -> https://github.com/intelowlproject/IntelOwl/issues/379 -celery -A apps.backend worker -Q service_schedule,service_schedule_priority -n schedule_worker@%h -P threads -c 100 --maxtasksperchild=50 -O fair --time-limit=1800 +celery -A apps.backend worker -Q service_schedule,service_schedule_priority -n schedule_worker@%h -P threads -c 100 --max-tasks-per-child=50 -O fair --time-limit=1800 diff --git a/bin/hooks/start_cmds/celery/start-pworker b/bin/hooks/start_cmds/celery/start-pworker index a77b311d56..e657e735fb 100755 --- a/bin/hooks/start_cmds/celery/start-pworker +++ b/bin/hooks/start_cmds/celery/start-pworker @@ -1,2 +1,2 @@ #!/bin/bash -celery -A apps.backend worker -Q pipeline,pipeline_priority -n pipeline_worker@%h -P threads -c 100 --maxtasksperchild=50 -O fair --time-limit=1800 +celery -A apps.backend worker -Q pipeline,pipeline_priority -n pipeline_worker@%h -P threads -c 100 --max-tasks-per-child=50 -O fair --time-limit=1800 diff --git a/common/api/base.py b/common/api/base.py index 53c949475c..f4c30b57b5 100644 --- a/common/api/base.py +++ b/common/api/base.py @@ -21,7 +21,7 @@ from django.core.cache import cache from django.utils import translation from django.utils.module_loading import import_string -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from apps.exceptions import ApiRequestError, ApiResultError, AppBaseException from apps.prometheus import metrics diff --git a/common/api/exception.py b/common/api/exception.py index 50edfc5121..23a95e7e39 100644 --- a/common/api/exception.py +++ b/common/api/exception.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ -from django.utils.encoding import force_text +from django.utils.encoding import force_str from apps.exceptions import AppBaseException @@ -19,7 +19,7 @@ class DataAPIException(AppBaseException): def __init__(self, api_obj, error_message, response=None): self.api_obj = api_obj - self.error_message = force_text(error_message) + self.error_message = force_str(error_message) self.response = response if self.response is not None: diff --git a/common/api/models.py b/common/api/models.py index 9de028e506..15b5589b21 100644 --- a/common/api/models.py +++ b/common/api/models.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ class DataAPIRecord(models.Model): diff --git a/common/api/modules/bk_node.py b/common/api/modules/bk_node.py index f0f390b43c..3f65e5a029 100644 --- a/common/api/modules/bk_node.py +++ b/common/api/modules/bk_node.py @@ -8,7 +8,7 @@ an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from ..base import DataAPI from ..domains import BK_NODE_APIGATEWAY_ROOT diff --git a/common/api/modules/cc.py b/common/api/modules/cc.py index 4837be1050..b3bd491391 100644 --- a/common/api/modules/cc.py +++ b/common/api/modules/cc.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from ..base import BaseApi, DataAPI from ..domains import CC_APIGATEWAY_ROOT_V2 diff --git a/common/api/modules/esb.py b/common/api/modules/esb.py index 5e057093ec..8a9d29fa97 100644 --- a/common/api/modules/esb.py +++ b/common/api/modules/esb.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ from django.conf import settings -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from ..base import BaseApi, DataAPI from ..domains import ESB_APIGATEWAY_ROOT_V2 diff --git a/common/api/modules/gse.py b/common/api/modules/gse.py index e77730a1cf..b70f4fa033 100644 --- a/common/api/modules/gse.py +++ b/common/api/modules/gse.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from ..base import BaseApi, DataAPI from ..domains import GSE_APIGATEWAY_ROOT diff --git a/common/api/modules/gse_v2.py b/common/api/modules/gse_v2.py index 10c4b11b00..1a17f6b0ac 100644 --- a/common/api/modules/gse_v2.py +++ b/common/api/modules/gse_v2.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from ..base import BaseApi, DataAPI from ..domains import GSE_APIGATEWAY_ROOT_V2 diff --git a/common/api/modules/job.py b/common/api/modules/job.py index b9944905fe..33dfaebf42 100644 --- a/common/api/modules/job.py +++ b/common/api/modules/job.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from ..base import BaseApi, DataAPI from ..domains import JOB_APIGATEWAY_ROOT_V3 diff --git a/common/api/modules/sops.py b/common/api/modules/sops.py index df786258e4..93bfba176a 100644 --- a/common/api/modules/sops.py +++ b/common/api/modules/sops.py @@ -9,7 +9,7 @@ specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from ..base import BaseApi, DataAPI from ..domains import SOPS_APIGATEWAY_ROOT diff --git a/common/api/modules/yunti.py b/common/api/modules/yunti.py index 1ba77eb611..6d459f0fa3 100644 --- a/common/api/modules/yunti.py +++ b/common/api/modules/yunti.py @@ -15,7 +15,7 @@ import typing from django.conf import settings -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from ..base import DataAPI from ..domains import BKAPP_YUNTI_API_ROOT diff --git a/common/api/tasks.py b/common/api/tasks.py index 04fe0c64cc..98aa378965 100644 --- a/common/api/tasks.py +++ b/common/api/tasks.py @@ -10,13 +10,13 @@ """ from datetime import datetime, timedelta +from celery import current_app from celery.schedules import crontab -from celery.task import periodic_task from common.api.models import DataAPIRecord -@periodic_task(run_every=crontab(minute="0", hour="0")) +@current_app.task(run_every=crontab(minute="0", hour="0")) def delete_api_log(): """ 每天清理dataapi日志 diff --git a/common/context_processors.py b/common/context_processors.py index fe10009e87..26a86e29e7 100644 --- a/common/context_processors.py +++ b/common/context_processors.py @@ -15,7 +15,7 @@ from blueapps.account.conf import ConfFixture from django.conf import settings from django.utils.translation import get_language -from django.utils.translation import ugettext as _ +from django.utils.translation import gettext as _ from version_log.utils import get_latest_version from apps.core.concurrent.cache import FuncCacheDecorator diff --git a/config/default.py b/config/default.py index 9c5e88962c..1f752959c6 100644 --- a/config/default.py +++ b/config/default.py @@ -18,7 +18,7 @@ from blueapps.conf.default_settings import * # noqa from django.http import HttpResponseRedirect from django.urls import reverse -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ import env from apps.utils.encrypt.crypto import Interceptor @@ -55,7 +55,6 @@ # 请在这里加入你的自定义 APP INSTALLED_APPS += ( - "django_mysql", "drf_yasg", "rest_framework", "common.api", @@ -264,7 +263,7 @@ "apps.node_man.periodic_tasks.add_biz_to_gse2_gray_scope", ) -BK_NODEMAN_CELERY_RESULT_BACKEND_BROKER_URL = "amqp://{user}:{passwd}@{host}:{port}/{vhost}".format( +BK_NODEMAN_CELERY_RESULT_BACKEND_BROKER_URL = "{user}:{passwd}@{host}:{port}/{vhost}".format( user=os.getenv("RABBITMQ_USER"), passwd=os.getenv("RABBITMQ_PASSWORD"), host=os.getenv("RABBITMQ_HOST"), @@ -278,7 +277,7 @@ INSTALLED_APPS += ("django_celery_beat", "django_celery_results") CELERY_ENABLE_UTC = False CELERYBEAT_SCHEDULER = "django_celery_beat.schedulers.DatabaseScheduler" - CELERY_RESULT_BACKEND = BK_NODEMAN_CELERY_RESULT_BACKEND_BROKER_URL + CELERY_RESULT_BACKEND = f"rpc://{BK_NODEMAN_CELERY_RESULT_BACKEND_BROKER_URL}" CELERY_RESULT_PERSISTENT = True # celery3 的配置，升级后先行注释，待确认无用后废弃 # CELERY_TASK_RESULT_EXPIRES = 60 * 30 # 30分钟丢弃结果 @@ -442,6 +441,9 @@ DATAAPI_TIME_ZONE = "Etc/GMT-8" DATETIME_FORMAT = "%Y-%m-%d %H:%M:%S%z" +# 4.2 django.utils的timezone默认使用zoneinfo +USE_DEPRECATED_PYTZ = True + # 翻译 USE_I18N = True USE_L10N = True @@ -758,7 +760,7 @@ def get_standard_redis_mode(cls, config_redis_mode: str, default: Optional[str] ) # BROKER_URL - BROKER_URL = BK_NODEMAN_CELERY_RESULT_BACKEND_BROKER_URL + BROKER_URL = f"amqp://{BK_NODEMAN_CELERY_RESULT_BACKEND_BROKER_URL}" REDBEAT_KEY_PREFIX = "nodeman" diff --git a/impacket/smbserver.py b/impacket/smbserver.py index a5452ada93..7c8e21d682 100644 --- a/impacket/smbserver.py +++ b/impacket/smbserver.py @@ -21,6 +21,7 @@ import calendar +import configparser import datetime import errno import fnmatch @@ -33,6 +34,7 @@ import random import shutil import socket +import socketserver import string import struct import sys @@ -41,7 +43,6 @@ from binascii import a2b_hex, hexlify, unhexlify from six import PY2, b, text_type -from six.moves import configparser, socketserver # For signing from impacket import nmb, ntlm, smb diff --git a/pipeline/apps.py b/pipeline/apps.py index 959f3fa664..14e3f411da 100644 --- a/pipeline/apps.py +++ b/pipeline/apps.py @@ -18,7 +18,11 @@ from django.apps import AppConfig from django.conf import settings from redis.sentinel import Sentinel -from rediscluster import RedisCluster + +try: + from redis.cluster import RedisCluster +except ImportError: + from rediscluster import RedisCluster logger = logging.getLogger("root") diff --git a/pipeline/component_framework/models.py b/pipeline/component_framework/models.py index 0975e17d8f..1e7caff87a 100644 --- a/pipeline/component_framework/models.py +++ b/pipeline/component_framework/models.py @@ -12,7 +12,7 @@ """ from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from pipeline.component_framework.constants import LEGACY_PLUGINS_VERSION from pipeline.component_framework.library import ComponentLibrary diff --git a/pipeline/contrib/celery_tools/___init__.py b/pipeline/contrib/celery_tools/___init__.py new file mode 100644 index 0000000000..26a6d1c27a --- /dev/null +++ b/pipeline/contrib/celery_tools/___init__.py @@ -0,0 +1,12 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" diff --git a/pipeline/contrib/celery_tools/periodic.py b/pipeline/contrib/celery_tools/periodic.py new file mode 100644 index 0000000000..57534b57c6 --- /dev/null +++ b/pipeline/contrib/celery_tools/periodic.py @@ -0,0 +1,46 @@ +# -*- coding: utf-8 -*- +""" +Tencent is pleased to support the open source community by making 蓝鲸智云PaaS平台社区版 (BlueKing PaaS Community +Edition) available. +Copyright (C) 2017 THL A29 Limited, a Tencent company. All rights reserved. +Licensed under the MIT License (the "License"); you may not use this file except in compliance with the License. +You may obtain a copy of the License at +http://opensource.org/licenses/MIT +Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on +an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the +specific language governing permissions and limitations under the License. +""" +from celery import Task, current_app +from celery.schedules import maybe_schedule + + +class PipelinePeriodicTask(Task): + """A task that adds itself to the :setting:`beat_schedule` setting.""" + + abstract = True + ignore_result = True + relative = False + options = None + compat = True + + def __init__(self): + if not hasattr(self, "run_every"): + raise NotImplementedError("Periodic tasks must have a run_every attribute") + self.run_every = maybe_schedule(self.run_every, self.relative) + super(PipelinePeriodicTask, self).__init__() + + @classmethod + def on_bound(cls, app): + app.conf.beat_schedule[cls.name] = { + "task": cls.name, + "schedule": cls.run_every, + "args": (), + "kwargs": {}, + "options": cls.options or {}, + "relative": cls.relative, + } + + +def periodic_task(*args, **options): + """Deprecated decorator, please use :setting:`beat_schedule`.""" + return current_app.task(**dict({"base": PipelinePeriodicTask}, **options)) diff --git a/pipeline/contrib/external_plugins/models/base.py b/pipeline/contrib/external_plugins/models/base.py index bb27d625aa..36b20461fb 100644 --- a/pipeline/contrib/external_plugins/models/base.py +++ b/pipeline/contrib/external_plugins/models/base.py @@ -17,7 +17,7 @@ from copy import deepcopy from django.db import IntegrityError, models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from pipeline.component_framework.library import ComponentLibrary from pipeline.contrib.external_plugins import exceptions diff --git a/pipeline/contrib/external_plugins/models/source.py b/pipeline/contrib/external_plugins/models/source.py index 7de8333973..15b3230317 100644 --- a/pipeline/contrib/external_plugins/models/source.py +++ b/pipeline/contrib/external_plugins/models/source.py @@ -12,11 +12,21 @@ """ from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from pipeline.conf import settings -from pipeline.contrib.external_plugins.models.base import FILE_SYSTEM, GIT, S3, ExternalPackageSource, package_source -from pipeline.contrib.external_plugins.utils.importer import FSModuleImporter, GitRepoModuleImporter, S3ModuleImporter +from pipeline.contrib.external_plugins.models.base import ( + FILE_SYSTEM, + GIT, + S3, + ExternalPackageSource, + package_source, +) +from pipeline.contrib.external_plugins.utils.importer import ( + FSModuleImporter, + GitRepoModuleImporter, + S3ModuleImporter, +) @package_source diff --git a/pipeline/contrib/periodic_task/djcelery/tzcrontab.py b/pipeline/contrib/periodic_task/djcelery/tzcrontab.py index 9d87dcf086..341d84b412 100644 --- a/pipeline/contrib/periodic_task/djcelery/tzcrontab.py +++ b/pipeline/contrib/periodic_task/djcelery/tzcrontab.py @@ -17,7 +17,7 @@ import pytz from celery import schedules -from celery.utils.timeutils import is_naive, make_aware +from celery.utils.time import is_naive, make_aware schedstate = namedtuple("schedstate", ("is_due", "next")) logger = logging.getLogger("celery") @@ -101,12 +101,12 @@ def __reduce__(self): def __eq__(self, other): if isinstance(other, schedules.crontab): return ( - other.month_of_year == self.month_of_year and - other.day_of_month == self.day_of_month and - other.day_of_week == self.day_of_week and - other.hour == self.hour and - other.minute == self.minute and - other.tz == self.tz + other.month_of_year == self.month_of_year + and other.day_of_month == self.day_of_month + and other.day_of_week == self.day_of_week + and other.hour == self.hour + and other.minute == self.minute + and other.tz == self.tz ) return NotImplemented diff --git a/pipeline/contrib/periodic_task/models.py b/pipeline/contrib/periodic_task/models.py index bdaaa5c4dc..b311c9297f 100644 --- a/pipeline/contrib/periodic_task/models.py +++ b/pipeline/contrib/periodic_task/models.py @@ -19,7 +19,7 @@ from django.core.exceptions import MultipleObjectsReturned, ValidationError from django.db import models from django.db.models import signals -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from pipeline.constants import PIPELINE_DEFAULT_PRIORITY from pipeline.contrib.periodic_task.djcelery import managers @@ -28,7 +28,12 @@ from pipeline.contrib.periodic_task.djcelery.utils import now from pipeline.contrib.periodic_task.signals import periodic_task_start_failed from pipeline.exceptions import InvalidOperationException -from pipeline.models import CompressJSONField, PipelineInstance, PipelineTemplate, Snapshot +from pipeline.models import ( + CompressJSONField, + PipelineInstance, + PipelineTemplate, + Snapshot, +) from pipeline.utils.uniqid import uniqid PERIOD_CHOICES = ( @@ -43,7 +48,11 @@ @python_2_unicode_compatible class IntervalSchedule(models.Model): every = models.IntegerField(_("every"), null=False) - period = models.CharField(_("period"), max_length=24, choices=PERIOD_CHOICES,) + period = models.CharField( + _("period"), + max_length=24, + choices=PERIOD_CHOICES, + ) class Meta: verbose_name = _("interval") @@ -83,9 +92,21 @@ def cronexp(field): class CrontabSchedule(models.Model): minute = models.CharField(_("minute"), max_length=64, default="*") hour = models.CharField(_("hour"), max_length=64, default="*") - day_of_week = models.CharField(_("day of week"), max_length=64, default="*",) - day_of_month = models.CharField(_("day of month"), max_length=64, default="*",) - month_of_year = models.CharField(_("month of year"), max_length=64, default="*",) + day_of_week = models.CharField( + _("day of week"), + max_length=64, + default="*", + ) + day_of_month = models.CharField( + _("day of month"), + max_length=64, + default="*", + ) + month_of_year = models.CharField( + _("month of year"), + max_length=64, + default="*", + ) timezone = timezone_field.TimeZoneField(default="UTC") class Meta: @@ -153,10 +174,19 @@ def last_change(cls): @python_2_unicode_compatible class DjCeleryPeriodicTask(models.Model): - name = models.CharField(_("name"), max_length=200, unique=True, help_text=_("Useful description"),) + name = models.CharField( + _("name"), + max_length=200, + unique=True, + help_text=_("Useful description"), + ) task = models.CharField(_("task name"), max_length=200) interval = models.ForeignKey( - IntervalSchedule, null=True, blank=True, verbose_name=_("interval"), on_delete=models.CASCADE, + IntervalSchedule, + null=True, + blank=True, + verbose_name=_("interval"), + on_delete=models.CASCADE, ) crontab = models.ForeignKey( CrontabSchedule, @@ -166,19 +196,60 @@ class DjCeleryPeriodicTask(models.Model): on_delete=models.CASCADE, help_text=_("Use one of interval/crontab"), ) - args = models.TextField(_("Arguments"), blank=True, default="[]", help_text=_("JSON encoded positional arguments"),) + args = models.TextField( + _("Arguments"), + blank=True, + default="[]", + help_text=_("JSON encoded positional arguments"), + ) kwargs = models.TextField( - _("Keyword arguments"), blank=True, default="{}", help_text=_("JSON encoded keyword arguments"), + _("Keyword arguments"), + blank=True, + default="{}", + help_text=_("JSON encoded keyword arguments"), ) queue = models.CharField( - _("queue"), max_length=200, blank=True, null=True, default=None, help_text=_("Queue defined in CELERY_QUEUES"), + _("queue"), + max_length=200, + blank=True, + null=True, + default=None, + help_text=_("Queue defined in CELERY_QUEUES"), + ) + exchange = models.CharField( + _("exchange"), + max_length=200, + blank=True, + null=True, + default=None, + ) + routing_key = models.CharField( + _("routing key"), + max_length=200, + blank=True, + null=True, + default=None, + ) + expires = models.DateTimeField( + _("expires"), + blank=True, + null=True, + ) + enabled = models.BooleanField( + _("enabled"), + default=True, + ) + last_run_at = models.DateTimeField( + auto_now=False, + auto_now_add=False, + editable=False, + blank=True, + null=True, + ) + total_run_count = models.PositiveIntegerField( + default=0, + editable=False, ) - exchange = models.CharField(_("exchange"), max_length=200, blank=True, null=True, default=None,) - routing_key = models.CharField(_("routing key"), max_length=200, blank=True, null=True, default=None,) - expires = models.DateTimeField(_("expires"), blank=True, null=True,) - enabled = models.BooleanField(_("enabled"), default=True,) - last_run_at = models.DateTimeField(auto_now=False, auto_now_add=False, editable=False, blank=True, null=True,) - total_run_count = models.PositiveIntegerField(default=0, editable=False,) date_changed = models.DateTimeField(auto_now=True) description = models.TextField(_("description"), blank=True) diff --git a/pipeline/contrib/periodic_task/signals/__init__.py b/pipeline/contrib/periodic_task/signals/__init__.py index 0b24ad78b6..830f49ce6f 100644 --- a/pipeline/contrib/periodic_task/signals/__init__.py +++ b/pipeline/contrib/periodic_task/signals/__init__.py @@ -13,6 +13,6 @@ from django.dispatch import Signal -pre_periodic_task_start = Signal(providing_args=["periodic_task", "pipeline_instance"]) -post_periodic_task_start = Signal(providing_args=["periodic_task", "pipeline_instance"]) -periodic_task_start_failed = Signal(providing_args=["periodic_task", "history"]) +pre_periodic_task_start = Signal(["periodic_task", "pipeline_instance"]) +post_periodic_task_start = Signal(["periodic_task", "pipeline_instance"]) +periodic_task_start_failed = Signal(["periodic_task", "history"]) diff --git a/pipeline/contrib/periodic_task/tasks.py b/pipeline/contrib/periodic_task/tasks.py index f7bd50ffbc..c8104352a5 100644 --- a/pipeline/contrib/periodic_task/tasks.py +++ b/pipeline/contrib/periodic_task/tasks.py @@ -16,7 +16,7 @@ import traceback import pytz -from celery import task +from celery import current_app from django.utils import timezone from pipeline.contrib.periodic_task import signals @@ -27,7 +27,7 @@ logger = logging.getLogger("celery") -@task(ignore_result=True) +@current_app.task(ignore_result=True) def periodic_task_start(*args, **kwargs): try: periodic_task = PeriodicTask.objects.get(id=kwargs["period_task_id"]) diff --git a/pipeline/contrib/statistics/models.py b/pipeline/contrib/statistics/models.py index d575d8c451..2192153455 100644 --- a/pipeline/contrib/statistics/models.py +++ b/pipeline/contrib/statistics/models.py @@ -12,7 +12,7 @@ """ from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ class ComponentInTemplate(models.Model): diff --git a/pipeline/core/flow/activity/service_activity.py b/pipeline/core/flow/activity/service_activity.py index 0f3b21528b..e2285ba594 100644 --- a/pipeline/core/flow/activity/service_activity.py +++ b/pipeline/core/flow/activity/service_activity.py @@ -14,10 +14,15 @@ from abc import ABCMeta, abstractmethod from copy import deepcopy -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from pipeline.core.flow.activity.base import Activity -from pipeline.core.flow.io import BooleanItemSchema, InputItem, IntItemSchema, OutputItem +from pipeline.core.flow.io import ( + BooleanItemSchema, + InputItem, + IntItemSchema, + OutputItem, +) from pipeline.utils.utils import convert_bytes_to_str @@ -227,13 +232,13 @@ def next(self): class DefaultIntervalGenerator(AbstractIntervalGenerator): def next(self): super(DefaultIntervalGenerator, self).next() - return self.count ** 2 + return self.count**2 class SquareIntervalGenerator(AbstractIntervalGenerator): def next(self): super(SquareIntervalGenerator, self).next() - return self.count ** 2 + return self.count**2 class NullIntervalGenerator(AbstractIntervalGenerator): diff --git a/pipeline/core/flow/io.py b/pipeline/core/flow/io.py index 322d96bc86..d4c4aec196 100644 --- a/pipeline/core/flow/io.py +++ b/pipeline/core/flow/io.py @@ -12,7 +12,7 @@ """ import abc -from collections import Mapping +from collections.abc import Mapping class DataItem(object, metaclass=abc.ABCMeta): diff --git a/pipeline/core/flow/signals.py b/pipeline/core/flow/signals.py index a6093eb35a..0dc7514bd2 100644 --- a/pipeline/core/flow/signals.py +++ b/pipeline/core/flow/signals.py @@ -13,4 +13,4 @@ from django.dispatch import Signal -post_new_end_event_register = Signal(providing_args=["node_type", "node_cls"]) +post_new_end_event_register = Signal(["node_type", "node_cls"]) diff --git a/pipeline/core/signals/__init__.py b/pipeline/core/signals/__init__.py index 0a35e84a34..90f9a80285 100644 --- a/pipeline/core/signals/__init__.py +++ b/pipeline/core/signals/__init__.py @@ -13,4 +13,4 @@ from django.dispatch import Signal -pre_variable_register = Signal(providing_args=["variable_code"]) +pre_variable_register = Signal(["variable_code"]) diff --git a/pipeline/django_signal_valve/models.py b/pipeline/django_signal_valve/models.py index 75dcdb1b65..cccf3e9523 100644 --- a/pipeline/django_signal_valve/models.py +++ b/pipeline/django_signal_valve/models.py @@ -15,7 +15,7 @@ import zlib from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ class IOField(models.BinaryField): diff --git a/pipeline/engine/admin.py b/pipeline/engine/admin.py index 9ff0096a63..26ae9a6c75 100644 --- a/pipeline/engine/admin.py +++ b/pipeline/engine/admin.py @@ -12,7 +12,7 @@ """ from django.contrib import admin -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from pipeline.engine import models from pipeline.engine.conf.function_switch import FREEZE_ENGINE diff --git a/pipeline/engine/conf/function_switch.py b/pipeline/engine/conf/function_switch.py index c9cd01d0e2..30f32ed63e 100644 --- a/pipeline/engine/conf/function_switch.py +++ b/pipeline/engine/conf/function_switch.py @@ -11,7 +11,7 @@ specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ FREEZE_ENGINE = "FREEZE_ENGINE" diff --git a/pipeline/engine/models/core.py b/pipeline/engine/models/core.py index 925f40d788..9db350e902 100644 --- a/pipeline/engine/models/core.py +++ b/pipeline/engine/models/core.py @@ -15,10 +15,10 @@ import logging import traceback -from celery.task.control import revoke +from celery import current_app from django.db import models, transaction from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from pipeline.conf import settings as pipeline_settings from pipeline.constants import PIPELINE_DEFAULT_PRIORITY @@ -621,7 +621,13 @@ class NodeRelationship(models.Model): objects = RelationshipManager() def __unicode__(self): - return str("#{} -({})-> #{}".format(self.ancestor_id, self.distance, self.descendant_id,)) + return str( + "#{} -({})-> #{}".format( + self.ancestor_id, + self.distance, + self.descendant_id, + ) + ) class StatusManager(models.Manager): @@ -1164,7 +1170,7 @@ def start_task(self, process_id, start_func, kwargs): def revoke(self, process_id, kill=False): task = self.get(process_id=process_id) kwargs = {} if not kill else {"signal": "SIGKILL"} - revoke(task.celery_task_id, terminate=True, **kwargs) + current_app.control.revoke(task.celery_task_id, terminate=True, **kwargs) self.destroy(process_id) @@ -1221,7 +1227,7 @@ def start_task(self, node_id, start_func, kwargs): def revoke(self, node_id): task = self.get(node_id=node_id) - revoke(task.celery_task_id) + current_app.control.revoke(task.celery_task_id) self.destroy(node_id) diff --git a/pipeline/engine/models/data.py b/pipeline/engine/models/data.py index 94c5fa3085..a1d3d0a47d 100644 --- a/pipeline/engine/models/data.py +++ b/pipeline/engine/models/data.py @@ -12,7 +12,7 @@ """ from django.db import models, transaction -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from pipeline.engine.models.fields import IOField diff --git a/pipeline/engine/models/function.py b/pipeline/engine/models/function.py index 9965edd9e5..97dc7e2a9b 100644 --- a/pipeline/engine/models/function.py +++ b/pipeline/engine/models/function.py @@ -15,7 +15,7 @@ import traceback from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from pipeline.engine.conf import function_switch diff --git a/pipeline/engine/signals/__init__.py b/pipeline/engine/signals/__init__.py index 211d0a9684..6c4681d187 100644 --- a/pipeline/engine/signals/__init__.py +++ b/pipeline/engine/signals/__init__.py @@ -13,23 +13,23 @@ from django.dispatch import Signal -pipeline_ready = Signal(providing_args=["process_id"]) -pipeline_end = Signal(providing_args=["root_pipeline_id"]) -pipeline_revoke = Signal(providing_args=["root_pipeline_id"]) -child_process_ready = Signal(providing_args=["child_id"]) -process_ready = Signal(providing_args=["parent_id", "current_node_id", "call_from_child"]) -batch_process_ready = Signal(providing_args=["process_id_list", "pipeline_id"]) -wake_from_schedule = Signal(providing_args=["process_id, activity_id"]) -schedule_ready = Signal(providing_args=["schedule_id", "countdown", "process_id", "data_id"]) -process_unfreeze = Signal(providing_args=["process_id"]) +pipeline_ready = Signal(["process_id"]) +pipeline_end = Signal(["root_pipeline_id"]) +pipeline_revoke = Signal(["root_pipeline_id"]) +child_process_ready = Signal(["child_id"]) +process_ready = Signal(["parent_id", "current_node_id", "call_from_child"]) +batch_process_ready = Signal(["process_id_list", "pipeline_id"]) +wake_from_schedule = Signal(["process_id, activity_id"]) +schedule_ready = Signal(["schedule_id", "countdown", "process_id", "data_id"]) +process_unfreeze = Signal(["process_id"]) # activity failed signal -activity_failed = Signal(providing_args=["pipeline_id", "pipeline_activity_id", "subprocess_id_stack"]) +activity_failed = Signal(["pipeline_id", "pipeline_activity_id", "subprocess_id_stack"]) # signal for developer (do not use valve to pass them!) -service_schedule_fail = Signal(providing_args=["activity_shell", "schedule_service", "ex_data"]) -service_schedule_success = Signal(providing_args=["activity_shell", "schedule_service"]) -node_skip_call = Signal(providing_args=["process", "node"]) -node_retry_ready = Signal(providing_args=["process", "node"]) +service_schedule_fail = Signal(["activity_shell", "schedule_service", "ex_data"]) +service_schedule_success = Signal(["activity_shell", "schedule_service"]) +node_skip_call = Signal(["process", "node"]) +node_retry_ready = Signal(["process", "node"]) -service_activity_timeout_monitor_start = Signal(providing_args=["node_id", "version", "root_pipeline_id", "countdown"]) -service_activity_timeout_monitor_end = Signal(providing_args=["node_id", "version"]) +service_activity_timeout_monitor_start = Signal(["node_id", "version", "root_pipeline_id", "countdown"]) +service_activity_timeout_monitor_end = Signal(["node_id", "version"]) diff --git a/pipeline/engine/tasks.py b/pipeline/engine/tasks.py index c0db0d463c..2c51589152 100644 --- a/pipeline/engine/tasks.py +++ b/pipeline/engine/tasks.py @@ -13,8 +13,7 @@ import logging -from celery import task -from celery.decorators import periodic_task +from celery import current_app from celery.schedules import crontab from pipeline.conf import default_settings @@ -22,12 +21,18 @@ from pipeline.engine import api, signals, states from pipeline.engine.core import runtime, schedule from pipeline.engine.health import zombie -from pipeline.engine.models import NodeCeleryTask, NodeRelationship, PipelineProcess, ProcessCeleryTask, Status +from pipeline.engine.models import ( + NodeCeleryTask, + NodeRelationship, + PipelineProcess, + ProcessCeleryTask, + Status, +) logger = logging.getLogger("celery") -@task(ignore_result=True) +@current_app.task(ignore_result=True) def process_unfreeze(process_id): process = PipelineProcess.objects.get(id=process_id) if not process.is_alive: @@ -37,7 +42,7 @@ def process_unfreeze(process_id): runtime.run_loop(process) -@task(ignore_result=True) +@current_app.task(ignore_result=True) def start(process_id): process = PipelineProcess.objects.get(id=process_id) if not process.is_alive: @@ -56,7 +61,7 @@ def start(process_id): runtime.run_loop(process) -@task(ignore_result=True) +@current_app.task(ignore_result=True) def dispatch(child_id): process = PipelineProcess.objects.get(id=child_id) if not process.is_alive: @@ -66,7 +71,7 @@ def dispatch(child_id): runtime.run_loop(process) -@task(ignore_result=True) +@current_app.task(ignore_result=True) def process_wake_up(process_id, current_node_id=None, call_from_child=False): process = PipelineProcess.objects.get(id=process_id) if not process.is_alive: @@ -92,7 +97,7 @@ def process_wake_up(process_id, current_node_id=None, call_from_child=False): runtime.run_loop(process) -@task(ignore_result=True) +@current_app.task(ignore_result=True) def wake_up(process_id): process = PipelineProcess.objects.get(id=process_id) if not process.is_alive: @@ -103,7 +108,7 @@ def wake_up(process_id): runtime.run_loop(process) -@task(ignore_result=True) +@current_app.task(ignore_result=True) def batch_wake_up(process_id_list, pipeline_id): action_result = Status.objects.transit(pipeline_id, to_state=states.RUNNING, is_pipeline=True) if not action_result.result: @@ -114,7 +119,7 @@ def batch_wake_up(process_id_list, pipeline_id): ProcessCeleryTask.objects.bind(process_id, task_id) -@task(ignore_result=True) +@current_app.task(ignore_result=True) def wake_from_schedule(process_id, service_act_id): process = PipelineProcess.objects.get(id=process_id) process.wake_up() @@ -124,12 +129,12 @@ def wake_from_schedule(process_id, service_act_id): runtime.run_loop(process) -@task(ignore_result=True) +@current_app.task(ignore_result=True) def service_schedule(process_id, schedule_id, data_id=None): schedule.schedule(process_id, schedule_id, data_id) -@task(ignore_result=True) +@current_app.task(ignore_result=True) def node_timeout_check(node_id, version, root_pipeline_id): NodeCeleryTask.objects.destroy(node_id) state = Status.objects.state_for(node_id, version=version, may_not_exist=True) @@ -144,7 +149,7 @@ def node_timeout_check(node_id, version, root_pipeline_id): logger.warning("node {} - {} timeout kill failed".format(node_id, version)) -@periodic_task(run_every=(crontab(**default_settings.ENGINE_ZOMBIE_PROCESS_HEAL_CRON)), ignore_result=True) +@current_app.task(run_every=(crontab(**default_settings.ENGINE_ZOMBIE_PROCESS_HEAL_CRON)), ignore_result=True) def heal_zombie_process(): logger.info("Zombie process heal start") diff --git a/pipeline/log/models.py b/pipeline/log/models.py index 18e17d42a1..f9fde59f66 100644 --- a/pipeline/log/models.py +++ b/pipeline/log/models.py @@ -13,7 +13,7 @@ from django.db import models from django.utils import timezone -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ class LogEntryManager(models.Manager): diff --git a/pipeline/log/tasks.py b/pipeline/log/tasks.py index 28dcae11a0..cfa2acd9f1 100644 --- a/pipeline/log/tasks.py +++ b/pipeline/log/tasks.py @@ -13,10 +13,10 @@ import logging -from celery.decorators import periodic_task from celery.schedules import crontab from django.conf import settings +from pipeline.contrib.celery_tools.periodic import periodic_task from pipeline.log.models import LogEntry logger = logging.getLogger(__name__) diff --git a/pipeline/models.py b/pipeline/models.py index 489c8a1f17..b3b9d0f646 100644 --- a/pipeline/models.py +++ b/pipeline/models.py @@ -21,7 +21,7 @@ from django.db import models, transaction from django.utils import timezone from django.utils.module_loading import import_string -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from pipeline.conf import settings from pipeline.constants import PIPELINE_DEFAULT_PRIORITY diff --git a/pipeline/validators/connection.py b/pipeline/validators/connection.py index 238979dd68..6cf53bdc01 100644 --- a/pipeline/validators/connection.py +++ b/pipeline/validators/connection.py @@ -11,7 +11,7 @@ specific language governing permissions and limitations under the License. """ -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from pipeline.core.constants import PE from pipeline.exceptions import ConnectionValidateError diff --git a/pipeline/validators/gateway.py b/pipeline/validators/gateway.py index de856cd485..1c284c84ed 100644 --- a/pipeline/validators/gateway.py +++ b/pipeline/validators/gateway.py @@ -13,7 +13,7 @@ import queue -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from pipeline import exceptions from pipeline.core.constants import PE diff --git a/pipeline/variable_framework/models.py b/pipeline/variable_framework/models.py index e1557de4fe..86f5088893 100644 --- a/pipeline/variable_framework/models.py +++ b/pipeline/variable_framework/models.py @@ -12,7 +12,7 @@ """ from django.db import models -from django.utils.translation import ugettext_lazy as _ +from django.utils.translation import gettext_lazy as _ from pipeline.core.data.library import VariableLibrary diff --git a/requirements.txt b/requirements.txt index 7b019f3436..7849d3e157 100644 --- a/requirements.txt +++ b/requirements.txt @@ -2,62 +2,61 @@ # -*- coding: utf-8 -*- # 运行时依赖安装 -wheel==0.37.0 +wheel==0.43.0 -blueapps[opentelemetry,bkcrypto]==4.14.0 +blueapps[opentelemetry,bkcrypto]==4.15.3 -Django==3.2.4 +Django==4.2 -requests==2.22.0 -celery==4.4.7 -django-celery-beat==2.2.0 -django-celery-results==2.0.1 +requests==2.31.0 +celery==5.2.7 +django-celery-beat==2.7.0 +django-celery-results==2.5.1 celery-redbeat==2.0.0 -djangorestframework==3.12.4 +djangorestframework==3.15.1 # django-utils-six==2.0 django-filter==21.1 -django-mysql==3.3.0 +# django-mysql==3.3.0 django-redis==5.3.0 -Mako==1.0.6 -MarkupSafe==1.1.1 -pyCryptodome==3.9.7 -pyOpenSSL==21.0.0 -pynacl==1.4.0 -gunicorn==19.6.0 -arrow==0.15.4 -PyJWT==1.7.1 +Mako==1.3.2 +MarkupSafe==2.0.1 +pyCryptodome==3.20.0 +pyOpenSSL==24.1.0 +pynacl==1.5.0 +gunicorn==23.0.0 +arrow==1.3.0 +PyJWT==2.8.0 whitenoise==5.2.0 -pymysql==1.0.2 -eventlet==0.25.1 -gevent==1.2.2 -paramiko==2.7.2 -jinja2==2.11.1 +pymysql==1.1.1 +eventlet==0.35.2 +gevent==24.2.1 +paramiko==3.4.0 +jinja2==3.1.3 jinja2schema==0.1.4 PyYAML==6.0 python-json-logger==0.1.7 packaging==21.3 -django-dbconn-retry==0.1.5 +django-dbconn-retry==0.1.8 wrapt==1.15.0 -drf-yasg==1.20.3 +drf-yasg==1.21.8 dacite==1.8.1 # pipeline -ujson==1.35 -pyparsing==2.2.0 -redis==3.5.3 -redis-py-cluster==2.1.3 -django-timezone-field==4.1.0 +ujson==5.9.0 +pyparsing==3.2.0 +redis==5.0.3 +django-timezone-field==7.0 mock==2.0.0 factory_boy==2.11.1 boto3==1.9.130 -rsa==4.0 -dnspython==1.16.0 +rsa==4.9 +dnspython==2.7.0 django-cors-headers==3.5.0 # 版本日志 -mistune==0.8.4 -django-versionlog==1.6.0 +mistune==3.0.2 +django-versionlog==1.8.1 # 腾讯云SDK tencentcloud-sdk-python==3.0.1210 @@ -75,39 +74,39 @@ raven==6.1.0 # for apm ddtrace==0.14.1 -bkstorages==1.0.1 +bkstorages==2.0.0 ruamel.yaml==0.17.16 kazoo==2.8.0 -asgiref==3.4.1 +# asgiref==3.4.1 -asyncssh==2.8.1 +asyncssh==2.14.2 # prometheus django-prometheus==2.2.0 # opentelemetry -protobuf==3.19.4 -opentelemetry-api==1.11.1 -opentelemetry-sdk==1.11.1 +protobuf==3.19.6 +opentelemetry-api==1.23.0 +opentelemetry-sdk==1.23.0 opentelemetry-exporter-otlp==1.11.1 opentelemetry-exporter-jaeger==1.11.1 opentelemetry-exporter-jaeger-proto-grpc==1.11.1 opentelemetry-exporter-jaeger-thrift==1.11.1 -opentelemetry-instrumentation==0.30b1 -opentelemetry-instrumentation-celery==0.30b1 -opentelemetry-instrumentation-django==0.30b1 -opentelemetry-instrumentation-dbapi==0.30b1 -opentelemetry-instrumentation-redis==0.30b1 -opentelemetry-instrumentation-logging==0.30b1 -opentelemetry-instrumentation-requests==0.30b1 +opentelemetry-instrumentation==0.44b0 +opentelemetry-instrumentation-celery==0.44b0 +opentelemetry-instrumentation-django==0.44b0 +opentelemetry-instrumentation-dbapi==0.44b0 +opentelemetry-instrumentation-redis==0.44b0 +opentelemetry-instrumentation-logging==0.44b0 +opentelemetry-instrumentation-requests==0.44b0 bk-monitor-report==1.2.1 apigw-manager[cryptography]==1.1.8 -cryptography==3.3.2 +cryptography==42.0.5 # notice -bk-notice-sdk==1.2.0 +bk-notice-sdk==1.3.2 diff --git a/support-files/kubernetes/images/family_bucket/Dockerfile b/support-files/kubernetes/images/family_bucket/Dockerfile index 2762d7923e..9e290e1dc0 100644 --- a/support-files/kubernetes/images/family_bucket/Dockerfile +++ b/support-files/kubernetes/images/family_bucket/Dockerfile @@ -11,7 +11,7 @@ RUN npm install RUN npm run build -FROM python:3.6.12-slim-buster AS base +FROM python:3.11.10-slim-bookworm AS base ENV LC_ALL=C.UTF-8 \ LANG=C.UTF-8 @@ -28,11 +28,7 @@ ENV PATH="/venv/bin:$PATH" RUN set -ex && \ chmod 1777 /tmp && \ - rm /etc/apt/sources.list && \ - echo "deb https://mirrors.cloud.tencent.com/debian buster main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb https://mirrors.cloud.tencent.com/debian buster-updates main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src https://mirrors.cloud.tencent.com/debian buster main contrib non-free" >> /etc/apt/sources.list && \ - echo "deb-src https://mirrors.cloud.tencent.com/debian buster-updates main contrib non-free" >> /etc/apt/sources.list + sed -i 's/deb.debian.org/mirrors.tencent.com/g' /etc/apt/sources.list.d/debian.sources RUN set -ex && mkdir ~/.pip && printf '[global]\nindex-url = https://mirrors.tencent.com/pypi/simple/' > ~/.pip/pip.conf @@ -46,7 +42,7 @@ WORKDIR / RUN set -ex && \ apt-get update && \ apt-get install -y --no-install-recommends \ - gcc gettext mariadb-client libmariadbclient-dev default-libmysqlclient-dev && \ + gcc gettext mariadb-client libmariadb-dev default-libmysqlclient-dev && \ rm -rf /var/lib/apt/lists/* COPY requirements.txt / diff --git a/support-files/supervisord.conf.bak b/support-files/supervisord.conf.bak index 4985da9fbd..ff94ab0be4 100644 --- a/support-files/supervisord.conf.bak +++ b/support-files/supervisord.conf.bak @@ -22,7 +22,7 @@ autorestart = true environment = {{.environment}} [program: {{.app_code}}_celery] -command = /cache/.bk/env/bin/python {{.app_container_path}}code/manage.py celery worker -n {{.node_name}}_{{.app_code}} -Q saas -c 8 -l INFO --maxtasksperchild=50 +command = /cache/.bk/env/bin/python {{.app_container_path}}code/manage.py celery worker -n {{.node_name}}_{{.app_code}} -Q saas -c 8 -l INFO --max-tasks-per-child=50 directory = {{.app_container_path}}code/ stdout_logfile = {{.app_container_path}}logs/{{.app_code}}/celery.log redirect_stderr = true diff --git a/support-files/templates/#etc#supervisor-bknodeman-nodeman.conf b/support-files/templates/#etc#supervisor-bknodeman-nodeman.conf index d55d3811f4..cd091a05ec 100644 --- a/support-files/templates/#etc#supervisor-bknodeman-nodeman.conf +++ b/support-files/templates/#etc#supervisor-bknodeman-nodeman.conf @@ -79,7 +79,7 @@ redirect_stderr=true directory=__BK_HOME__/bknodeman/nodeman [program:nodeman_celery_default] -command=/bin/bash -c "sleep 10 && source bin/environ.sh && exec celery -A apps.backend worker -Q default -n default_%(process_num)02d@%%h -c 3 --maxtasksperchild=50 -O fair --time-limit=1800" +command=/bin/bash -c "sleep 10 && source bin/environ.sh && exec celery -A apps.backend worker -Q default -n default_%(process_num)02d@%%h -c 3 --max-tasks-per-child=50 -O fair --time-limit=1800" numprocs=1 process_name = %(program_name)s_%(process_num)02d autostart=true @@ -105,7 +105,7 @@ redirect_stderr=true directory=__BK_HOME__/bknodeman/nodeman [program:nodeman_celery_backend_additional] -command=/bin/bash -c "sleep 10 && source bin/environ.sh && exec celery -A apps.backend worker -Q backend_additional_task -n baworker_%(process_num)02d@%%h -c 10 -O fair --time-limit=1800 --maxtasksperchild=50" +command=/bin/bash -c "sleep 10 && source bin/environ.sh && exec celery -A apps.backend worker -Q backend_additional_task -n baworker_%(process_num)02d@%%h -c 10 -O fair --time-limit=1800 --max-tasks-per-child=50" numprocs=1 process_name = %(program_name)s_%(process_num)02d autostart=true diff --git a/urls.py b/urls.py index 4c370edc3a..9859aa5a25 100644 --- a/urls.py +++ b/urls.py @@ -11,13 +11,11 @@ from bk_notice_sdk import config as notice_config from django.conf import settings -from django.conf.urls import include, url from django.contrib import admin -from django.urls import re_path +from django.urls import include, re_path from drf_yasg import openapi from drf_yasg.views import get_schema_view from rest_framework import permissions - from version_log import config schema_view = get_schema_view( @@ -34,13 +32,13 @@ ) urlpatterns = [ - url(r"^admin_nodeman/", admin.site.urls), - url(r"^account/", include("blueapps.account.urls")), - url(r"^backend/", include("apps.backend.urls")), - url(r"^core/", include("apps.core.urls")), - url(r"^", include("apps.node_man.urls")), - url(r"^{}".format(config.ENTRANCE_URL), include("version_log.urls", namespace="version_log")), - url(r"^{}".format(notice_config.ENTRANCE_URL), include(("bk_notice_sdk.urls", "notice"), namespace="notice")), + re_path(r"^admin_nodeman/", admin.site.urls), + re_path(r"^account/", include("blueapps.account.urls")), + re_path(r"^backend/", include("apps.backend.urls")), + re_path(r"^core/", include("apps.core.urls")), + re_path(r"^", include("apps.node_man.urls")), + re_path(r"^{}".format(config.ENTRANCE_URL), include("version_log.urls", namespace="version_log")), + re_path(r"^{}".format(notice_config.ENTRANCE_URL), include(("bk_notice_sdk.urls", "notice"), namespace="notice")), ] if settings.ENVIRONMENT not in ["production", "prod"]: