[Linting] Backport Ruff upgrade [1.6.x] (mlrun#5253)

jond01 · Mar 6, 2024 · e1db48c · e1db48c
1 parent e45c26b
commit e1db48c
Show file tree

Hide file tree

Showing 93 changed files with 385 additions and 319 deletions.
diff --git a/automation/system_test/prepare.py b/automation/system_test/prepare.py
@@ -394,9 +394,9 @@ def _enrich_env(self):
         spark_service_name = self._get_service_name("app=spark,component=spark-master")
         self._env_config["MLRUN_IGUAZIO_API_URL"] = f"https://{api_url_host}"
         self._env_config["V3IO_FRAMESD"] = f"https://{framesd_host}"
-        self._env_config[
-            "MLRUN_SYSTEM_TESTS_DEFAULT_SPARK_SERVICE"
-        ] = spark_service_name
+        self._env_config["MLRUN_SYSTEM_TESTS_DEFAULT_SPARK_SERVICE"] = (
+            spark_service_name
+        )
         self._env_config["V3IO_API"] = f"https://{v3io_api_host}"
         self._env_config["MLRUN_DBPATH"] = f"https://{mlrun_api_url}"
 

diff --git a/dev-requirements.txt b/dev-requirements.txt
@@ -3,7 +3,7 @@ twine~=3.1
 build~=1.0
 
 # formatting & linting
-ruff~=0.1.8
+ruff~=0.3.0
 import-linter~=1.8
 
 # testing

diff --git a/mlrun/config.py b/mlrun/config.py
@@ -960,10 +960,10 @@ def get_default_function_pod_resources(
             with_gpu = (
                 with_gpu_requests if requirement == "requests" else with_gpu_limits
             )
-            resources[
-                requirement
-            ] = self.get_default_function_pod_requirement_resources(
-                requirement, with_gpu
+            resources[requirement] = (
+                self.get_default_function_pod_requirement_resources(
+                    requirement, with_gpu
+                )
             )
         return resources
 

diff --git a/mlrun/datastore/azure_blob.py b/mlrun/datastore/azure_blob.py
@@ -175,9 +175,9 @@ def get_spark_options(self):
 
         if "client_secret" in st or "client_id" in st or "tenant_id" in st:
             res[f"spark.hadoop.fs.azure.account.auth.type.{host}"] = "OAuth"
-            res[
-                f"spark.hadoop.fs.azure.account.oauth.provider.type.{host}"
-            ] = "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider"
+            res[f"spark.hadoop.fs.azure.account.oauth.provider.type.{host}"] = (
+                "org.apache.hadoop.fs.azurebfs.oauth2.ClientCredsTokenProvider"
+            )
             if "client_id" in st:
                 res[f"spark.hadoop.fs.azure.account.oauth2.client.id.{host}"] = st[
                     "client_id"
@@ -188,14 +188,14 @@ def get_spark_options(self):
                 ]
             if "tenant_id" in st:
                 tenant_id = st["tenant_id"]
-                res[
-                    f"spark.hadoop.fs.azure.account.oauth2.client.endpoint.{host}"
-                ] = f"https://login.microsoftonline.com/{tenant_id}/oauth2/token"
+                res[f"spark.hadoop.fs.azure.account.oauth2.client.endpoint.{host}"] = (
+                    f"https://login.microsoftonline.com/{tenant_id}/oauth2/token"
+                )
 
         if "sas_token" in st:
             res[f"spark.hadoop.fs.azure.account.auth.type.{host}"] = "SAS"
-            res[
-                f"spark.hadoop.fs.azure.sas.token.provider.type.{host}"
-            ] = "org.apache.hadoop.fs.azurebfs.sas.FixedSASTokenProvider"
+            res[f"spark.hadoop.fs.azure.sas.token.provider.type.{host}"] = (
+                "org.apache.hadoop.fs.azurebfs.sas.FixedSASTokenProvider"
+            )
             res[f"spark.hadoop.fs.azure.sas.fixed.token.{host}"] = st["sas_token"]
         return res
diff --git a/mlrun/datastore/google_cloud_storage.py b/mlrun/datastore/google_cloud_storage.py
@@ -147,13 +147,13 @@ def get_spark_options(self):
             if "project_id" in credentials:
                 res["spark.hadoop.fs.gs.project.id"] = credentials["project_id"]
             if "private_key_id" in credentials:
-                res[
-                    "spark.hadoop.fs.gs.auth.service.account.private.key.id"
-                ] = credentials["private_key_id"]
+                res["spark.hadoop.fs.gs.auth.service.account.private.key.id"] = (
+                    credentials["private_key_id"]
+                )
             if "private_key" in credentials:
-                res[
-                    "spark.hadoop.fs.gs.auth.service.account.private.key"
-                ] = credentials["private_key"]
+                res["spark.hadoop.fs.gs.auth.service.account.private.key"] = (
+                    credentials["private_key"]
+                )
             if "client_email" in credentials:
                 res["spark.hadoop.fs.gs.auth.service.account.email"] = credentials[
                     "client_email"

diff --git a/mlrun/db/httpdb.py b/mlrun/db/httpdb.py
@@ -1137,17 +1137,17 @@ def list_runtime_resources(
             structured_dict = {}
             for project, job_runtime_resources_map in response.json().items():
                 for job_id, runtime_resources in job_runtime_resources_map.items():
-                    structured_dict.setdefault(project, {})[
-                        job_id
-                    ] = mlrun.common.schemas.RuntimeResources(**runtime_resources)
+                    structured_dict.setdefault(project, {})[job_id] = (
+                        mlrun.common.schemas.RuntimeResources(**runtime_resources)
+                    )
             return structured_dict
         elif group_by == mlrun.common.schemas.ListRuntimeResourcesGroupByField.project:
             structured_dict = {}
             for project, kind_runtime_resources_map in response.json().items():
                 for kind, runtime_resources in kind_runtime_resources_map.items():
-                    structured_dict.setdefault(project, {})[
-                        kind
-                    ] = mlrun.common.schemas.RuntimeResources(**runtime_resources)
+                    structured_dict.setdefault(project, {})[kind] = (
+                        mlrun.common.schemas.RuntimeResources(**runtime_resources)
+                    )
             return structured_dict
         else:
             raise NotImplementedError(
@@ -1206,9 +1206,9 @@ def delete_runtime_resources(
         structured_dict = {}
         for project, kind_runtime_resources_map in response.json().items():
             for kind, runtime_resources in kind_runtime_resources_map.items():
-                structured_dict.setdefault(project, {})[
-                    kind
-                ] = mlrun.common.schemas.RuntimeResources(**runtime_resources)
+                structured_dict.setdefault(project, {})[kind] = (
+                    mlrun.common.schemas.RuntimeResources(**runtime_resources)
+                )
         return structured_dict
 
     def create_schedule(

diff --git a/mlrun/execution.py b/mlrun/execution.py
@@ -559,9 +559,9 @@ def log_iteration_results(self, best, summary: list, task: dict, commit=False):
             for k, v in get_in(task, ["status", "results"], {}).items():
                 self._results[k] = v
             for artifact in get_in(task, ["status", run_keys.artifacts], []):
-                self._artifacts_manager.artifacts[
-                    artifact["metadata"]["key"]
-                ] = artifact
+                self._artifacts_manager.artifacts[artifact["metadata"]["key"]] = (
+                    artifact
+                )
                 self._artifacts_manager.link_artifact(
                     self.project,
                     self.name,

diff --git a/mlrun/frameworks/tf_keras/callbacks/logging_callback.py b/mlrun/frameworks/tf_keras/callbacks/logging_callback.py
@@ -389,9 +389,9 @@ def _add_auto_hyperparameters(self):
         ):
             try:
                 self._get_hyperparameter(key_chain=learning_rate_key_chain)
-                self._dynamic_hyperparameters_keys[
-                    learning_rate_key
-                ] = learning_rate_key_chain
+                self._dynamic_hyperparameters_keys[learning_rate_key] = (
+                    learning_rate_key_chain
+                )
             except (KeyError, IndexError, ValueError):
                 pass
 

diff --git a/mlrun/frameworks/tf_keras/model_handler.py b/mlrun/frameworks/tf_keras/model_handler.py
@@ -263,13 +263,13 @@ def save(
         # Update the paths and log artifacts if context is available:
         if self._weights_file is not None:
             if self._context is not None:
-                artifacts[
-                    self._get_weights_file_artifact_name()
-                ] = self._context.log_artifact(
-                    self._weights_file,
-                    local_path=self._weights_file,
-                    artifact_path=output_path,
-                    db_key=False,
+                artifacts[self._get_weights_file_artifact_name()] = (
+                    self._context.log_artifact(
+                        self._weights_file,
+                        local_path=self._weights_file,
+                        artifact_path=output_path,
+                        db_key=False,
+                    )
                 )
 
         return artifacts if self._context is not None else None

diff --git a/mlrun/kfpops.py b/mlrun/kfpops.py
@@ -408,9 +408,9 @@ def mlrun_pipeline(
         cmd += ["--label", f"{label}={val}"]
     for output in outputs:
         cmd += ["-o", str(output)]
-        file_outputs[
-            output.replace(".", "_")
-        ] = f"/tmp/{output}"  # not using path.join to avoid windows "\"
+        file_outputs[output.replace(".", "_")] = (
+            f"/tmp/{output}"  # not using path.join to avoid windows "\"
+        )
     if project:
         cmd += ["--project", project]
     if handler:

diff --git a/mlrun/model_monitoring/api.py b/mlrun/model_monitoring/api.py
@@ -436,9 +436,9 @@ def _generate_model_endpoint(
         ] = possible_drift_threshold
 
     model_endpoint.spec.monitoring_mode = monitoring_mode
-    model_endpoint.status.first_request = (
-        model_endpoint.status.last_request
-    ) = datetime_now().isoformat()
+    model_endpoint.status.first_request = model_endpoint.status.last_request = (
+        datetime_now().isoformat()
+    )
     if sample_set_statistics:
         model_endpoint.status.feature_stats = sample_set_statistics
 
@@ -476,11 +476,11 @@ def trigger_drift_batch_job(
         db_session = mlrun.get_run_db()
 
     # Register the monitoring batch job (do nothing if already exist) and get the job function as a dictionary
-    batch_function_dict: typing.Dict[
-        str, typing.Any
-    ] = db_session.deploy_monitoring_batch_job(
-        project=project,
-        default_batch_image=default_batch_image,
+    batch_function_dict: typing.Dict[str, typing.Any] = (
+        db_session.deploy_monitoring_batch_job(
+            project=project,
+            default_batch_image=default_batch_image,
+        )
     )
 
     # Prepare current run params

diff --git a/mlrun/model_monitoring/stores/kv_model_endpoint_store.py b/mlrun/model_monitoring/stores/kv_model_endpoint_store.py
@@ -540,24 +540,24 @@ def validate_old_schema_fields(endpoint: dict):
             and endpoint[mlrun.common.schemas.model_monitoring.EventFieldType.METRICS]
             == "null"
         ):
-            endpoint[
-                mlrun.common.schemas.model_monitoring.EventFieldType.METRICS
-            ] = json.dumps(
-                {
-                    mlrun.common.schemas.model_monitoring.EventKeyMetrics.GENERIC: {
-                        mlrun.common.schemas.model_monitoring.EventLiveStats.LATENCY_AVG_1H: 0,
-                        mlrun.common.schemas.model_monitoring.EventLiveStats.PREDICTIONS_PER_SECOND: 0,
+            endpoint[mlrun.common.schemas.model_monitoring.EventFieldType.METRICS] = (
+                json.dumps(
+                    {
+                        mlrun.common.schemas.model_monitoring.EventKeyMetrics.GENERIC: {
+                            mlrun.common.schemas.model_monitoring.EventLiveStats.LATENCY_AVG_1H: 0,
+                            mlrun.common.schemas.model_monitoring.EventLiveStats.PREDICTIONS_PER_SECOND: 0,
+                        }
                     }
-                }
+                )
             )
         # Validate key `uid` instead of `endpoint_id`
         # For backwards compatibility reasons, we replace the `endpoint_id` with `uid` which is the updated key name
         if mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_ID in endpoint:
-            endpoint[
-                mlrun.common.schemas.model_monitoring.EventFieldType.UID
-            ] = endpoint[
-                mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_ID
-            ]
+            endpoint[mlrun.common.schemas.model_monitoring.EventFieldType.UID] = (
+                endpoint[
+                    mlrun.common.schemas.model_monitoring.EventFieldType.ENDPOINT_ID
+                ]
+            )
 
     @staticmethod
     def _encode_field(field: typing.Union[str, bytes]) -> bytes:

diff --git a/mlrun/model_monitoring/stores/sql_model_endpoint_store.py b/mlrun/model_monitoring/stores/sql_model_endpoint_store.py
@@ -31,7 +31,6 @@
 
 
 class SQLModelEndpointStore(ModelEndpointStore):
-
     """
     Handles the DB operations when the DB target is from type SQL. For the SQL operations, we use SQLAlchemy, a Python
     SQL toolkit that handles the communication with the database.  When using SQL for storing the model endpoints

diff --git a/mlrun/package/packagers/pandas_packagers.py b/mlrun/package/packagers/pandas_packagers.py
@@ -838,9 +838,9 @@ def _prepare_result(obj: Union[list, dict, tuple]) -> Any:
         """
         if isinstance(obj, dict):
             for key, value in obj.items():
-                obj[
-                    PandasDataFramePackager._prepare_result(obj=key)
-                ] = PandasDataFramePackager._prepare_result(obj=value)
+                obj[PandasDataFramePackager._prepare_result(obj=key)] = (
+                    PandasDataFramePackager._prepare_result(obj=value)
+                )
         elif isinstance(obj, list):
             for i, value in enumerate(obj):
                 obj[i] = PandasDataFramePackager._prepare_result(obj=value)

diff --git a/mlrun/runtimes/function.py b/mlrun/runtimes/function.py
@@ -432,15 +432,15 @@ def with_http(
                 raise ValueError(
                     "gateway timeout must be greater than the worker timeout"
                 )
-            annotations[
-                "nginx.ingress.kubernetes.io/proxy-connect-timeout"
-            ] = f"{gateway_timeout}"
-            annotations[
-                "nginx.ingress.kubernetes.io/proxy-read-timeout"
-            ] = f"{gateway_timeout}"
-            annotations[
-                "nginx.ingress.kubernetes.io/proxy-send-timeout"
-            ] = f"{gateway_timeout}"
+            annotations["nginx.ingress.kubernetes.io/proxy-connect-timeout"] = (
+                f"{gateway_timeout}"
+            )
+            annotations["nginx.ingress.kubernetes.io/proxy-read-timeout"] = (
+                f"{gateway_timeout}"
+            )
+            annotations["nginx.ingress.kubernetes.io/proxy-send-timeout"] = (
+                f"{gateway_timeout}"
+            )
 
         trigger = nuclio.HttpTrigger(
             workers=workers,

diff --git a/mlrun/runtimes/mpijob/abstract.py b/mlrun/runtimes/mpijob/abstract.py
@@ -196,13 +196,13 @@ def with_autotune(
         if steps_per_sample is not None:
             horovod_autotune_settings["autotune-steps-per-sample"] = steps_per_sample
         if bayes_opt_max_samples is not None:
-            horovod_autotune_settings[
-                "autotune-bayes-opt-max-samples"
-            ] = bayes_opt_max_samples
+            horovod_autotune_settings["autotune-bayes-opt-max-samples"] = (
+                bayes_opt_max_samples
+            )
         if gaussian_process_noise is not None:
-            horovod_autotune_settings[
-                "autotune-gaussian-process-noise"
-            ] = gaussian_process_noise
+            horovod_autotune_settings["autotune-gaussian-process-noise"] = (
+                gaussian_process_noise
+            )
 
         self.set_envs(horovod_autotune_settings)
 

diff --git a/mlrun/runtimes/pod.py b/mlrun/runtimes/pod.py
@@ -430,9 +430,9 @@ def enrich_resources_with_default_pod_resources(
                         )
                         is None
                     ):
-                        resources[resource_requirement][
-                            resource_type
-                        ] = default_resources[resource_requirement][resource_type]
+                        resources[resource_requirement][resource_type] = (
+                            default_resources[resource_requirement][resource_type]
+                        )
         # This enables the user to define that no defaults would be applied on the resources
         elif resources == {}:
             return resources

diff --git a/mlrun/runtimes/serving.py b/mlrun/runtimes/serving.py
@@ -523,9 +523,9 @@ def _deploy_function_refs(self, builder_env: dict = None):
             function_object.metadata.tag = self.metadata.tag
 
             function_object.metadata.labels = function_object.metadata.labels or {}
-            function_object.metadata.labels[
-                "mlrun/parent-function"
-            ] = self.metadata.name
+            function_object.metadata.labels["mlrun/parent-function"] = (
+                self.metadata.name
+            )
             function_object._is_child_function = True
             if not function_object.spec.graph:
                 # copy the current graph only if the child doesnt have a graph of his own

diff --git a/mlrun/runtimes/sparkjob/spark3job.py b/mlrun/runtimes/sparkjob/spark3job.py
@@ -345,9 +345,9 @@ def enrich_resources_with_default_pod_resources(
                         )
                         is None
                     ):
-                        resources[resource_requirement][
-                            resource_type
-                        ] = default_resources[resource_requirement][resource_type]
+                        resources[resource_requirement][resource_type] = (
+                            default_resources[resource_requirement][resource_type]
+                        )
         else:
             resources = default_resources
 

diff --git a/mlrun/utils/async_http.py b/mlrun/utils/async_http.py
@@ -139,9 +139,9 @@ async def _do_request(self) -> aiohttp.ClientResponse:
 
                 # enrich user agent
                 # will help traceability and debugging
-                headers[
-                    aiohttp.hdrs.USER_AGENT
-                ] = f"{aiohttp.http.SERVER_SOFTWARE} mlrun/{config.version}"
+                headers[aiohttp.hdrs.USER_AGENT] = (
+                    f"{aiohttp.http.SERVER_SOFTWARE} mlrun/{config.version}"
+                )
 
                 response: typing.Optional[
                     aiohttp.ClientResponse

diff --git a/mlrun/utils/http.py b/mlrun/utils/http.py
@@ -110,9 +110,9 @@ def __init__(
     def request(self, method, url, **kwargs):
         retry_count = 0
         kwargs.setdefault("headers", {})
-        kwargs["headers"][
-            "User-Agent"
-        ] = f"{requests.utils.default_user_agent()} mlrun/{config.version}"
+        kwargs["headers"]["User-Agent"] = (
+            f"{requests.utils.default_user_agent()} mlrun/{config.version}"
+        )
         while True:
             try:
                 response = super().request(method, url, **kwargs)