[refactor] Integrate Ruff for lint & format (#3125)

.github/workflows/pull-request.yml

@@ -57,7 +57,10 @@ jobs:
 
       - name: code style checks
         run: |
-          flake8 aim
+          set -e
+          ruff check .
+          ruff format . --check
+
 
       - name: unit-tests
         run: |

.gitignore

@@ -123,3 +123,5 @@ coverage_html_report/
 # generated cpp files
 # @TODO remove this line once there will be usage of cpp files
 aim/storage/**/*.cpp
+
+.ruff_cache

aim/__about__.py

@@ -1,5 +1,6 @@
-import time
 import os
+import time
+
 
 here = os.path.abspath(os.path.dirname(__file__))
 
@@ -9,18 +10,16 @@
 with open(version_file) as vf:
     __version__ = vf.read().strip()
 
-_this_year = time.strftime("%Y")
-__author__ = "Gev Sogomonian, Gor Arakelyan et al."
-__author_email__ = "[email protected]"
-__license__ = "Apache-2.0"
-__copyright__ = f"2019-{_this_year}, {__author__}."
-__homepage__ = "https://github.com/aimhubio/aim"
-__docs_url__ = "https://github.com/aimhubio/aim"
+_this_year = time.strftime('%Y')
+__author__ = 'Gev Sogomonian, Gor Arakelyan et al.'
+__author_email__ = '[email protected]'
+__license__ = 'Apache-2.0'
+__copyright__ = f'2019-{_this_year}, {__author__}.'
+__homepage__ = 'https://github.com/aimhubio/aim'
+__docs_url__ = 'https://github.com/aimhubio/aim'
 # this has to be simple string, see: https://github.com/pypa/twine/issues/522
-__docs__ = (
-    "Aim is a lightweight library for ML researchers to store and retrieve AI metadata."
-)
+__docs__ = 'Aim is a lightweight library for ML researchers to store and retrieve AI metadata.'
 __long_docs__ = """
 """
 
-__all__ = ["__author__", "__author_email__", "__copyright__", "__docs__", "__homepage__", "__license__", "__version__"]
+__all__ = ['__author__', '__author_email__', '__copyright__', '__docs__', '__homepage__', '__license__', '__version__']

aim/__init__.py

@@ -1,8 +1,9 @@
 import aimrocks
-from aim.sdk import *
-from aim.ext.notebook.notebook import load_ipython_extension
-from aim.cli.manager.manager import run_process
 
+from aim.cli.manager.manager import run_process
+from aim.ext.notebook.notebook import load_ipython_extension
+from aim.sdk import *  # noqa: F403
 from aim.utils.deprecation import python_version_deprecation_check
 
+
 python_version_deprecation_check()

aim/__version__.py

@@ -1,5 +1,6 @@
 import os
 
+
 here = os.path.abspath(os.path.dirname(__file__))
 
 version_file = f'{here}/VERSION'

aim/cli/cli.py

@@ -1,14 +1,23 @@
 import click
-from click import core
 
-from aim.cli.configs import *  # noqa F403
+from aim.cli.configs import (
+    CONVERT,
+    INIT_NAME,
+    RUNS_NAME,
+    SERVER_NAME,
+    STORAGE,
+    UP_NAME,
+    VERSION_NAME,
+)
+from aim.cli.convert import commands as convert_commands
 from aim.cli.init import commands as init_commands
-from aim.cli.version import commands as version_commands
-from aim.cli.up import commands as up_commands
-from aim.cli.server import commands as server_commands
 from aim.cli.runs import commands as runs_commands
-from aim.cli.convert import commands as convert_commands
+from aim.cli.server import commands as server_commands
 from aim.cli.storage import commands as storage_commands
+from aim.cli.up import commands as up_commands
+from aim.cli.version import commands as version_commands
+from click import core
+
 
 core._verify_python3_env = lambda: None
 

aim/cli/convert/commands.py

@@ -1,22 +1,19 @@
 import os
 
 import click
-from click import ClickException
 
-from aim.sdk.repo import Repo
-from aim.sdk.utils import clean_repo_path
 from aim.cli.convert.processors import (
-    parse_tb_logs,
     parse_mlflow_logs,
+    parse_tb_logs,
     parse_wandb_logs,
 )
+from aim.sdk.repo import Repo
+from aim.sdk.utils import clean_repo_path
+from click import ClickException
 
 
 @click.group()
-@click.option('--repo', required=False, type=click.Path(exists=True,
-                                                        file_okay=False,
-                                                        dir_okay=True,
-                                                        writable=True))
+@click.option('--repo', required=False, type=click.Path(exists=True, file_okay=False, dir_okay=True, writable=True))
 @click.pass_context
 def convert(ctx, repo):
     ctx.ensure_object(dict)
@@ -29,10 +26,7 @@ def convert(ctx, repo):
 
 @convert.command(name='tensorboard')
 @click.pass_context
-@click.option('--logdir', required=True, type=click.Path(exists=True,
-                                                         readable=True,
-                                                         dir_okay=True,
-                                                         resolve_path=True))
+@click.option('--logdir', required=True, type=click.Path(exists=True, readable=True, dir_okay=True, resolve_path=True))
 @click.option('--flat', '-f', required=False, is_flag=True, default=False)
 @click.option('--no-cache', required=False, is_flag=True, default=False)
 def convert_tensorboard(ctx, logdir, flat, no_cache):
@@ -42,14 +36,13 @@ def convert_tensorboard(ctx, logdir, flat, no_cache):
 
 @convert.command(name='tf')
 @click.pass_context
-@click.option('--logdir', required=True, type=click.Path(exists=True,
-                                                         readable=True,
-                                                         dir_okay=True,
-                                                         resolve_path=True))
+@click.option('--logdir', required=True, type=click.Path(exists=True, readable=True, dir_okay=True, resolve_path=True))
 @click.option('--flat', '-f', required=False, is_flag=True, default=False)
 def convert_tensorflow(ctx, logdir, flat):
-    click.secho('WARN: Command \'tf\' is deprecated and will be removed in future releases,'
-                ' please use \'tensorboard\' instead.', fg='red')
+    click.secho(
+        "WARN: Command 'tf' is deprecated and will be removed in future releases," " please use 'tensorboard' instead.",
+        fg='red',
+    )
     repo_inst = ctx.obj['repo_inst']
     parse_tb_logs(logdir, repo_inst, flat)
 
@@ -60,9 +53,9 @@ def convert_tensorflow(ctx, logdir, flat):
 @click.option('--experiment', '-e', required=False, default=None)
 def convert_mlflow(ctx, tracking_uri=None, **kwargs):
     repo_inst = ctx.obj['repo_inst']
-    tracking_uri = tracking_uri or os.environ.get("MLFLOW_TRACKING_URI")
+    tracking_uri = tracking_uri or os.environ.get('MLFLOW_TRACKING_URI')
     if not tracking_uri:
-        raise ClickException("MLFlow tracking URI must be provided either trough ENV or CLI.")
+        raise ClickException('MLFlow tracking URI must be provided either trough ENV or CLI.')
     parse_mlflow_logs(repo_inst, tracking_uri, **kwargs)
 
 

aim/cli/convert/processors/__init__.py

@@ -1,3 +1,3 @@
-from .tensorboard import parse_tb_logs
 from .mlflow import parse_mlflow_logs
+from .tensorboard import parse_tb_logs
 from .wandb import parse_wandb_logs

aim/cli/convert/processors/mlflow.py

@@ -1,9 +1,11 @@
 import os.path
+
 from tempfile import TemporaryDirectory
 
 import click
 
-from aim import Run, Image, Text, Audio
+from aim import Audio, Image, Run, Text
+
 
 IMAGE_EXTENSIONS = ('jpg', 'bmp', 'jpeg', 'png', 'gif', 'svg')
 HTML_EXTENSIONS = ('html',)
@@ -34,9 +36,7 @@ def parse_mlflow_logs(repo_inst, tracking_uri, experiment):
     try:
         import mlflow
     except ImportError:
-        click.echo(
-            'Could not process mlflow logs - failed to import "mlflow" module.', err=True
-        )
+        click.echo('Could not process mlflow logs - failed to import "mlflow" module.', err=True)
         return
 
     client = mlflow.tracking.client.MlflowClient(tracking_uri=tracking_uri)
@@ -65,14 +65,12 @@ def parse_mlflow_logs(repo_inst, tracking_uri, experiment):
                 experiment=ex.experiment_id,
             )
             aim_run['mlflow_run_id'] = run.info.run_id
-            aim_run['mlflow_run_name'] = run.data.tags.get("mlflow.runName")
-            aim_run.description = run.data.tags.get("mlflow.note.content")
+            aim_run['mlflow_run_name'] = run.data.tags.get('mlflow.runName')
+            aim_run.description = run.data.tags.get('mlflow.note.content')
 
             # Collect params & tags
             aim_run['params'] = run.data.params
-            aim_run['tags'] = {
-                k: v for k, v in run.data.tags.items() if not k.startswith('mlflow')
-            }
+            aim_run['tags'] = {k: v for k, v in run.data.tags.items() if not k.startswith('mlflow')}
 
             # Collect metrics
             for key in run.data.metrics.keys():
@@ -102,51 +100,34 @@ def parse_mlflow_logs(repo_inst, tracking_uri, experiment):
                             # TODO [AP] plotly does not provide interface to load from html
                             # TODO [AP] need to implement html custom object?
                             if not __html_warning_issued:
-                                click.secho(
-                                    'Handler for html file types is not yet implemented.', fg='yellow'
-                                )
+                                click.secho('Handler for html file types is not yet implemented.', fg='yellow')
                                 __html_warning_issued = True
                             continue
                         elif file_info.path.endswith(IMAGE_EXTENSIONS):
                             aim_object = Image
-                            kwargs = dict(
-                                image=downloaded_path,
-                                caption=file_info.path
-                            )
+                            kwargs = dict(image=downloaded_path, caption=file_info.path)
                             container = img_batch
                         elif file_info.path.endswith(TEXT_EXTENSIONS):
                             with open(downloaded_path) as fh:
                                 content = fh.read()
                             aim_object = Text
-                            kwargs = dict(
-                                text=content
-                            )
+                            kwargs = dict(text=content)
                             container = text_batch
                         elif file_info.path.endswith(AUDIO_EXTENSIONS):
                             audio_format = os.path.splitext(file_info.path)[1].lstrip('.')
                             aim_object = Audio
-                            kwargs = dict(
-                                data=downloaded_path,
-                                caption=file_info.path,
-                                format=audio_format
-                            )
+                            kwargs = dict(data=downloaded_path, caption=file_info.path, format=audio_format)
                             container = audio_batch
                         else:
-                            click.secho(
-                                f'Unresolved or unsupported type for artifact {file_info.path}', fg='yellow'
-                            )
+                            click.secho(f'Unresolved or unsupported type for artifact {file_info.path}', fg='yellow')
                             continue
 
                         try:
                             item = aim_object(**kwargs)
                         except Exception as exc:
-                            click.echo(
-                                f'Could not convert artifact {file_info.path} into aim object - {exc}', err=True
-                            )
+                            click.echo(f'Could not convert artifact {file_info.path} into aim object - {exc}', err=True)
                             continue
                         container.append(item)
 
-                    for content_type, seq in (('image', img_batch),
-                                              ('text', text_batch),
-                                              ('audio', audio_batch)):
+                    for content_type, seq in (('image', img_batch), ('text', text_batch), ('audio', audio_batch)):
                         aim_run.track(seq, step=0, name=loc or 'root', context={'type': content_type})