Use black to lint all python files

.github/workflows/black.yml

@@ -0,0 +1,10 @@
+name: Lint
+
+on: [push, pull_request]
+
+jobs:
+  lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: psf/black@stable

packaging_automation/common_validations.py

@@ -13,7 +13,8 @@ def is_version(version: str):
         raise ValueError("version should be non-empty and should not be None")
     if not re.match(CITUS_PATCH_VERSION_PATTERN, version):
         raise ValueError(
-            "version should include three level of digits separated by dots, e.g: 10.0.1")
+            "version should include three level of digits separated by dots, e.g: 10.0.1"
+        )
 
 
 @parameter_validation
@@ -22,7 +23,8 @@ def is_tag(tag: str):
         raise ValueError("tag should be non-empty and should not be None")
     if not re.match(f"v{CITUS_PATCH_VERSION_PATTERN}", tag):
         raise ValueError(
-            "tag should start with 'v' and should include three level of digits separated by dots, e.g: v10.0.1")
+            "tag should start with 'v' and should include three level of digits separated by dots, e.g: v10.0.1"
+        )
 
 
 @parameter_validation

packaging_automation/dbconfig.py

@@ -18,12 +18,13 @@ class DbParams:
 
 def db_connection_string(db_params: DbParams, is_test=False):
     database_name = db_params.db_name if not is_test else f"{db_params.db_name}-test"
-    return f'postgresql+psycopg2://{db_params.user_name}:{db_params.password}@{db_params.host_and_port}/{database_name}'
+    return f"postgresql+psycopg2://{db_params.user_name}:{db_params.password}@{db_params.host_and_port}/{database_name}"
 
 
 def db_session(db_params: DbParams, is_test: bool, create_db_objects: bool = True):
     db_engine = create_engine(
-        db_connection_string(db_params=db_params, is_test=is_test))
+        db_connection_string(db_params=db_params, is_test=is_test)
+    )
     if create_db_objects:
         Base.metadata.create_all(db_engine)
     Session = sessionmaker(db_engine)

packaging_automation/docker_statistics_collector.py

@@ -5,8 +5,8 @@
 import requests
 from sqlalchemy import Column, DATE, INTEGER, TIMESTAMP, desc
 
-from .common_tool_methods import (str_array_to_str)
-from .dbconfig import (Base, DbParams, db_session)
+from .common_tool_methods import str_array_to_str
+from .dbconfig import Base, DbParams, db_session
 
 
 class DockerStats(Base):
@@ -21,63 +21,107 @@ class DockerStats(Base):
 docker_repositories = ["citus", "membership-manager"]
 
 
-def fetch_and_store_docker_statistics(repository_name: str, db_parameters: DbParams, is_test: bool = False,
-                                      test_day_shift_index: int = 0,
-                                      test_total_pull_count: int = 0):
+def fetch_and_store_docker_statistics(
+    repository_name: str,
+    db_parameters: DbParams,
+    is_test: bool = False,
+    test_day_shift_index: int = 0,
+    test_total_pull_count: int = 0,
+):
     if repository_name not in docker_repositories:
-        raise ValueError(f"Repository name should be in {str_array_to_str(docker_repositories)}")
+        raise ValueError(
+            f"Repository name should be in {str_array_to_str(docker_repositories)}"
+        )
     if not is_test and (test_day_shift_index != 0 or test_total_pull_count != 0):
-        raise ValueError("test_day_shift_index and test_total_pull_count parameters are test "
-                         "parameters. Please don't use these parameters other than testing.")
-
-    result = requests.get(f"https://hub.docker.com/v2/repositories/citusdata/{repository_name}/")
-    total_pull_count = int(result.json()["pull_count"]) if test_total_pull_count == 0 else test_total_pull_count
-
-    session = db_session(db_params=db_parameters, is_test=is_test, create_db_objects=True)
+        raise ValueError(
+            "test_day_shift_index and test_total_pull_count parameters are test "
+            "parameters. Please don't use these parameters other than testing."
+        )
+
+    result = requests.get(
+        f"https://hub.docker.com/v2/repositories/citusdata/{repository_name}/"
+    )
+    total_pull_count = (
+        int(result.json()["pull_count"])
+        if test_total_pull_count == 0
+        else test_total_pull_count
+    )
+
+    session = db_session(
+        db_params=db_parameters, is_test=is_test, create_db_objects=True
+    )
 
     fetch_date = datetime.now() + timedelta(days=test_day_shift_index)
     validate_same_day_record_existence(fetch_date, session)
-    day_diff, mod_pull_diff, pull_diff = calculate_diff_params(fetch_date, session, total_pull_count)
+    day_diff, mod_pull_diff, pull_diff = calculate_diff_params(
+        fetch_date, session, total_pull_count
+    )
     for i in range(0, day_diff):
-        daily_pull_count = ((pull_diff - mod_pull_diff) / day_diff
-                            if i > 0 else (pull_diff - mod_pull_diff) / day_diff + mod_pull_diff)
-        stat_param = DockerStats(fetch_date=fetch_date, total_pull_count=total_pull_count,
-                                 daily_pull_count=daily_pull_count,
-                                 stat_date=fetch_date.date() - timedelta(days=i))
+        daily_pull_count = (
+            (pull_diff - mod_pull_diff) / day_diff
+            if i > 0
+            else (pull_diff - mod_pull_diff) / day_diff + mod_pull_diff
+        )
+        stat_param = DockerStats(
+            fetch_date=fetch_date,
+            total_pull_count=total_pull_count,
+            daily_pull_count=daily_pull_count,
+            stat_date=fetch_date.date() - timedelta(days=i),
+        )
         session.add(stat_param)
     session.commit()
 
 
 def calculate_diff_params(fetch_date, session, total_pull_count):
-    last_stat_record = session.query(DockerStats).order_by(desc(DockerStats.stat_date)).first()
-    day_diff = (fetch_date.date() - last_stat_record.stat_date).days if last_stat_record else 1
-    pull_diff = total_pull_count - last_stat_record.total_pull_count if last_stat_record else total_pull_count
+    last_stat_record = (
+        session.query(DockerStats).order_by(desc(DockerStats.stat_date)).first()
+    )
+    day_diff = (
+        (fetch_date.date() - last_stat_record.stat_date).days if last_stat_record else 1
+    )
+    pull_diff = (
+        total_pull_count - last_stat_record.total_pull_count
+        if last_stat_record
+        else total_pull_count
+    )
     mod_pull_diff = pull_diff % day_diff
     return day_diff, mod_pull_diff, pull_diff
 
 
 def validate_same_day_record_existence(fetch_date, session):
-    same_day_record = session.query(DockerStats).filter_by(stat_date=fetch_date.date()).first()
+    same_day_record = (
+        session.query(DockerStats).filter_by(stat_date=fetch_date.date()).first()
+    )
     if same_day_record:
-        print(f"Docker download record for date {fetch_date.date()} already exists. No need to add record.")
+        print(
+            f"Docker download record for date {fetch_date.date()} already exists. No need to add record."
+        )
         sys.exit(0)
 
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument('--repo_name', choices=docker_repositories)
-    parser.add_argument('--db_user_name', required=True)
-    parser.add_argument('--db_password', required=True)
-    parser.add_argument('--db_host_and_port', required=True)
-    parser.add_argument('--db_name', required=True)
-    parser.add_argument('--is_test', action="store_true")
-    parser.add_argument('--test_day_shift_index', nargs='?', default=0)
-    parser.add_argument('--test_total_pull_count', nargs='?', default=0)
+    parser.add_argument("--repo_name", choices=docker_repositories)
+    parser.add_argument("--db_user_name", required=True)
+    parser.add_argument("--db_password", required=True)
+    parser.add_argument("--db_host_and_port", required=True)
+    parser.add_argument("--db_name", required=True)
+    parser.add_argument("--is_test", action="store_true")
+    parser.add_argument("--test_day_shift_index", nargs="?", default=0)
+    parser.add_argument("--test_total_pull_count", nargs="?", default=0)
 
     arguments = parser.parse_args()
-    db_params = DbParams(user_name=arguments.db_user_name, password=arguments.db_password,
-                         host_and_port=arguments.db_host_and_port, db_name=arguments.db_name)
-
-    fetch_and_store_docker_statistics(repository_name=arguments.repo_name, is_test=arguments.is_test,
-                                      db_parameters=db_params, test_day_shift_index=int(arguments.test_day_shift_index),
-                                      test_total_pull_count=int(arguments.test_total_pull_count))
+    db_params = DbParams(
+        user_name=arguments.db_user_name,
+        password=arguments.db_password,
+        host_and_port=arguments.db_host_and_port,
+        db_name=arguments.db_name,
+    )
+
+    fetch_and_store_docker_statistics(
+        repository_name=arguments.repo_name,
+        is_test=arguments.is_test,
+        db_parameters=db_params,
+        test_day_shift_index=int(arguments.test_day_shift_index),
+        test_total_pull_count=int(arguments.test_total_pull_count),
+    )

packaging_automation/get_postgres_versions.py

@@ -1,11 +1,11 @@
 import argparse
 import json
 
-from .test_citus_package import (get_postgres_versions_from_matrix_file)
+from .test_citus_package import get_postgres_versions_from_matrix_file
 
 if __name__ == "__main__":
     parser = argparse.ArgumentParser()
-    parser.add_argument('--project_version', required=True)
+    parser.add_argument("--project_version", required=True)
 
     args = parser.parse_args()
     postgres_versions = get_postgres_versions_from_matrix_file(args.project_version)