add command to show import job progress

crate · Oct 16, 2023 · 59a28f4 · 59a28f4
1 parent 1a58d74
commit 59a28f4
Show file tree

Hide file tree

Showing 5 changed files with 119 additions and 0 deletions.
diff --git a/CHANGES.rst b/CHANGES.rst
@@ -13,6 +13,8 @@ Unreleased
 
 - Fixed ``import-job create from-url`` throwing an error despite working.
 
+- Added support for showing import job progress.
+
 1.7.0 - 2023/09/11
 ==================
 

diff --git a/croud/__main__.py b/croud/__main__.py
@@ -50,6 +50,7 @@
     export_jobs_create,
     export_jobs_delete,
     export_jobs_list,
+    import_job_progress,
     import_jobs_create_from_file,
     import_jobs_create_from_s3,
     import_jobs_create_from_url,
@@ -811,6 +812,31 @@
                             },
                         },
                     },
+                    "progress": {
+                        "help": "Shows the progress of an import job.",
+                        "extra_args": [
+                            Argument(
+                                "--cluster-id", type=str, required=True,
+                                help="The cluster the import jobs belong to."
+                            ),
+                            Argument(
+                                "--import-job-id", type=str,
+                                required=True,
+                                help="The ID of the Import Job."
+                            ),
+                            Argument(
+                                "--limit", type=str, required=False,
+                                help="The number of files returned."
+                                     "Use keywork 'ALL' to have no limit applied."
+                            ),
+                            Argument(
+                                "--offset", type=int, required=False,
+                                help="The offset to skip before beginning to "
+                                     "return the files."
+                            ),
+                        ],
+                        "resolver": import_job_progress,
+                    },
                 },
             },
             "export-jobs": {

diff --git a/croud/clusters/commands.py b/croud/clusters/commands.py
@@ -313,6 +313,41 @@ def import_jobs_list(args: Namespace) -> None:
     )
 
 
+def import_job_progress(args: Namespace) -> None:
+    client = Client.from_args(args)
+
+    params = {}
+    if args.limit:
+        params["limit"] = args.limit
+    if args.offset:
+        params["offset"] = args.offset
+
+    url = (
+        f"/api/v2/clusters/{args.cluster_id}/import-jobs/{args.import_job_id}/progress/"
+    )
+    data, errors = client.get(url, params=params)
+    print_response(
+        data=data.get("progress", {}) if data else {},
+        errors=errors,
+        keys=[
+            "percent",
+            "records",
+            "failed_records",
+            "total_records",
+            "total_files",
+            "files",
+        ],
+        output_fmt=get_output_format(args),
+        transforms={
+            "files": _format_files,
+        },
+    )
+
+
+def _format_files(field):
+    return ",\n".join(str(f) for f in field) if field else None
+
+
 def clusters_upgrade(args: Namespace) -> None:
     body = {"crate_version": args.version}
     client = Client.from_args(args)

diff --git a/docs/commands/clusters.rst b/docs/commands/clusters.rst
@@ -726,6 +726,31 @@ Example
    ==> Success: Success.
 
 
+``clusters import-jobs progress``
+=============================
+
+.. argparse::
+   :module: croud.__main__
+   :func: get_parser
+   :prog: croud
+   :path: clusters import-jobs progress
+
+Example
+-------
+
+.. code-block:: console
+
+   sh$ ❯ croud clusters import-jobs progress \
+         --cluster-id e1e38d92-a650-48f1-8a70-8133f2d5c400 \
+         --import-job-id 00de6048-3af6-41da-bfaa-661199d1c106
+   +-----------+-----------+------------------+-----------------+---------------+-----------------------------------------------------------------------------------+
+   |   percent |   records |   failed_records |   total_records |   total_files | files                                                                             |
+   |-----------+-----------+------------------+-----------------+---------------+-----------------------------------------------------------------------------------|
+   |       100 |       891 |                0 |             891 |             2 | {'failed_records': 0, 'name': 'file1.csv', 'records': 800, 'total_records': 891}, |
+   |           |           |                  |                 |               | {'failed_records': 0, 'name': 'file2.csv', 'records': 91, 'total_records': 891}   |
+   +-----------+-----------+------------------+-----------------+---------------+-----------------------------------------------------------------------------------+
+
+
 ``clusters export-jobs``
 ========================
 

diff --git a/tests/commands/test_clusters.py b/tests/commands/test_clusters.py
@@ -1741,6 +1741,37 @@ def test_import_job_list(mock_request, output_format):
     )
 
 
+@mock.patch.object(Client, "request", return_value=({}, None))
+@pytest.mark.parametrize(
+    "params", [{}, {"offset": 2}, {"limit": "2"}, {"limit": "ALL", "offset": 2}]
+)
+def test_import_job_progress(mock_request, params):
+    cluster_id = gen_uuid()
+    import_job_id = gen_uuid()
+
+    args = [
+        "croud",
+        "clusters",
+        "import-jobs",
+        "progress",
+        "--cluster-id",
+        cluster_id,
+        "--import-job-id",
+        import_job_id,
+    ]
+    for param_key, param_value in params.items():
+        args.append(f"--{param_key}")
+        args.append(f"{param_value}")
+
+    call_command(*args)
+    assert_rest(
+        mock_request,
+        RequestMethod.GET,
+        f"/api/v2/clusters/{cluster_id}/import-jobs/{import_job_id}/progress/",
+        params=params,
+    )
+
+
 @mock.patch.object(
     Client,
     "request",