diff --git a/CHANGES.rst b/CHANGES.rst index da65d544..1e9acac1 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -13,6 +13,8 @@ Unreleased - Fixed ``import-job create from-url`` throwing an error despite working. +- Added support for showing import job progress. + 1.7.0 - 2023/09/11 ================== diff --git a/croud/__main__.py b/croud/__main__.py index 316b01e9..07dbc6ba 100644 --- a/croud/__main__.py +++ b/croud/__main__.py @@ -50,6 +50,7 @@ export_jobs_create, export_jobs_delete, export_jobs_list, + import_job_progress, import_jobs_create_from_file, import_jobs_create_from_s3, import_jobs_create_from_url, @@ -811,6 +812,31 @@ }, }, }, + "progress": { + "help": "Shows the progress of an import job.", + "extra_args": [ + Argument( + "--cluster-id", type=str, required=True, + help="The cluster the import jobs belong to." + ), + Argument( + "--import-job-id", type=str, + required=True, + help="The ID of the Import Job." + ), + Argument( + "--limit", type=str, required=False, + help="The number of files returned." + "Use keywork 'ALL' to have no limit applied." + ), + Argument( + "--offset", type=int, required=False, + help="The offset to skip before beginning to " + "return the files." + ), + ], + "resolver": import_job_progress, + }, }, }, "export-jobs": { diff --git a/croud/clusters/commands.py b/croud/clusters/commands.py index 64ff26ed..affc5251 100644 --- a/croud/clusters/commands.py +++ b/croud/clusters/commands.py @@ -313,6 +313,41 @@ def import_jobs_list(args: Namespace) -> None: ) +def import_job_progress(args: Namespace) -> None: + client = Client.from_args(args) + + params = {} + if args.limit: + params["limit"] = args.limit + if args.offset: + params["offset"] = args.offset + + url = ( + f"/api/v2/clusters/{args.cluster_id}/import-jobs/{args.import_job_id}/progress/" + ) + data, errors = client.get(url, params=params) + print_response( + data=data.get("progress", {}) if data else {}, + errors=errors, + keys=[ + "percent", + "records", + "failed_records", + "total_records", + "total_files", + "files", + ], + output_fmt=get_output_format(args), + transforms={ + "files": _format_files, + }, + ) + + +def _format_files(field): + return ",\n".join(str(f) for f in field) if field else None + + def clusters_upgrade(args: Namespace) -> None: body = {"crate_version": args.version} client = Client.from_args(args) diff --git a/docs/commands/clusters.rst b/docs/commands/clusters.rst index daac790e..9aeb3158 100644 --- a/docs/commands/clusters.rst +++ b/docs/commands/clusters.rst @@ -726,6 +726,31 @@ Example ==> Success: Success. +``clusters import-jobs progress`` +================================= + +.. argparse:: + :module: croud.__main__ + :func: get_parser + :prog: croud + :path: clusters import-jobs progress + +Example +------- + +.. code-block:: console + + sh$ ❯ croud clusters import-jobs progress \ + --cluster-id e1e38d92-a650-48f1-8a70-8133f2d5c400 \ + --import-job-id 00de6048-3af6-41da-bfaa-661199d1c106 + +-----------+-----------+------------------+-----------------+---------------+-----------------------------------------------------------------------------------+ + | percent | records | failed_records | total_records | total_files | files | + |-----------+-----------+------------------+-----------------+---------------+-----------------------------------------------------------------------------------| + | 100 | 891 | 0 | 891 | 2 | {'failed_records': 0, 'name': 'file1.csv', 'records': 800, 'total_records': 891}, | + | | | | | | {'failed_records': 0, 'name': 'file2.csv', 'records': 91, 'total_records': 891} | + +-----------+-----------+------------------+-----------------+---------------+-----------------------------------------------------------------------------------+ + + ``clusters export-jobs`` ======================== diff --git a/tests/commands/test_clusters.py b/tests/commands/test_clusters.py index 15b7f682..c23d208d 100644 --- a/tests/commands/test_clusters.py +++ b/tests/commands/test_clusters.py @@ -1741,6 +1741,37 @@ def test_import_job_list(mock_request, output_format): ) +@mock.patch.object(Client, "request", return_value=({}, None)) +@pytest.mark.parametrize( + "params", [{}, {"offset": 2}, {"limit": "2"}, {"limit": "ALL", "offset": 2}] +) +def test_import_job_progress(mock_request, params): + cluster_id = gen_uuid() + import_job_id = gen_uuid() + + args = [ + "croud", + "clusters", + "import-jobs", + "progress", + "--cluster-id", + cluster_id, + "--import-job-id", + import_job_id, + ] + for param_key, param_value in params.items(): + args.append(f"--{param_key}") + args.append(f"{param_value}") + + call_command(*args) + assert_rest( + mock_request, + RequestMethod.GET, + f"/api/v2/clusters/{cluster_id}/import-jobs/{import_job_id}/progress/", + params=params, + ) + + @mock.patch.object( Client, "request",