From 397263fe02d1703e663c4efff26f4e0fa53e4b1f Mon Sep 17 00:00:00 2001 From: Pauline L Date: Wed, 18 Oct 2023 10:53:06 +0200 Subject: [PATCH] add command to show import job progress (#467) --- CHANGES.rst | 2 ++ croud/__main__.py | 31 ++++++++++++++++++ croud/clusters/commands.py | 42 ++++++++++++++++++++++++ docs/commands/clusters.rst | 40 +++++++++++++++++++++++ tests/commands/test_clusters.py | 58 +++++++++++++++++++++++++++++++++ 5 files changed, 173 insertions(+) diff --git a/CHANGES.rst b/CHANGES.rst index da65d544..1e9acac1 100644 --- a/CHANGES.rst +++ b/CHANGES.rst @@ -13,6 +13,8 @@ Unreleased - Fixed ``import-job create from-url`` throwing an error despite working. +- Added support for showing import job progress. + 1.7.0 - 2023/09/11 ================== diff --git a/croud/__main__.py b/croud/__main__.py index 316b01e9..340936aa 100644 --- a/croud/__main__.py +++ b/croud/__main__.py @@ -50,6 +50,7 @@ export_jobs_create, export_jobs_delete, export_jobs_list, + import_job_progress, import_jobs_create_from_file, import_jobs_create_from_s3, import_jobs_create_from_url, @@ -811,6 +812,36 @@ }, }, }, + "progress": { + "help": "Shows the progress of an import job.", + "extra_args": [ + Argument( + "--cluster-id", type=str, required=True, + help="The cluster the import jobs belong to." + ), + Argument( + "--import-job-id", type=str, + required=True, + help="The ID of the Import Job." + ), + Argument( + "--limit", type=str, required=False, + help="The number of files returned." + "Use keywork 'ALL' to have no limit applied." + ), + Argument( + "--offset", type=int, required=False, + help="The offset to skip before beginning to " + "return the files." + ), + Argument( + "--summary", type=lambda x: bool(strtobool(str(x))), + required=False, + help="Show only global progress." + ), + ], + "resolver": import_job_progress, + }, }, }, "export-jobs": { diff --git a/croud/clusters/commands.py b/croud/clusters/commands.py index 64ff26ed..79ef0933 100644 --- a/croud/clusters/commands.py +++ b/croud/clusters/commands.py @@ -313,6 +313,48 @@ def import_jobs_list(args: Namespace) -> None: ) +def import_job_progress(args: Namespace) -> None: + client = Client.from_args(args) + + params = {} + if args.limit: + params["limit"] = args.limit + if args.offset: + params["offset"] = args.offset + + url = ( + f"/api/v2/clusters/{args.cluster_id}/import-jobs/{args.import_job_id}/progress/" + ) + data, errors = client.get(url, params=params) + + if args.summary: + print_response( + data=data.get("progress", {}) if data else {}, + errors=errors, + keys=[ + "percent", + "records", + "failed_records", + "total_records", + "total_files", + ], + output_fmt=get_output_format(args), + ) + else: + print_response( + data=data.get("progress", {}).get("files", []) if data else {}, + errors=errors, + keys=[ + "name", + "percent", + "records", + "failed_records", + "total_records", + ], + output_fmt=get_output_format(args), + ) + + def clusters_upgrade(args: Namespace) -> None: body = {"crate_version": args.version} client = Client.from_args(args) diff --git a/docs/commands/clusters.rst b/docs/commands/clusters.rst index daac790e..7b2c7461 100644 --- a/docs/commands/clusters.rst +++ b/docs/commands/clusters.rst @@ -726,6 +726,46 @@ Example ==> Success: Success. +``clusters import-jobs progress`` +================================= + +.. argparse:: + :module: croud.__main__ + :func: get_parser + :prog: croud + :path: clusters import-jobs progress + +Examples +-------- + +.. code-block:: console + + sh$ ❯ croud clusters import-jobs progress \ + --cluster-id e1e38d92-a650-48f1-8a70-8133f2d5c400 \ + --import-job-id 00de6048-3af6-41da-bfaa-661199d1c106 \ + --summary true + +-----------+-----------+------------------+-----------------+---------------+ + | percent | records | failed_records | total_records | total_files | + |-----------+-----------+------------------+-----------------+---------------+ + | 100 | 891 | 0 | 891 | 2 | + +-----------+-----------+------------------+-----------------+---------------+ + + +.. code-block:: console + + sh$ ❯ croud clusters import-jobs progress \ + --cluster-id e1e38d92-a650-48f1-8a70-8133f2d5c400 \ + --import-job-id 00de6048-3af6-41da-bfaa-661199d1c106 \ + --limit ALL + --offset 0 + +-----------+-----------+-----------+------------------+-----------------+ + | name | percent | records | failed_records | total_records | + |-----------+-----------+-----------+------------------+-----------------| + | file1.csv | 100 | 800 | 0 | 800 | + | file2.csv | 100 | 91 | 0 | 91 | + +-----------+-----------+-----------+------------------+-----------------+ + + ``clusters export-jobs`` ======================== diff --git a/tests/commands/test_clusters.py b/tests/commands/test_clusters.py index 15b7f682..ab37f8bc 100644 --- a/tests/commands/test_clusters.py +++ b/tests/commands/test_clusters.py @@ -1741,6 +1741,64 @@ def test_import_job_list(mock_request, output_format): ) +@mock.patch.object(Client, "request", return_value=({}, None)) +def test_import_job_progress_summary(mock_request): + cluster_id = gen_uuid() + import_job_id = gen_uuid() + + args = [ + "croud", + "clusters", + "import-jobs", + "progress", + "--summary", + "true", + "--cluster-id", + cluster_id, + "--import-job-id", + import_job_id, + ] + + call_command(*args) + assert_rest( + mock_request, + RequestMethod.GET, + f"/api/v2/clusters/{cluster_id}/import-jobs/{import_job_id}/progress/", + params={}, + ) + + +@mock.patch.object(Client, "request", return_value=({}, None)) +@pytest.mark.parametrize( + "params", [{}, {"offset": 2}, {"limit": "2"}, {"limit": "ALL", "offset": 2}] +) +def test_import_job_progress_files(mock_request, params): + cluster_id = gen_uuid() + import_job_id = gen_uuid() + + args = [ + "croud", + "clusters", + "import-jobs", + "progress", + "--cluster-id", + cluster_id, + "--import-job-id", + import_job_id, + ] + for param_key, param_value in params.items(): + args.append(f"--{param_key}") + args.append(f"{param_value}") + + call_command(*args) + assert_rest( + mock_request, + RequestMethod.GET, + f"/api/v2/clusters/{cluster_id}/import-jobs/{import_job_id}/progress/", + params=params, + ) + + @mock.patch.object( Client, "request",