From 44b1d08dd6683a208cadcff380fa957a31521136 Mon Sep 17 00:00:00 2001 From: Pauline Laharanne Date: Tue, 17 Oct 2023 10:43:14 +0200 Subject: [PATCH] add summary param to show only global progress or file by file --- croud/__main__.py | 5 ++++ croud/clusters/commands.py | 45 +++++++++++++++++++-------------- docs/commands/clusters.rst | 33 +++++++++++++++++------- tests/commands/test_clusters.py | 29 ++++++++++++++++++++- 4 files changed, 83 insertions(+), 29 deletions(-) diff --git a/croud/__main__.py b/croud/__main__.py index 07dbc6ba..340936aa 100644 --- a/croud/__main__.py +++ b/croud/__main__.py @@ -834,6 +834,11 @@ help="The offset to skip before beginning to " "return the files." ), + Argument( + "--summary", type=lambda x: bool(strtobool(str(x))), + required=False, + help="Show only global progress." + ), ], "resolver": import_job_progress, }, diff --git a/croud/clusters/commands.py b/croud/clusters/commands.py index affc5251..79ef0933 100644 --- a/croud/clusters/commands.py +++ b/croud/clusters/commands.py @@ -326,26 +326,33 @@ def import_job_progress(args: Namespace) -> None: f"/api/v2/clusters/{args.cluster_id}/import-jobs/{args.import_job_id}/progress/" ) data, errors = client.get(url, params=params) - print_response( - data=data.get("progress", {}) if data else {}, - errors=errors, - keys=[ - "percent", - "records", - "failed_records", - "total_records", - "total_files", - "files", - ], - output_fmt=get_output_format(args), - transforms={ - "files": _format_files, - }, - ) - -def _format_files(field): - return ",\n".join(str(f) for f in field) if field else None + if args.summary: + print_response( + data=data.get("progress", {}) if data else {}, + errors=errors, + keys=[ + "percent", + "records", + "failed_records", + "total_records", + "total_files", + ], + output_fmt=get_output_format(args), + ) + else: + print_response( + data=data.get("progress", {}).get("files", []) if data else {}, + errors=errors, + keys=[ + "name", + "percent", + "records", + "failed_records", + "total_records", + ], + output_fmt=get_output_format(args), + ) def clusters_upgrade(args: Namespace) -> None: diff --git a/docs/commands/clusters.rst b/docs/commands/clusters.rst index 9aeb3158..7b2c7461 100644 --- a/docs/commands/clusters.rst +++ b/docs/commands/clusters.rst @@ -735,20 +735,35 @@ Example :prog: croud :path: clusters import-jobs progress -Example -------- +Examples +-------- .. code-block:: console sh$ ❯ croud clusters import-jobs progress \ --cluster-id e1e38d92-a650-48f1-8a70-8133f2d5c400 \ - --import-job-id 00de6048-3af6-41da-bfaa-661199d1c106 - +-----------+-----------+------------------+-----------------+---------------+-----------------------------------------------------------------------------------+ - | percent | records | failed_records | total_records | total_files | files | - |-----------+-----------+------------------+-----------------+---------------+-----------------------------------------------------------------------------------| - | 100 | 891 | 0 | 891 | 2 | {'failed_records': 0, 'name': 'file1.csv', 'records': 800, 'total_records': 891}, | - | | | | | | {'failed_records': 0, 'name': 'file2.csv', 'records': 91, 'total_records': 891} | - +-----------+-----------+------------------+-----------------+---------------+-----------------------------------------------------------------------------------+ + --import-job-id 00de6048-3af6-41da-bfaa-661199d1c106 \ + --summary true + +-----------+-----------+------------------+-----------------+---------------+ + | percent | records | failed_records | total_records | total_files | + |-----------+-----------+------------------+-----------------+---------------+ + | 100 | 891 | 0 | 891 | 2 | + +-----------+-----------+------------------+-----------------+---------------+ + + +.. code-block:: console + + sh$ ❯ croud clusters import-jobs progress \ + --cluster-id e1e38d92-a650-48f1-8a70-8133f2d5c400 \ + --import-job-id 00de6048-3af6-41da-bfaa-661199d1c106 \ + --limit ALL + --offset 0 + +-----------+-----------+-----------+------------------+-----------------+ + | name | percent | records | failed_records | total_records | + |-----------+-----------+-----------+------------------+-----------------| + | file1.csv | 100 | 800 | 0 | 800 | + | file2.csv | 100 | 91 | 0 | 91 | + +-----------+-----------+-----------+------------------+-----------------+ ``clusters export-jobs`` diff --git a/tests/commands/test_clusters.py b/tests/commands/test_clusters.py index c23d208d..ab37f8bc 100644 --- a/tests/commands/test_clusters.py +++ b/tests/commands/test_clusters.py @@ -1741,11 +1741,38 @@ def test_import_job_list(mock_request, output_format): ) +@mock.patch.object(Client, "request", return_value=({}, None)) +def test_import_job_progress_summary(mock_request): + cluster_id = gen_uuid() + import_job_id = gen_uuid() + + args = [ + "croud", + "clusters", + "import-jobs", + "progress", + "--summary", + "true", + "--cluster-id", + cluster_id, + "--import-job-id", + import_job_id, + ] + + call_command(*args) + assert_rest( + mock_request, + RequestMethod.GET, + f"/api/v2/clusters/{cluster_id}/import-jobs/{import_job_id}/progress/", + params={}, + ) + + @mock.patch.object(Client, "request", return_value=({}, None)) @pytest.mark.parametrize( "params", [{}, {"offset": 2}, {"limit": "2"}, {"limit": "ALL", "offset": 2}] ) -def test_import_job_progress(mock_request, params): +def test_import_job_progress_files(mock_request, params): cluster_id = gen_uuid() import_job_id = gen_uuid()