Skip to content

Commit

Permalink
Add test for repeated exp summary data in reports
Browse files Browse the repository at this point in the history 
Tests that repeat data is correctly imported for reports. Updates
existing report test.
  • Loading branch information
@dapomeroy
dapomeroy committed Oct 11, 2024
1 parent 8aec434 commit ade3463
Show file tree
Hide file tree
Showing 2 changed files with 75 additions and 49 deletions.
121 changes: 75 additions & 46 deletions lib/ramble/ramble/test/reports.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -14,14 +14,6 @@
# - non-repeated experiments
# - repeated experiments

# Test conversion from nested dict to dataframe

# Test conversion from dataframe to chart-ready pivot
# - data is not summarized (1 experiment / 1 chart)
# - data is summarized / not repeats (n experiments / 1 chart)
# - data is summarized from repeats (n_repeats / 1 chart)
# - mix of summarized and non-summarized data

# Test normalization of data, and error when first value is zero

# Test that PDF is generated and contains data (size > some value?)
Expand Down Expand Up @@ -56,13 +48,15 @@
"units": "",
"origin": "dummy_app",
"origin_type": "application",
"fom_type": {"name": "MEASURE", "better_direction": "INDETERMINATE"},
},
{
"name": "fom_2",
"value": 50,
"units": "",
"origin": "dummy_app",
"origin_type": "application",
"fom_type": {"name": "MEASURE", "better_direction": "INDETERMINATE"},
},
],
},
Expand All @@ -88,13 +82,15 @@
"units": "",
"origin": "dummy_app",
"origin_type": "application",
"fom_type": {"name": "MEASURE", "better_direction": "INDETERMINATE"},
},
{
"name": "fom_2",
"value": 55,
"units": "",
"origin": "dummy_app",
"origin_type": "application",
"fom_type": {"name": "MEASURE", "better_direction": "INDETERMINATE"},
},
],
},
Expand All @@ -104,7 +100,7 @@
}


def prep_dict(
def create_test_exp(
success,
name,
n_nodes,
Expand All @@ -117,12 +113,13 @@ def prep_dict(
units,
origin,
origin_type,
fom_type,
better_direction,
fv,
ifv,
normalized=False,
):
return {
test_exp_dict = {
"RAMBLE_STATUS": success,
"name": name,
"n_nodes": n_nodes,
Expand All @@ -135,20 +132,24 @@ def prep_dict(
"fom_units": units,
"fom_origin": origin,
"fom_origin_type": origin_type,
"fom_type": fom_type,
"better_direction": better_direction,
"series": ns,
"normalized_fom_value" if normalized else "fom_value": fv,
"ideal_perf_value": ifv,
}
# ideal_perf_value is not calculated for plots without better_direction
if ifv:
test_exp_dict["ideal_perf_value"] = ifv
return test_exp_dict


@pytest.mark.parametrize(
"values",
[
(StrongScalingPlot, "fom_1", 42.0, 42.0, 42.0, 28.0, 28.0, 21.0, False),
(StrongScalingPlot, "fom_1", 42.0, 1.0, 1.0, 28.0, 1.5, 2.0, True),
(WeakScalingPlot, "fom_2", 50, 50, 50.0, 55, 55.0, 50.0, False),
(WeakScalingPlot, "fom_2", 50.0, 1.0, 1.0, 55.0, 1.1, 1.0, True),
(WeakScalingPlot, "fom_2", 50, 50, None, 55, 55.0, None, False),
(WeakScalingPlot, "fom_2", 50.0, 1.0, None, 55.0, 1.1, None, True),
],
)
def test_scaling_plots(mutable_mock_workspace_path, tmpdir_factory, values):
Expand All @@ -159,11 +160,11 @@ def test_scaling_plots(mutable_mock_workspace_path, tmpdir_factory, values):

plot_type, fom_name, fom1, nfv1, ideal1, fom2, nfv2, ideal2, normalize = values

test_spec = [[fom_name, "n_nodes"]]
test_spec = [fom_name, "n_nodes"]

ideal_data = []
ideal_data.append(
prep_dict(
create_test_exp(
"SUCCESS",
"exp_1",
1,
Expand All @@ -176,14 +177,15 @@ def test_scaling_plots(mutable_mock_workspace_path, tmpdir_factory, values):
"",
"dummy_app",
"application",
FomType.MEASURE,
BetterDirection.INDETERMINATE,
nfv1,
ideal1,
normalized=normalize,
)
)
ideal_data.append(
prep_dict(
create_test_exp(
"SUCCESS",
"exp_2",
2,
Expand All @@ -196,6 +198,7 @@ def test_scaling_plots(mutable_mock_workspace_path, tmpdir_factory, values):
"",
"dummy_app",
"application",
FomType.MEASURE,
BetterDirection.INDETERMINATE,
nfv2,
ideal2,
Expand Down Expand Up @@ -225,6 +228,10 @@ def test_scaling_plots(mutable_mock_workspace_path, tmpdir_factory, values):
)
plot.generate_plot_data()

# Sort columns alphabetically, order is not important
plot.output_df.sort_index(axis=1, inplace=True)
ideal_df.sort_index(axis=1, inplace=True)

assert plot.output_df.equals(ideal_df)
assert os.path.isfile(pdf_path)

Expand Down Expand Up @@ -252,27 +259,31 @@ def test_scaling_plots(mutable_mock_workspace_path, tmpdir_factory, values):
"origin": "dummy_app",
"origin_type": "summary::n_total_repeats",
"name": "Experiment Summary",
"fom_type": {"name": "MEASURE", "better_direction": "INDETERMINATE"},
},
{
"value": 2,
"units": "repeats",
"origin": "dummy_app",
"origin_type": "summary::n_successful_repeats",
"name": "Experiment Summary",
"fom_type": {"name": "MEASURE", "better_direction": "INDETERMINATE"},
},
{
"value": 28.0,
"units": "s",
"origin": "dummy_app",
"origin_type": "summary::min",
"name": "fom_1",
"fom_type": {"name": "TIME", "better_direction": "LOWER"},
},
{
"value": 30.0,
"units": "s",
"origin": "dummy_app",
"origin_type": "summary::max",
"name": "fom_1",
"fom_type": {"name": "TIME", "better_direction": "LOWER"},
},
{
"value": 29.0,
Expand All @@ -282,34 +293,6 @@ def test_scaling_plots(mutable_mock_workspace_path, tmpdir_factory, values):
"name": "fom_1",
"fom_type": {"name": "TIME", "better_direction": "LOWER"},
},
{
"value": 29.0,
"units": "s",
"origin": "dummy_app",
"origin_type": "summary::median",
"name": "fom_1",
},
{
"value": 2.0,
"units": "s^2",
"origin": "dummy_app",
"origin_type": "summary::variance",
"name": "fom_1",
},
{
"value": 1.4,
"units": "s",
"origin": "dummy_app",
"origin_type": "summary::stdev",
"name": "fom_1",
},
{
"value": 0.0,
"units": "",
"origin": "dummy_app",
"origin_type": "summary::cv",
"name": "fom_1",
},
],
},
],
Expand Down Expand Up @@ -384,14 +367,60 @@ def test_scaling_plots(mutable_mock_workspace_path, tmpdir_factory, values):
},
],
},
{
"RAMBLE_STATUS": "SUCCESS",
"name": "single_exp_1",
"n_nodes": 1,
# "application_namespace": "test_app",
# "workload_name": "test_workload",
"simplified_workload_namespace": "test_app_test_workload",
"RAMBLE_VARIABLES": {},
"RAMBLE_RAW_VARIABLES": {},
"CONTEXTS": [
{
"name": "null",
"display_name": "null",
"foms": [
{
"name": "fom_1",
"value": 42.0,
"units": "",
"origin": "dummy_app",
"origin_type": "application",
},
{
"name": "fom_2",
"value": 50,
"units": "",
"origin": "dummy_app",
"origin_type": "application",
},
],
},
],
},
]
}


def test_repeated_import(mutable_mock_workspace_path):
def test_repeat_import(mutable_mock_workspace_path):
where_query = None
results_df = prepare_data(repeat_results, where_query)
print(results_df.shape)

print(results_df)
# DF contains only summary exp and not individual repeats
assert "repeat_exp_1" in results_df.values
assert "repeat_exp_1.1" not in results_df.values
assert "single_exp_1" in results_df.values

# Summary FOMs are present in DF, types converted to objects
row_mean = results_df.query("fom_origin_type == 'summary::mean'")
assert row_mean["fom_value"].values == [29.0]
assert row_mean["fom_type"].values == [FomType.TIME]
assert row_mean["better_direction"].values == [BetterDirection.LOWER]

single_exp_rows = results_df.query("name == 'single_exp_1' and fom_name == 'fom_1'")
assert single_exp_rows["fom_value"].values == [42.0]


# TODO: test fom plot
Expand Down
3 changes: 0 additions & 3 deletions lib/ramble/ramble/test/success_criteria/repeat_success_strict.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@ def test_repeat_success_strict(mutable_config, mutable_mock_workspace_path, mock

with open(os.path.join(ws.root, "results.latest.txt")) as f:
data = f.read()
print(data)
assert "FAILED" not in data
assert "summary::n_total_repeats = 2 repeats" in data
assert "summary::n_successful_repeats = 2 repeats" in data
Expand All @@ -78,7 +77,6 @@ def test_repeat_success_strict(mutable_config, mutable_mock_workspace_path, mock

with open(os.path.join(ws.root, "results.latest.txt")) as f:
data = f.read()
print(data)
assert "SUCCESS" in data
assert "FAILED" in data
assert "summary::n_total_repeats = 2 repeats" in data
Expand All @@ -95,7 +93,6 @@ def test_repeat_success_strict(mutable_config, mutable_mock_workspace_path, mock

with open(os.path.join(ws.root, "results.latest.txt")) as f:
data = f.read()
print(data)
assert "SUCCESS" not in data
assert "summary::n_total_repeats" not in data
assert "summary::n_successful_repeats" not in data

0 comments on commit ade3463

Please sign in to comment.