#346 Merge branch 'main' of github.com:NOAA-GFDL/fre-cli into 346.no…

…-comb-file

docs/tools.rst

@@ -63,3 +63,9 @@ fre yamltools
 =================
 
 .. include:: tools/yamltools.rst
+
+
+fre list
+=================
+
+.. include:: tools/listtools.rst

docs/tools/listtools.rst

@@ -0,0 +1,16 @@
+``exps``
+-----------------
+
+``fre list exps [options]``
+   - Purpose: Lists available post-processing experiments included in the yaml configurations
+   - Options:
+        - `-y, --yamlfile [experiment yaml]`
+
+``platforms``
+-----------------
+
+``fre list platforms [options]``
+   - Purpose: Lists available platforms included in the yaml configurations
+   - Options:
+        - `-y, --yamlfile [experiment yaml]`
+

fre/cmor/cmor_mixer.py

@@ -17,7 +17,7 @@
 from .cmor_helpers import *
 
 # ----- \start consts # TODO make this an input argument flag or smth.
-DEBUG_MODE_RUN_ONE = True
+DEBUG_MODE_RUN_ONE = False
 # ----- \end consts
 
 

fre/fre.py

@@ -21,14 +21,23 @@
     version_minor = "0" + version_unexpanded_split[1]
 else:
     version_minor = version_unexpanded_split[1]
-version = version_unexpanded_split[0] + '.' + version_minor
+# if the patch version is present, then use it. otherwise, omit
+try:
+    len(version_unexpanded_split[2])
+    if len(version_unexpanded_split[2]) == 1:
+        version_patch = "0" + version_unexpanded_split[2]
+    else:
+        version_patch = version_unexpanded_split[2]
+    version = version_unexpanded_split[0] + '.' + version_minor + '.' + version_patch
+except IndexError:
+    version = version_unexpanded_split[0] + '.' + version_minor
 
 # click and lazy group loading
 @click.group(
     cls = LazyGroup,
     lazy_subcommands = {"pp": ".pp.frepp.pp_cli",
                        "catalog": ".catalog.frecatalog.catalog_cli",
-                       "list": ".list.frelist.list_cli",
+                       "list": ".list_.frelist.list_cli",
                        "check": ".check.frecheck.check_cli",
                        "run": ".run.frerun.run_cli",
                        "yamltools": ".yamltools.freyamltools.yamltools_cli",

fre/list_/frelist.py

@@ -0,0 +1,32 @@
+''' fre lister '''
+
+import click
+from fre.list_ import list_experiments_script
+from fre.list_ import list_platforms_script
+
+@click.group(help=click.style(" - access fre list subcommands", fg=(232,204,91)))
+def list_cli():
+    ''' entry point to fre list click commands '''
+
+@list_cli.command()
+@click.option("-y",
+              "--yamlfile",
+              type=str,
+              help="YAML file to be used for parsing",
+              required=True)
+def exps(yamlfile):
+    """ - List experiments  available"""
+    list_experiments_script.list_experiments_subtool(yamlfile)
+
+@list_cli.command()
+@click.option("-y",
+              "--yamlfile",
+              type=str,
+              help="YAML file to be used for parsing",
+              required=True)
+def platforms(yamlfile):
+    """ - List platforms available """
+    list_platforms_script.list_platforms_subtool(yamlfile)
+
+if __name__ == "__main__":
+    list_cli()

fre/list_/list_experiments_script.py

@@ -0,0 +1,59 @@
+"""
+Script combines the model yaml with exp, platform, and target to list experiment information.
+"""
+from pathlib import Path
+import yaml
+import fre.yamltools.combine_yamls as cy
+
+# To look into: ignore undefined alias error msg for listing?
+# Found this somewhere but don't fully understand yet
+#class NoAliasDumper(yaml.SafeDumper):
+#    def ignore_aliases(self, data):
+#        return True
+
+def quick_combine(yml, exp, platform, target):
+    """
+    Create intermediate combined model and exp. yaml
+    This is done to avoid an "undefined alias" error
+    """
+    # Combine model / experiment
+    comb = cy.init_pp_yaml(yml,exp,platform,target)
+    comb.combine_model()
+
+def remove(combined):
+    """
+    Remove intermediate combined yaml.
+    """
+    if Path(combined).exists():
+        Path(combined).unlink()
+        print("Remove intermediate combined yaml:\n",
+              f"   {combined} removed.")
+    else:
+        raise ValueError(f"{combined} could not be found to remove.")
+
+def list_experiments_subtool(yamlfile):
+    """
+    List the post-processing experiments available
+    """
+    # Regsiter tag handler
+    yaml.add_constructor('!join', cy.join_constructor)
+
+    e = "None"
+    p = "None"
+    t = "None"
+
+    combined = f"combined-{e}.yaml"
+
+    # Combine model / experiment
+    quick_combine(yamlfile,e,p,t)
+
+    # Print experiment names
+    c = cy.yaml_load(combined)
+
+    print("\nPost-processing experiments available:")
+    for i in c.get("experiments"):
+        print(f'   - {i.get("name")}')
+    print("\n")
+
+    # Clean intermediate combined yaml
+    remove(combined)

fre/list_/list_platforms_script.py

@@ -0,0 +1,85 @@
+"""
+Script combines the model yaml with exp, platform, and target to list experiment information.
+"""
+
+from pathlib import Path
+import yaml
+import json
+from jsonschema import validate, ValidationError, SchemaError
+import fre.yamltools.combine_yamls as cy
+
+# To look into: ignore undefined alias error msg for listing?
+# Found this somewhere but don't fully understand yet
+#class NoAliasDumper(yaml.SafeDumper):
+#    def ignore_aliases(self, data):
+#        return True
+
+def quick_combine(yml, platform, target):
+    """
+    Combine the intermediate model and platforms yaml.
+    This is done to avoid an "undefined alias" error
+    """
+    # Combine model / experiment
+    comb = cy.init_compile_yaml(yml,platform,target)
+    comb.combine_model()
+    comb.combine_platforms()
+    comb.clean_yaml()
+
+def remove(combined):
+    """
+    Remove intermediate combined yaml.
+    """
+    if Path(combined).exists():
+        Path(combined).unlink()
+        print("Remove intermediate combined yaml:\n",
+              f"   {combined} removed.")
+    else:
+        raise ValueError(f"{combined} could not be found to remove.")
+
+def validate_yaml(loaded_yaml):
+    """
+    Validate the intermediate combined yaml
+    """
+    # Validate combined yaml
+    frelist_dir = Path(__file__).resolve().parents[2]
+    schema_path = f"{frelist_dir}/fre/gfdl_msd_schemas/FRE/fre_make.json"
+    with open(schema_path, 'r') as s:
+        schema = json.load(s)
+
+    print("\nValidating intermediate yaml:")
+    try:
+        validate(instance=loaded_yaml, schema=schema)
+        print("    Intermediate combined yaml VALID.")
+    except:
+        raise ValueError("\n\nIntermediate combined yaml NOT VALID.")
+
+def list_platforms_subtool(yamlfile):
+    """
+    List the platforms available
+    """
+    # Regsiter tag handler
+    yaml.add_constructor('!join', cy.join_constructor)
+
+    e = yamlfile.split("/")[-1].split(".")[0]
+    p = "None"
+    t = "None"
+
+    combined = f"combined-{e}.yaml"
+    yamlpath = Path(yamlfile).parent
+
+    # Combine model / experiment
+    quick_combine(yamlfile,p,t)
+
+    # Print experiment names
+    yml = cy.yaml_load(f"{yamlpath}/{combined}")
+
+    # Validate the yaml
+    validate_yaml(yml)
+
+    print("\nPlatforms available:")
+    for i in yml.get("platforms"):
+        print(f'    - {i.get("name")}')
+    print("\n")
+
+    # Clean the intermediate combined yaml
+    remove(f"{yamlpath}/{combined}")

fre/list_/tests/test_list_experiments_script.py

@@ -0,0 +1,74 @@
+"""
+Test fre list exps
+"""
+import pytest
+from pathlib import Path
+import yaml
+from fre.list_ import list_experiments_script
+
+# SET-UP
+TEST_DIR = Path("fre/make/tests")
+NM_EXAMPLE = Path("null_example")
+YAMLFILE = "null_model.yaml"
+EXP_NAME = "None"
+
+# yaml file checks
+def test_modelyaml_exists():
+    ''' Make sure model yaml exists '''
+    assert Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}").exists()
+
+def test_exp_list(capfd):
+    ''' test list exps '''
+    list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}")
+
+    #Capture output
+    out,err=capfd.readouterr()
+    if "Post-processing experiments available" in out:
+        assert True
+    else:
+        assert False
+
+def test_int_combine(capfd):
+    ''' test intermediate combine step is happening '''
+    list_experiments_script.list_experiments_subtool(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}")
+
+    #Capture output
+    out,err=capfd.readouterr()
+    check_out = ["Combining yaml files", "model yaml"]
+    for i in check_out:
+        if i in out:
+            assert True
+        else:
+           assert False
+
+def test_nocombinedyaml():
+    ''' test intermediate combined yaml was cleaned at end of listing '''
+    assert Path(f"./combined-{EXP_NAME}.yaml").exists() == False
+
+# Test individual functions operating correctly: combine and clean
+def test_correct_combine():
+    ''' test that combined yaml includes necessary keys '''
+    p = "None"
+    t = "None"
+    yamlfilepath = Path(f"{TEST_DIR}/{NM_EXAMPLE}/{YAMLFILE}")
+
+    # Combine model / experiment
+    list_experiments_script.quick_combine(yamlfilepath,EXP_NAME,p,t)
+    assert Path(f"./combined-{EXP_NAME}.yaml").exists()
+
+    with open(f"combined-{EXP_NAME}.yaml", 'r') as yf:
+        y = yaml.load(yf,Loader=yaml.Loader)
+
+    req_keys = ["name","platform","target","fre_properties","experiments"]
+    for k in req_keys:
+        if k in y.keys():
+            assert True
+        else:
+            assert False   
+
+def test_yamlremove():
+   ''' test intermediate combined yaml removed '''
+   # Remove combined yaml file
+   list_experiments_script.remove(f"combined-{EXP_NAME}.yaml")
+
+   assert Path(f"./combined-{EXP_NAME}.yaml").exists() == False