diff --git a/.gitmodules b/.gitmodules index e97725e3e..c9571cf78 100644 --- a/.gitmodules +++ b/.gitmodules @@ -26,129 +26,128 @@ # repository branches. # - [submodule "ccs_config"] - path = ccs_config - url = https://github.com/ESMCI/ccs_config_cesm.git - fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git - fxtag = ccs_config_cesm1.0.0 + path = ccs_config + url = https://github.com/ESMCI/ccs_config_cesm.git + fxDONOTUSEurl = https://github.com/ESMCI/ccs_config_cesm.git + fxtag = ccs_config_cesm1.0.1 fxrequired = ToplevelRequired -[submodule "share"] - path = share - url = https://github.com/ESCOMP/CESM_share - fxDONOTUSEurl = https://github.com/ESCOMP/CESM_share - fxtag = share1.1.2 - fxrequired = ToplevelRequired - [submodule "cime"] - path = cime - url = https://github.com/ESMCI/cime - fxDONOTUSEurl = https://github.com/ESMCI/cime + path = cime + url = https://github.com/ESMCI/cime + fxDONOTUSEurl = https://github.com/ESMCI/cime fxtag = cime6.1.0 fxrequired = ToplevelRequired -[submodule "mpi-serial"] - path = libraries/mpi-serial - url = https://github.com/ESMCI/mpi-serial - fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial - fxtag = MPIserial_2.5.0 +[submodule "fms"] + path = libraries/FMS + url = https://github.com/ESCOMP/FMS_interface + fxDONOTUSEurl = https://github.com/ESCOMP/FMS_interface fxrequired = ToplevelRequired + fxtag = fi_240516 -[submodule "cam"] - path = components/cam - url = https://www.github.com/ESCOMP/CAM - fxDONOTUSEurl = https://www.github.com/ESCOMP/CAM - fxtag = cam6_4_001 +[submodule "share"] + path = share + url = https://github.com/ESCOMP/CESM_share + fxDONOTUSEurl = https://github.com/ESCOMP/CESM_share + fxtag = share1.1.2 fxrequired = ToplevelRequired -[submodule "ww3"] - path = components/ww3 - url = https://github.com/ESCOMP/WW3_interface - fxDONOTUSEurl = https://github.com/ESCOMP/WW3_interface - fxtag = ww3i_0.0.2 - fxrequired = ToplevelRequired - -[submodule "rtm"] - path = components/rtm - url = https://github.com/ESCOMP/RTM - fxDONOTUSEurl = https://github.com/ESCOMP/RTM +[submodule "cam"] + path = components/cam + url = https://www.github.com/ESCOMP/CAM + fxDONOTUSEurl = https://www.github.com/ESCOMP/CAM + fxtag = cam6_4_016 fxrequired = ToplevelRequired - fxtag = rtm1_0_80 -[submodule "pysect"] - path = tools/statistical_ensemble_test/pyCECT - url = https://github.com/NCAR/PyCECT - fxDONOTUSEurl = https://github.com/NCAR/PyCECT +[submodule "clm"] + path = components/clm + url = https://github.com/ESCOMP/CTSM + fxDONOTUSEurl = https://github.com/ESCOMP/CTSM fxrequired = ToplevelRequired - fxtag = 3.2.2 + fxtag = ctsm5.2.009 -[submodule "mosart"] - path = components/mosart - url = https://github.com/ESCOMP/MOSART - fxDONOTUSEurl = https://github.com/ESCOMP/MOSART +[submodule "cice"] + path = components/cice + url = https://github.com/ESCOMP/CESM_CICE + fxDONOTUSEurl = https://github.com/ESCOMP/CESM_CICE fxrequired = ToplevelRequired - fxtag = mosart1.1.02 + fxtag = cesm_cice6_5_0_12 -[submodule "mizuroute"] - path = components/mizuroute - url = https://github.com/ESCOMP/mizuRoute - fxDONOTUSEurl = https://github.com/ESCOMP/mizuRoute +[submodule "mom"] + path = components/mom + url = https://github.com/ESCOMP/MOM_interface + fxDONOTUSEurl = https://github.com/ESCOMP/MOM_interface fxrequired = ToplevelRequired - fxtag = cesm-coupling.n02_v2.1.3 + fxtag = mi_240705 -[submodule "fms"] - path = libraries/FMS - url = https://github.com/ESCOMP/FMS_interface - fxDONOTUSEurl = https://github.com/ESCOMP/FMS_interface - fxrequired = ToplevelRequired - fxtag = fi_240516 - -[submodule "parallelio"] - path = libraries/parallelio - url = https://github.com/NCAR/ParallelIO - fxDONOTUSEurl = https://github.com/NCAR/ParallelIO +[submodule "cism"] + path = components/cism + url = https://github.com/ESCOMP/cism-wrapper.git + fxDONOTUSEurl = https://github.com/ESCOMP/cism-wrapper.git + fxtag = cismwrap_2_2_002 fxrequired = ToplevelRequired - fxtag = pio2_6_2 [submodule "cdeps"] - path = components/cdeps - url = https://github.com/ESCOMP/CDEPS - fxDONOTUSEurl = https://github.com/ESCOMP/CDEPS + path = components/cdeps + url = https://github.com/ESCOMP/CDEPS + fxDONOTUSEurl = https://github.com/ESCOMP/CDEPS fxrequired = ToplevelRequired fxtag = cdeps1.0.43 [submodule "cmeps"] - path = components/cmeps - url = https://github.com/ESCOMP/CMEPS.git - fxDONOTUSEurl = https://github.com/ESCOMP/CMEPS.git + path = components/cmeps + url = https://github.com/ESCOMP/CMEPS.git + fxDONOTUSEurl = https://github.com/ESCOMP/CMEPS.git fxrequired = ToplevelRequired - fxtag = cmeps1.0.2 + fxtag = cmeps1.0.6 -[submodule "cice"] - path = components/cice - url = https://github.com/ESCOMP/CESM_CICE - fxDONOTUSEurl = https://github.com/ESCOMP/CESM_CICE +[submodule "rtm"] + path = components/rtm + url = https://github.com/ESCOMP/RTM + fxDONOTUSEurl = https://github.com/ESCOMP/RTM fxrequired = ToplevelRequired - fxtag = cesm_cice6_5_0_9 + fxtag = rtm1_0_80 -[submodule "cism"] - path = components/cism - url = https://github.com/ESCOMP/cism-wrapper.git - fxDONOTUSEurl = https://github.com/ESCOMP/cism-wrapper.git - fxtag = cismwrap_2_2_002 +[submodule "ww3"] + path = components/ww3 + url = https://github.com/ESCOMP/WW3_interface + fxDONOTUSEurl = https://github.com/ESCOMP/WW3_interface + fxtag = ww3i_0.0.2 + fxrequired = ToplevelRequired + +[submodule "mizuroute"] + path = components/mizuroute + url = https://github.com/ESCOMP/mizuRoute + fxDONOTUSEurl = https://github.com/ESCOMP/mizuRoute + fxrequired = ToplevelRequired + fxtag = cesm-coupling.n02_v2.1.3 + +[submodule "mosart"] + path = components/mosart + url = https://github.com/ESCOMP/MOSART + fxDONOTUSEurl = https://github.com/ESCOMP/MOSART fxrequired = ToplevelRequired + fxtag = mosart1.1.02 -[submodule "clm"] - path = components/clm - url = https://github.com/ESCOMP/CTSM - fxDONOTUSEurl = https://github.com/ESCOMP/CTSM +[submodule "parallelio"] + path = libraries/parallelio + url = https://github.com/NCAR/ParallelIO + fxDONOTUSEurl = https://github.com/NCAR/ParallelIO fxrequired = ToplevelRequired - fxtag = ctsm5.2.009 + fxtag = pio2_6_2 -[submodule "mom"] - path = components/mom - url = https://github.com/ESCOMP/MOM_interface - fxDONOTUSEurl = https://github.com/ESCOMP/MOM_interface +[submodule "mpi-serial"] + path = libraries/mpi-serial + url = https://github.com/ESMCI/mpi-serial + fxDONOTUSEurl = https://github.com/ESMCI/mpi-serial + fxtag = MPIserial_2.5.0 fxrequired = ToplevelRequired - fxtag = mi_240705 + +[submodule "pysect"] + path = tools/statistical_ensemble_test/pyCECT + url = https://github.com/NCAR/PyCECT + fxDONOTUSEurl = https://github.com/NCAR/PyCECT + fxrequired = ToplevelRequired + fxtag = 3.2.2 diff --git a/.lib/git-fleximod/git_fleximod/cli.py b/.lib/git-fleximod/git_fleximod/cli.py index 25fce68fd..b6f728f88 100644 --- a/.lib/git-fleximod/git_fleximod/cli.py +++ b/.lib/git-fleximod/git_fleximod/cli.py @@ -2,7 +2,7 @@ import argparse from git_fleximod import utils -__version__ = "0.7.9" +__version__ = "0.8.4" def find_root_dir(filename=".gitmodules"): """ finds the highest directory in tree diff --git a/.lib/git-fleximod/git_fleximod/git_fleximod.py b/.lib/git-fleximod/git_fleximod/git_fleximod.py index ed24c4e75..50e0ef83d 100755 --- a/.lib/git-fleximod/git_fleximod/git_fleximod.py +++ b/.lib/git-fleximod/git_fleximod/git_fleximod.py @@ -13,14 +13,14 @@ from git_fleximod import cli from git_fleximod.gitinterface import GitInterface from git_fleximod.gitmodules import GitModules -from configparser import NoOptionError +from git_fleximod.submodule import Submodule # logger variable is global logger = None def fxrequired_allowed_values(): - return ["ToplevelRequired", "ToplevelOptional", "AlwaysRequired", "AlwaysOptional"] + return ["ToplevelRequired", "ToplevelOptional", "AlwaysRequired", "AlwaysOptional", "TopLevelRequired", "TopLevelOptional"] def commandline_arguments(args=None): @@ -33,14 +33,9 @@ def commandline_arguments(args=None): # explicitly listing a component overrides the optional flag if options.optional or options.components: - fxrequired = [ - "ToplevelRequired", - "ToplevelOptional", - "AlwaysRequired", - "AlwaysOptional", - ] + fxrequired = fxrequired_allowed_values() else: - fxrequired = ["ToplevelRequired", "AlwaysRequired"] + fxrequired = ["ToplevelRequired", "AlwaysRequired", "TopLevelRequired"] action = options.action if not action: @@ -98,7 +93,8 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master """ logger.info("Called sparse_checkout for {}".format(name)) rgit = GitInterface(root_dir, logger) - superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") + superroot = git_toplevelroot(root_dir, logger) + if superroot: gitroot = superroot.strip() else: @@ -154,6 +150,8 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master if os.path.isdir(os.path.join(root_dir, path, ".git")): with utils.pushd(sprep_repo): + if os.path.isdir(os.path.join(topgit,".git")): + shutil.rmtree(os.path.join(topgit,".git")) shutil.move(".git", topgit) with open(".git", "w") as f: f.write("gitdir: " + os.path.relpath(topgit)) @@ -166,7 +164,9 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master return with utils.pushd(sprep_repo): - shutil.copy(sparsefile, gitsparse) + if os.path.isfile(sparsefile): + shutil.copy(sparsefile, gitsparse) + # Finally checkout the repo sprepo_git.git_operation("fetch", "origin", "--tags") @@ -176,285 +176,82 @@ def submodule_sparse_checkout(root_dir, name, url, path, sparsefile, tag="master rgit.config_set_value(f'submodule "{name}"', "active", "true") rgit.config_set_value(f'submodule "{name}"', "url", url) - -def single_submodule_checkout( - root, name, path, url=None, tag=None, force=False, optional=False -): - """ - This function checks out a single git submodule. - - Parameters: - root (str): The root directory for the git operation. - name (str): The name of the submodule. - path (str): The path to the submodule. - url (str, optional): The URL of the submodule. Defaults to None. - tag (str, optional): The tag to checkout. Defaults to None. - force (bool, optional): If set to True, forces the checkout operation. Defaults to False. - optional (bool, optional): If set to True, the submodule is considered optional. Defaults to False. - - Returns: - None - """ - # function implementation... - git = GitInterface(root, logger) - repodir = os.path.join(root, path) - logger.info("Checkout {} into {}/{}".format(name, root, path)) - # if url is provided update to the new url - tmpurl = None - repo_exists = False - if os.path.exists(os.path.join(repodir, ".git")): - logger.info("Submodule {} already checked out".format(name)) - repo_exists = True - # Look for a .gitmodules file in the newly checkedout repo - if not repo_exists and url: - # ssh urls cause problems for those who dont have git accounts with ssh keys defined - # but cime has one since e3sm prefers ssh to https, because the .gitmodules file was - # opened with a GitModules object we don't need to worry about restoring the file here - # it will be done by the GitModules class - if url.startswith("git@"): - tmpurl = url - url = url.replace("git@github.com:", "https://github.com/") - git.git_operation("clone", url, path) - smgit = GitInterface(repodir, logger) - if not tag: - tag = smgit.git_operation("describe", "--tags", "--always").rstrip() - smgit.git_operation("checkout", tag) - # Now need to move the .git dir to the submodule location - rootdotgit = os.path.join(root, ".git") - if os.path.isfile(rootdotgit): - with open(rootdotgit) as f: - line = f.readline() - if line.startswith("gitdir: "): - rootdotgit = line[8:].rstrip() - - newpath = os.path.abspath(os.path.join(root, rootdotgit, "modules", name)) - if os.path.exists(newpath): - shutil.rmtree(os.path.join(repodir, ".git")) - else: - shutil.move(os.path.join(repodir, ".git"), newpath) - - with open(os.path.join(repodir, ".git"), "w") as f: - f.write("gitdir: " + os.path.relpath(newpath, start=repodir)) - - if not os.path.exists(repodir): - parent = os.path.dirname(repodir) - if not os.path.isdir(parent): - os.makedirs(parent) - git.git_operation("submodule", "add", "--name", name, "--", url, path) - - if not repo_exists or not tmpurl: - git.git_operation("submodule", "update", "--init", "--", path) - - if os.path.exists(os.path.join(repodir, ".gitmodules")): - # recursively handle this checkout - print(f"Recursively checking out submodules of {name}") - gitmodules = GitModules(logger, confpath=repodir) - requiredlist = ["AlwaysRequired"] - if optional: - requiredlist.append("AlwaysOptional") - submodules_checkout(gitmodules, repodir, requiredlist, force=force) - if not os.path.exists(os.path.join(repodir, ".git")): - utils.fatal_error( - f"Failed to checkout {name} {repo_exists} {tmpurl} {repodir} {path}" - ) - - if tmpurl: - print(git.git_operation("restore", ".gitmodules")) - - return - -def add_remote(git, url): - remotes = git.git_operation("remote", "-v") - newremote = "newremote.00" - if url in remotes: - for line in remotes: - if url in line and "fetch" in line: - newremote = line.split()[0] - break - else: - i = 0 - while "newremote" in remotes: - i = i + 1 - newremote = f"newremote.{i:02d}" - git.git_operation("remote", "add", newremote, url) - return newremote - -def submodules_status(gitmodules, root_dir, toplevel=False): +def init_submodule_from_gitmodules(gitmodules, name, root_dir, logger): + path = gitmodules.get(name, "path") + url = gitmodules.get(name, "url") + assert path and url, f"Malformed .gitmodules file {path} {url}" + tag = gitmodules.get(name, "fxtag") + fxurl = gitmodules.get(name, "fxDONOTUSEurl") + fxsparse = gitmodules.get(name, "fxsparse") + fxrequired = gitmodules.get(name, "fxrequired") + return Submodule(root_dir, name, path, url, fxtag=tag, fxurl=fxurl, fxsparse=fxsparse, fxrequired=fxrequired, logger=logger) + +def submodules_status(gitmodules, root_dir, toplevel=False, depth=0): testfails = 0 localmods = 0 needsupdate = 0 + wrapper = textwrap.TextWrapper(initial_indent=' '*(depth*10), width=120,subsequent_indent=' '*(depth*20)) for name in gitmodules.sections(): - path = gitmodules.get(name, "path") - tag = gitmodules.get(name, "fxtag") - url = gitmodules.get(name, "url") - required = gitmodules.get(name, "fxrequired") - level = required and "Toplevel" in required - if not path: - utils.fatal_error("No path found in .gitmodules for {}".format(name)) - newpath = os.path.join(root_dir, path) - logger.debug("newpath is {}".format(newpath)) - if not os.path.exists(os.path.join(newpath, ".git")): - rootgit = GitInterface(root_dir, logger) - # submodule commands use path, not name - url = url.replace("git@github.com:", "https://github.com/") - tags = rootgit.git_operation("ls-remote", "--tags", url) - result = rootgit.git_operation("submodule","status",newpath).split() - ahash = None - if result: - ahash = result[0][1:] - hhash = None - atag = None + submod = init_submodule_from_gitmodules(gitmodules, name, root_dir, logger) + + result,n,l,t = submod.status() + if toplevel or not submod.toplevel(): + print(wrapper.fill(result)) + testfails += t + localmods += l + needsupdate += n + subdir = os.path.join(root_dir, submod.path) + if os.path.exists(os.path.join(subdir, ".gitmodules")): + gsubmod = GitModules(logger, confpath=subdir) + t,l,n = submodules_status(gsubmod, subdir, depth=depth+1) + if toplevel or not submod.toplevel(): + testfails += t + localmods += l + needsupdate += n - needsupdate += 1 - if not toplevel and level: - continue - for htag in tags.split("\n"): - if htag.endswith('^{}'): - htag = htag[:-3] - if ahash and not atag and ahash in htag: - atag = (htag.split()[1])[10:] - if tag and not hhash and htag.endswith(tag): - hhash = htag.split()[0] - if hhash and atag: - break - optional = " (optional)" if required and "Optional" in required else "" - if tag and (ahash == hhash or atag == tag): - print(f"e {name:>20} not checked out, aligned at tag {tag}{optional}") - elif tag: - ahash = rootgit.git_operation( - "submodule", "status", "{}".format(path) - ).rstrip() - ahash = ahash[1 : len(tag) + 1] - if tag == ahash: - print(f"e {name:>20} not checked out, aligned at hash {ahash}{optional}") - else: - print( - f"e {name:>20} not checked out, out of sync at tag {atag}, expected tag is {tag}{optional}" - ) - testfails += 1 - else: - print(f"e {name:>20} has no fxtag defined in .gitmodules{optional}") - testfails += 1 - else: - with utils.pushd(newpath): - git = GitInterface(newpath, logger) - atag = git.git_operation("describe", "--tags", "--always").rstrip() - ahash = git.git_operation("rev-list", "HEAD").partition("\n")[0] - rurl = git.git_operation("ls-remote","--get-url").rstrip() - if rurl != url: - remote = add_remote(git, url) - git.git_operation("fetch", remote) - if tag and atag == tag: - print(f" {name:>20} at tag {tag}") - elif tag and ahash[: len(tag)] == tag: - print(f" {name:>20} at hash {ahash}") - elif atag == ahash: - print(f" {name:>20} at hash {ahash}") - elif tag: - print( - f"s {name:>20} {atag} {ahash} is out of sync with .gitmodules {tag}" - ) - testfails += 1 - needsupdate += 1 - else: - print( - f"e {name:>20} has no fxtag defined in .gitmodules, module at {atag}" - ) - testfails += 1 - - status = git.git_operation("status", "--ignore-submodules", "-uno") - if "nothing to commit" not in status: - localmods = localmods + 1 - print("M" + textwrap.indent(status, " ")) - return testfails, localmods, needsupdate +def git_toplevelroot(root_dir, logger): + rgit = GitInterface(root_dir, logger) + superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") + return superroot def submodules_update(gitmodules, root_dir, requiredlist, force): - _, localmods, needsupdate = submodules_status(gitmodules, root_dir) - - if localmods and not force: - local_mods_output() - return - if needsupdate == 0: - return - for name in gitmodules.sections(): - fxtag = gitmodules.get(name, "fxtag") - path = gitmodules.get(name, "path") - url = gitmodules.get(name, "url") - logger.info( - "name={} path={} url={} fxtag={} requiredlist={} ".format( - name, os.path.join(root_dir, path), url, fxtag, requiredlist - ) - ) - - fxrequired = gitmodules.get(name, "fxrequired") - assert fxrequired in fxrequired_allowed_values() - rgit = GitInterface(root_dir, logger) - superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") - - fxsparse = gitmodules.get(name, "fxsparse") - + submod = init_submodule_from_gitmodules(gitmodules, name, root_dir, logger) + + _, needsupdate, localmods, testfails = submod.status() + if not submod.fxrequired: + submod.fxrequired = "AlwaysRequired" + fxrequired = submod.fxrequired + allowedvalues = fxrequired_allowed_values() + assert fxrequired in allowedvalues + + superroot = git_toplevelroot(root_dir, logger) + if ( fxrequired - and (superroot and "Toplevel" in fxrequired) - or fxrequired not in requiredlist + and ((superroot and "Toplevel" in fxrequired) + or fxrequired not in requiredlist) ): - if "ToplevelOptional" == fxrequired: - print("Skipping optional component {}".format(name)) - continue - if fxsparse: - logger.debug( - "Callng submodule_sparse_checkout({}, {}, {}, {}, {}, {}".format( - root_dir, name, url, path, fxsparse, fxtag - ) - ) - submodule_sparse_checkout(root_dir, name, url, path, fxsparse, tag=fxtag) - else: - logger.info( - "Calling submodule_checkout({},{},{},{})".format( - root_dir, name, path, url - ) - ) - - single_submodule_checkout( - root_dir, - name, - path, - url=url, - tag=fxtag, - force=force, - optional=("AlwaysOptional" in requiredlist), - ) - - if os.path.exists(os.path.join(path, ".git")): - submoddir = os.path.join(root_dir, path) - with utils.pushd(submoddir): - git = GitInterface(submoddir, logger) - # first make sure the url is correct - upstream = git.git_operation("ls-remote", "--get-url").rstrip() - newremote = "origin" - if upstream != url: - add_remote(git, url) - - tags = git.git_operation("tag", "-l") - if fxtag and fxtag not in tags: - git.git_operation("fetch", newremote, "--tags") - atag = git.git_operation("describe", "--tags", "--always").rstrip() - if fxtag and fxtag != atag: - try: - git.git_operation("checkout", fxtag) - print(f"{name:>20} updated to {fxtag}") - except Exception as error: - print(error) - elif not fxtag: - print(f"No fxtag found for submodule {name:>20}") - else: - print(f"{name:>20} up to date.") - + if "Optional" in fxrequired and "Optional" not in requiredlist: + if fxrequired.startswith("Always"): + print(f"Skipping optional component {name:>20}") + continue + optional = "AlwaysOptional" in requiredlist + if fxrequired in requiredlist: + submod.update() + repodir = os.path.join(root_dir, submod.path) + if os.path.exists(os.path.join(repodir, ".gitmodules")): + # recursively handle this checkout + print(f"Recursively checking out submodules of {name}") + gitsubmodules = GitModules(submod.logger, confpath=repodir) + newrequiredlist = ["AlwaysRequired"] + if optional: + newrequiredlist.append("AlwaysOptional") + submodules_update(gitsubmodules, repodir, newrequiredlist, force=force) def local_mods_output(): text = """\ @@ -469,62 +266,6 @@ def local_mods_output(): """ print(text) - -# checkout is done by update if required so this function may be depricated -def submodules_checkout(gitmodules, root_dir, requiredlist, force=False): - """ - This function checks out all git submodules based on the provided parameters. - - Parameters: - gitmodules (ConfigParser): The gitmodules configuration. - root_dir (str): The root directory for the git operation. - requiredlist (list): The list of required modules. - force (bool, optional): If set to True, forces the checkout operation. Defaults to False. - - Returns: - None - """ - # function implementation... - print("") - _, localmods, needsupdate = submodules_status(gitmodules, root_dir) - if localmods and not force: - local_mods_output() - return - if not needsupdate: - return - for name in gitmodules.sections(): - fxrequired = gitmodules.get(name, "fxrequired") - fxsparse = gitmodules.get(name, "fxsparse") - fxtag = gitmodules.get(name, "fxtag") - path = gitmodules.get(name, "path") - url = gitmodules.get(name, "url") - if fxrequired and fxrequired not in requiredlist: - if "Optional" in fxrequired: - print("Skipping optional component {}".format(name)) - continue - - if fxsparse: - logger.debug( - "Callng submodule_sparse_checkout({}, {}, {}, {}, {}, {}".format( - root_dir, name, url, path, fxsparse, fxtag - ) - ) - submodule_sparse_checkout(root_dir, name, url, path, fxsparse, tag=fxtag) - else: - logger.debug( - "Calling submodule_checkout({},{},{})".format(root_dir, name, path) - ) - single_submodule_checkout( - root_dir, - name, - path, - url=url, - tag=fxtag, - force=force, - optional="AlwaysOptional" in requiredlist, - ) - - def submodules_test(gitmodules, root_dir): """ This function tests the git submodules based on the provided parameters. diff --git a/.lib/git-fleximod/git_fleximod/gitinterface.py b/.lib/git-fleximod/git_fleximod/gitinterface.py index 93ae38ecd..583120144 100644 --- a/.lib/git-fleximod/git_fleximod/gitinterface.py +++ b/.lib/git-fleximod/git_fleximod/gitinterface.py @@ -49,8 +49,14 @@ def _init_git_repo(self): # pylint: disable=unused-argument def git_operation(self, operation, *args, **kwargs): - command = self._git_command(operation, *args) - self.logger.info(command) + newargs = [] + for a in args: + # Do not use ssh interface + if isinstance(a, str): + a = a.replace("git@github.com:", "https://github.com/") + newargs.append(a) + + command = self._git_command(operation, *newargs) if isinstance(command, list): try: return utils.execute_subprocess(command, output_to_caller=True) @@ -62,7 +68,11 @@ def git_operation(self, operation, *args, **kwargs): def config_get_value(self, section, name): if self._use_module: config = self.repo.config_reader() - return config.get_value(section, name) + try: + val = config.get_value(section, name) + except: + val = None + return val else: cmd = ("git", "-C", str(self.repo_path), "config", "--get", f"{section}.{name}") output = utils.execute_subprocess(cmd, output_to_caller=True) diff --git a/.lib/git-fleximod/git_fleximod/gitmodules.py b/.lib/git-fleximod/git_fleximod/gitmodules.py index 7e4e05394..cf8b350dd 100644 --- a/.lib/git-fleximod/git_fleximod/gitmodules.py +++ b/.lib/git-fleximod/git_fleximod/gitmodules.py @@ -1,4 +1,4 @@ -import shutil +import shutil, os from pathlib import Path from configparser import RawConfigParser, ConfigParser from .lstripreader import LstripReader diff --git a/.lib/git-fleximod/git_fleximod/submodule.py b/.lib/git-fleximod/git_fleximod/submodule.py new file mode 100644 index 000000000..70a3018a4 --- /dev/null +++ b/.lib/git-fleximod/git_fleximod/submodule.py @@ -0,0 +1,416 @@ +import os +import textwrap +import shutil +import string +from configparser import NoOptionError +from git_fleximod import utils +from git_fleximod.gitinterface import GitInterface + +class Submodule(): + """ + Represents a Git submodule with enhanced features for flexible management. + + Attributes: + name (str): The name of the submodule. + root_dir (str): The root directory of the main project. + path (str): The relative path from the root directory to the submodule. + url (str): The URL of the submodule repository. + fxurl (str): The URL for flexible submodule management (optional). + fxtag (str): The tag for flexible submodule management (optional). + fxsparse (str): Path to the sparse checkout file relative to the submodule path, see git-sparse-checkout for details (optional). + fxrequired (str): Indicates if the submodule is optional or required (optional). + logger (logging.Logger): Logger instance for logging (optional). + """ + def __init__(self, root_dir, name, path, url, fxtag=None, fxurl=None, fxsparse=None, fxrequired=None, logger=None): + """ + Initializes a new Submodule instance with the provided attributes. + """ + self.name = name + self.root_dir = root_dir + self.path = path + self.url = url + self.fxurl = fxurl + self.fxtag = fxtag + self.fxsparse = fxsparse + if fxrequired: + self.fxrequired = fxrequired + else: + self.fxrequired = "AlwaysRequired" + self.logger = logger + + def status(self): + """ + Checks the status of the submodule and returns 4 parameters: + - result (str): The status of the submodule. + - needsupdate (bool): An indicator if the submodule needs to be updated. + - localmods (bool): An indicator if the submodule has local modifications. + - testfails (bool): An indicator if the submodule has failed a test, this is used for testing purposes. + """ + + smpath = os.path.join(self.root_dir, self.path) + testfails = False + localmods = False + needsupdate = False + ahash = None + optional = "" + if "Optional" in self.fxrequired: + optional = " (optional)" + required = None + level = None + if not os.path.exists(os.path.join(smpath, ".git")): + rootgit = GitInterface(self.root_dir, self.logger) + # submodule commands use path, not name + tags = rootgit.git_operation("ls-remote", "--tags", self.url) + result = rootgit.git_operation("submodule","status",smpath).split() + + if result: + ahash = result[0][1:] + hhash = None + atag = None + for htag in tags.split("\n"): + if htag.endswith('^{}'): + htag = htag[:-3] + if ahash and not atag and ahash in htag: + atag = (htag.split()[1])[10:] + if self.fxtag and not hhash and htag.endswith(self.fxtag): + hhash = htag.split()[0] + if hhash and atag: + break + if self.fxtag and (ahash == hhash or atag == self.fxtag): + result = f"e {self.name:>20} not checked out, aligned at tag {self.fxtag}{optional}" + needsupdate = True + elif self.fxtag: + ahash = rootgit.git_operation( + "submodule", "status", "{}".format(self.path) + ).rstrip() + ahash = ahash[1 : len(self.fxtag) + 1] + if self.fxtag == ahash: + result = f"e {self.name:>20} not checked out, aligned at hash {ahash}{optional}" + else: + result = f"e {self.name:>20} not checked out, out of sync at tag {atag}, expected tag is {self.fxtag}{optional}" + testfails = True + needsupdate = True + else: + result = f"e {self.name:>20} has no fxtag defined in .gitmodules{optional}" + testfails = False + else: + with utils.pushd(smpath): + git = GitInterface(smpath, self.logger) + remote = git.git_operation("remote").rstrip() + if remote == '': + result = f"e {self.name:>20} has no associated remote" + testfails = True + needsupdate = True + return result, needsupdate, localmods, testfails + rurl = git.git_operation("ls-remote","--get-url").rstrip() + line = git.git_operation("log", "--pretty=format:\"%h %d\"").partition('\n')[0] + parts = line.split() + ahash = parts[0][1:] + atag = None + if len(parts) > 3: + idx = 0 + while idx < len(parts)-1: + idx = idx+1 + if parts[idx] == 'tag:': + atag = parts[idx+1] + while atag.endswith(')') or atag.endswith(',') or atag.endswith("\""): + atag = atag[:-1] + if atag == self.fxtag: + break + + + #print(f"line is {line} ahash is {ahash} atag is {atag} {parts}") + # atag = git.git_operation("describe", "--tags", "--always").rstrip() + # ahash = git.git_operation("rev-list", "HEAD").partition("\n")[0] + + recurse = False + if rurl != self.url: + remote = self._add_remote(git) + git.git_operation("fetch", remote) + if self.fxtag and atag == self.fxtag: + result = f" {self.name:>20} at tag {self.fxtag}" + recurse = True + testfails = False + elif self.fxtag and (ahash[: len(self.fxtag)] == self.fxtag or (self.fxtag.find(ahash)==0)): + result = f" {self.name:>20} at hash {ahash}" + recurse = True + testfails = False + elif atag == ahash: + result = f" {self.name:>20} at hash {ahash}" + recurse = True + elif self.fxtag: + result = f"s {self.name:>20} {atag} {ahash} is out of sync with .gitmodules {self.fxtag}" + testfails = True + needsupdate = True + else: + if atag: + result = f"e {self.name:>20} has no fxtag defined in .gitmodules, module at {atag}" + else: + result = f"e {self.name:>20} has no fxtag defined in .gitmodules, module at {ahash}" + testfails = False + + status = git.git_operation("status", "--ignore-submodules", "-uno") + if "nothing to commit" not in status: + localmods = True + result = "M" + textwrap.indent(status, " ") +# print(f"result {result} needsupdate {needsupdate} localmods {localmods} testfails {testfails}") + return result, needsupdate, localmods, testfails + + + def _add_remote(self, git): + """ + Adds a new remote to the submodule if it does not already exist. + + This method checks the existing remotes of the submodule. If the submodule's URL is not already listed as a remote, + it attempts to add a new remote. The name for the new remote is generated dynamically to avoid conflicts. If no + remotes exist, it defaults to naming the new remote 'origin'. + + Args: + git (GitInterface): An instance of GitInterface to perform git operations. + + Returns: + str: The name of the new remote if added, or the name of the existing remote that matches the submodule's URL. + """ + remotes = git.git_operation("remote", "-v").splitlines() + upstream = None + if remotes: + upstream = git.git_operation("ls-remote", "--get-url").rstrip() + newremote = "newremote.00" + tmpurl = self.url.replace("git@github.com:", "https://github.com/") + line = next((s for s in remotes if self.url in s or tmpurl in s), None) + if line: + newremote = line.split()[0] + return newremote + else: + i = 0 + while "newremote" in remotes: + i = i + 1 + newremote = f"newremote.{i:02d}" + else: + newremote = "origin" + git.git_operation("remote", "add", newremote, self.url) + return newremote + + def toplevel(self): + """ + Returns True if the submodule is Toplevel (either Required or Optional) + """ + return True if "Top" in self.fxrequired else False + + def sparse_checkout(self): + """ + Performs a sparse checkout of the submodule. + + This method optimizes the checkout process by only checking out files specified in the submodule's sparse-checkout configuration, + rather than the entire submodule content. It achieves this by first ensuring the `.git/info/sparse-checkout` file is created and + configured in the submodule's directory. Then, it proceeds to checkout the desired tag. If the submodule has already been checked out, + this method will not perform the checkout again. + + This approach is particularly beneficial for submodules with a large number of files, as it significantly reduces the time and disk space + required for the checkout process by avoiding the unnecessary checkout and subsequent removal of unneeded files. + + Returns: + None + """ + self.logger.info("Called sparse_checkout for {}".format(self.name)) + rgit = GitInterface(self.root_dir, self.logger) + superroot = rgit.git_operation("rev-parse", "--show-superproject-working-tree") + if superroot: + gitroot = superroot.strip() + else: + gitroot = self.root_dir.strip() + assert os.path.isdir(os.path.join(gitroot, ".git")) + # first create the module directory + if not os.path.isdir(os.path.join(self.root_dir, self.path)): + os.makedirs(os.path.join(self.root_dir, self.path)) + + # initialize a new git repo and set the sparse checkout flag + sprep_repo = os.path.join(self.root_dir, self.path) + sprepo_git = GitInterface(sprep_repo, self.logger) + if os.path.exists(os.path.join(sprep_repo, ".git")): + try: + self.logger.info("Submodule {} found".format(self.name)) + chk = sprepo_git.config_get_value("core", "sparseCheckout") + if chk == "true": + self.logger.info("Sparse submodule {} already checked out".format(self.name)) + return + except (NoOptionError): + self.logger.debug("Sparse submodule {} not present".format(self.name)) + except Exception as e: + utils.fatal_error("Unexpected error {} occured.".format(e)) + + sprepo_git.config_set_value("core", "sparseCheckout", "true") + + # set the repository remote + + self.logger.info("Setting remote origin in {}/{}".format(self.root_dir, self.path)) + status = sprepo_git.git_operation("remote", "-v") + if self.url not in status: + sprepo_git.git_operation("remote", "add", "origin", self.url) + + topgit = os.path.join(gitroot, ".git") + + if gitroot != self.root_dir and os.path.isfile(os.path.join(self.root_dir, ".git")): + with open(os.path.join(self.root_dir, ".git")) as f: + gitpath = os.path.relpath( + os.path.join(self.root_dir, f.read().split()[1]), + start=os.path.join(self.root_dir, self.path), + ) + topgit = os.path.join(gitpath, "modules") + else: + topgit = os.path.relpath( + os.path.join(self.root_dir, ".git", "modules"), + start=os.path.join(self.root_dir, self.path), + ) + + with utils.pushd(sprep_repo): + if not os.path.isdir(topgit): + os.makedirs(topgit) + topgit += os.sep + self.name + + if os.path.isdir(os.path.join(self.root_dir, self.path, ".git")): + with utils.pushd(sprep_repo): + if os.path.isdir(os.path.join(topgit,".git")): + shutil.rmtree(os.path.join(topgit,".git")) + shutil.move(".git", topgit) + with open(".git", "w") as f: + f.write("gitdir: " + os.path.relpath(topgit)) + # assert(os.path.isdir(os.path.relpath(topgit, start=sprep_repo))) + gitsparse = os.path.abspath(os.path.join(topgit, "info", "sparse-checkout")) + if os.path.isfile(gitsparse): + self.logger.warning( + "submodule {} is already initialized {}".format(self.name, topgit) + ) + return + + with utils.pushd(sprep_repo): + if os.path.isfile(self.fxsparse): + shutil.copy(self.fxsparse, gitsparse) + + + # Finally checkout the repo + sprepo_git.git_operation("fetch", "origin", "--tags") + sprepo_git.git_operation("checkout", self.fxtag) + + print(f"Successfully checked out {self.name:>20} at {self.fxtag}") + rgit.config_set_value(f'submodule "{self.name}"', "active", "true") + rgit.config_set_value(f'submodule "{self.name}"', "url", self.url) + rgit.config_set_value(f'submodule "{self.name}"', "path", self.path) + + def update(self): + """ + Updates the submodule to the latest or specified version. + + This method handles the update process of the submodule, including checking out the submodule into the specified path, + handling sparse checkouts if configured, and updating the submodule's URL if necessary. It supports both SSH and HTTPS URLs, + automatically converting SSH URLs to HTTPS to avoid issues for users without SSH keys. + + The update process involves the following steps: + 1. If the submodule is configured for sparse checkout, it performs a sparse checkout. + 2. If the submodule is not already checked out, it clones the submodule using the provided URL. + 3. If a specific tag or hash is provided, it checks out that tag; otherwise, it checks out the latest version. + 4. If the root `.git` is a file (indicating a submodule or a worktree), additional steps are taken to integrate the submodule properly. + + Args: + None + Note: + - SSH URLs are automatically converted to HTTPS to accommodate users without SSH keys. + + Returns: + None + """ + git = GitInterface(self.root_dir, self.logger) + repodir = os.path.join(self.root_dir, self.path) + self.logger.info("Checkout {} into {}/{}".format(self.name, self.root_dir, self.path)) + # if url is provided update to the new url + tag = None + repo_exists = False + if os.path.exists(os.path.join(repodir, ".git")): + self.logger.info("Submodule {} already checked out".format(self.name)) + repo_exists = True + # Look for a .gitmodules file in the newly checkedout repo + if self.fxsparse: + print(f"Sparse checkout {self.name} fxsparse {self.fxsparse}") + self.sparse_checkout() + else: + if not repo_exists and self.url: + # ssh urls cause problems for those who dont have git accounts with ssh keys defined + # but cime has one since e3sm prefers ssh to https, because the .gitmodules file was + # opened with a GitModules object we don't need to worry about restoring the file here + # it will be done by the GitModules class + if self.url.startswith("git@"): + git.git_operation("clone", self.url, self.path) + smgit = GitInterface(repodir, self.logger) + if not tag: + tag = smgit.git_operation("describe", "--tags", "--always").rstrip() + smgit.git_operation("checkout", tag) + # Now need to move the .git dir to the submodule location + rootdotgit = os.path.join(self.root_dir, ".git") + if os.path.isfile(rootdotgit): + with open(rootdotgit) as f: + line = f.readline() + if line.startswith("gitdir: "): + rootdotgit = line[8:].rstrip() + + newpath = os.path.abspath(os.path.join(self.root_dir, rootdotgit, "modules", self.name)) + if os.path.exists(newpath): + shutil.rmtree(os.path.join(repodir, ".git")) + else: + shutil.move(os.path.join(repodir, ".git"), newpath) + + with open(os.path.join(repodir, ".git"), "w") as f: + f.write("gitdir: " + os.path.relpath(newpath, start=repodir)) + + if not os.path.exists(repodir): + parent = os.path.dirname(repodir) + if not os.path.isdir(parent): + os.makedirs(parent) + git.git_operation("submodule", "add", "--name", self.name, "--", self.url, self.path) + + if not repo_exists: + git.git_operation("submodule", "update", "--init", "--", self.path) + + if self.fxtag: + smgit = GitInterface(repodir, self.logger) + newremote = self._add_remote(smgit) + # Trying to distingush a tag from a hash + allowed = set(string.digits + 'abcdef') + if not set(self.fxtag) <= allowed: + # This is a tag + tag = f"refs/tags/{self.fxtag}:refs/tags/{self.fxtag}" + smgit.git_operation("fetch", newremote, tag) + smgit.git_operation("checkout", self.fxtag) + + if not os.path.exists(os.path.join(repodir, ".git")): + utils.fatal_error( + f"Failed to checkout {self.name} {repo_exists} {repodir} {self.path}" + ) + + + if os.path.exists(os.path.join(self.path, ".git")): + submoddir = os.path.join(self.root_dir, self.path) + with utils.pushd(submoddir): + git = GitInterface(submoddir, self.logger) + # first make sure the url is correct + newremote = self._add_remote(git) + tags = git.git_operation("tag", "-l") + fxtag = self.fxtag + if fxtag and fxtag not in tags: + git.git_operation("fetch", newremote, "--tags") + atag = git.git_operation("describe", "--tags", "--always").rstrip() + if fxtag and fxtag != atag: + try: + git.git_operation("checkout", fxtag) + print(f"{self.name:>20} updated to {fxtag}") + except Exception as error: + print(error) + + + elif not fxtag: + print(f"No fxtag found for submodule {self.name:>20}") + else: + print(f"{self.name:>20} up to date.") + + + + return diff --git a/.lib/git-fleximod/pyproject.toml b/.lib/git-fleximod/pyproject.toml index 52cc26e7a..850e57d59 100644 --- a/.lib/git-fleximod/pyproject.toml +++ b/.lib/git-fleximod/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "git-fleximod" -version = "0.7.9" +version = "0.8.4" description = "Extended support for git-submodule and git-sparse-checkout" authors = ["Jim Edwards "] maintainers = ["Jim Edwards "] diff --git a/.lib/git-fleximod/tbump.toml b/.lib/git-fleximod/tbump.toml index b1d08a561..bd82c557a 100644 --- a/.lib/git-fleximod/tbump.toml +++ b/.lib/git-fleximod/tbump.toml @@ -2,7 +2,7 @@ github_url = "https://github.com/jedwards4b/git-fleximod/" [version] -current = "0.7.9" +current = "0.8.4" # Example of a semver regexp. # Make sure this matches current_version before diff --git a/.lib/git-fleximod/tests/conftest.py b/.lib/git-fleximod/tests/conftest.py index 65ee85d23..81edbe713 100644 --- a/.lib/git-fleximod/tests/conftest.py +++ b/.lib/git-fleximod/tests/conftest.py @@ -119,8 +119,20 @@ def complex_repo(tmp_path, logger): str_path = str(test_dir) gitp = GitInterface(str_path, logger) gitp.git_operation("remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test2") - gitp.git_operation("fetch", "origin", "main") - gitp.git_operation("checkout", "main") + gitp.git_operation("fetch", "origin") + gitp.git_operation("checkout", "v0.0.1") + return test_dir + +@pytest.fixture +def complex_update(tmp_path, logger): + test_dir = tmp_path / "testcomplex" + test_dir.mkdir() + str_path = str(test_dir) + gitp = GitInterface(str_path, logger) + gitp.git_operation("remote", "add", "origin", "https://github.com/jedwards4b/fleximod-test2") + gitp.git_operation("fetch", "origin") + gitp.git_operation("checkout", "v0.0.2") + return test_dir @pytest.fixture diff --git a/.lib/git-fleximod/tests/test_e_complex_update.py b/.lib/git-fleximod/tests/test_e_complex_update.py new file mode 100644 index 000000000..0c3ab4c6a --- /dev/null +++ b/.lib/git-fleximod/tests/test_e_complex_update.py @@ -0,0 +1,69 @@ +import pytest +from pathlib import Path +from git_fleximod.gitinterface import GitInterface + +def test_complex_update(git_fleximod, complex_update, logger): + status = git_fleximod(complex_update, "status") + assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) + assert("ToplevelRequired not checked out, aligned at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired not checked out, aligned at tag MPIserial_2.4.0" in status.stdout) + assert("Complex not checked out, out of sync at tag testtag02, expected tag is testtag3" in status.stdout) + assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) + + # This should checkout and update test_submodule and complex_sub + result = git_fleximod(complex_update, "update") + assert result.returncode == 0 + + status = git_fleximod(complex_update, "status") + assert("ToplevelOptional not checked out, aligned at tag v5.3.2" in status.stdout) + assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) + assert("Complex at tag testtag3" in status.stdout) + + # now check the complex_sub + root = (complex_update / "modules" / "complex") + assert(not (root / "libraries" / "gptl" / ".git").exists()) + assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) + assert((root / "modules" / "mpi-serialAR" / ".git").exists()) + assert((root / "modules" / "mpi-serialSAR" / ".git").exists()) + assert(not (root / "modules" / "mpi-serial2" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / "m4").exists()) + assert(not (root / "modules" / "mpi-sparse" / "README").exists()) + + # update a single optional submodule + + result = git_fleximod(complex_update, "update ToplevelOptional") + assert result.returncode == 0 + + status = git_fleximod(complex_update, "status") + assert("ToplevelOptional at tag v5.3.2" in status.stdout) + assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) + assert("Complex at tag testtag3" in status.stdout) + assert("AlwaysOptional not checked out, out of sync at tag None, expected tag is MPIserial_2.3.0" in status.stdout) + + # Finally update optional + result = git_fleximod(complex_update, "update --optional") + assert result.returncode == 0 + + status = git_fleximod(complex_update, "status") + assert("ToplevelOptional at tag v5.3.2" in status.stdout) + assert("ToplevelRequired at tag MPIserial_2.5.0" in status.stdout) + assert("AlwaysRequired at tag MPIserial_2.4.0" in status.stdout) + assert("Complex at tag testtag3" in status.stdout) + assert("AlwaysOptional at tag MPIserial_2.3.0" in status.stdout) + + # now check the complex_sub + root = (complex_update / "modules" / "complex" ) + assert(not (root / "libraries" / "gptl" / ".git").exists()) + assert(not (root / "libraries" / "mpi-serial" / ".git").exists()) + assert(not (root / "modules" / "mpi-serial" / ".git").exists()) + assert((root / "modules" / "mpi-serialAR" / ".git").exists()) + assert((root / "modules" / "mpi-serialSAR" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / ".git").exists()) + assert((root / "modules" / "mpi-serial2" / ".git").exists()) + assert((root / "modules" / "mpi-sparse" / "m4").exists()) + assert(not (root / "modules" / "mpi-sparse" / "README").exists()) + + diff --git a/ChangeLog b/ChangeLog index 540de01a7..244cb5e05 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,3 +1,157 @@ +============================================================== +Tag name: cesm3_0_alpha02b +Originator(s): CSEG +Date: 29 July 2024 +One-line Summary: CAM answer changing tag + +components/cam https://github.com/ESCOMP/CAM/cam6_4_016 ** +components/cice https://github.com/ESCOMP/CESM_CICE/tree/cesm_cice6_5_0_12 ** +cime https://github.com/ESMCI/cime/tree/cime6.1.0 -- +share https://github.com/ESCOMP/CESM_share/tree/share1.1.2 -- +ccs_config https://github.com/ESMCI/ccs_config_cesm/tree/ccs_config_cesm1.0.1 ** +components/cmeps https://github.com/ESCOMP/CMEPS/tree/cmeps1.0.6 ** +components/cdeps https://github.com/ESCOMP/CDEPS/tree/cdeps1.0.43 -- +components/cism https://github.com/ESCOMP/cism-wrapper/tree/cismwrap_2_2_002 -- +components/clm https://github.com/ESCOMP/ctsm/tree/ctsm5.2.009 -- +components/fms https://github.com/ESCOMP/FMS_interface/tree/fi_240516 -- +components/mizuroute https://github.com/ESCOMP/mizuRoute/tree/cesm-coupling.n02_v2.1.3 -- +components/mom https://github.com/ESCOMP/MOM_interface/mi_240705 -- +components/mosart https://github.com/ESCOMP/mosart/tree/mosart1_1_02 -- +components/rtm https://github.com/ESCOMP/rtm/tree/rtm1_0_80 -- +components/ww3 https://github.com/ESCOMP/WW3-CESM/tree/ww3i_0.0.2 -- +libraries/parallelio https://github.com/NCAR/ParallilIO/tree/pio2_6_2 -- + +cam + Cheryl Craig 2024-07-23 - cam6_4_016 - components/cam (cesm3_0_beta02) + https://github.com/ESCOMP/CAM/tags/cam6_4_xxx + + Modify RRTMGP interface for MT configurations. Answers also change + for LT and cam6 tests using RRTMGP. + + + Cheryl Craig 2024-07-23 - cam6_4_015 - components/cam (cesm3_0_beta02) + https://github.com/ESCOMP/CAM/tags/cam6_4_015 + + For Brian Eaton + + Misc bug fixes, buildcpp reports errors in CAM configure, SILHS outputting subcolumns with zeroes fix, remove solar_htng_spctrl_scl from aquaplanet case + + + Francis Vitt 2024-07-22 - cam6_4_014 - components/cam (cesm3_0_beta02) + https://github.com/ESCOMP/CAM/tags/cam6_4_014 + + Clean up WACCMX use of ESMF gridded component + + + Francis Vitt 2024-07-21 - cam6_4_013 - components/cam (cesm3_0_beta02) + https://github.com/ESCOMP/CAM/tags/cam6_4_013 + + Aerosol wet removal bug fixes + + + Francis Vitt 2024-07-19 - cam6_4_012 - components/cam (cesm3_0_beta02) + https://github.com/ESCOMP/CAM/tags/cam6_4_012 + + Add climate-chemistry compset + + + Cheryl Craig 2024-07-19 - cam6_4_011 - components/cam (cesm3_0_beta02) + https://github.com/ESCOMP/CAM/tags/cam6_4_011 + + For Brian Eaton and Cheryl Craig + + Update submodules, git-fleximod; fix fv3 build; remove mct reference + + + Cheryl Craig 2024-07-16 - cam6_4_010 - components/cam (cesm3_0_beta02) + https://github.com/ESCOMP/CAM/tags/cam6_4_010 + + GW moving mountains + + Answer changing for CAM7 + + + Brian Dobbins 2024-07-11 - cam6_4_009 - components/cam (cesm3_0_beta02) + https://github.com/ESCOMP/CAM/tags/cam6_4_009 + + Changes older log-gamma function for an F2008 intrinsic. This is answer-changing, but only in certain WACCMX configurations. + + + Cheryl Craig 2024-07-10 - cam6_4_008 - components/cam (cesm3_0_beta02) + https://github.com/ESCOMP/CAM/tags/cam6_4_008 + + HB mods + dycore mods + + Answer changing for all SE dycore and CLUBB runs + + + Michael Waxmonsky 2024-07-10 - cam6_4_007 - components/cam (cesm3_0_beta02) + https://github.com/ESCOMP/CAM/tags/cam6_4_007 + + CCPP-ized TJ2016 #1070 + + + Brian Eaton 2024-07-03 - cam6_4_006 - components/cam (cesm3_0_beta02) + https://github.com/ESCOMP/CAM/tags/cam6_4_006 + + Fix CLUBB interface bug. + + + Brian Eaton 2024-07-01 - cam6_4_005 - components/cam (cesm3_0_beta02) + https://github.com/ESCOMP/CAM/tags/cam6_4_005 + + Limit vertical domain used by COSP + + + Francis Vitt 2024-06-29 - cam6_4_004 - components/cam (cesm3_0_beta02) + https://github.com/ESCOMP/CAM/tags/cam6_4_004 + + Misc corrections for WACCMX + + + Brian Eaton 2024-06-26 - cam6_4_002 - components/cam (cesm3_0_beta02) + https://github.com/ESCOMP/CAM/tags/cam6_4_002 + + Activate additional clubb diffusion in cam6. + + +ccs_config + James Edwards 2024-07-25 - ccs_config_cesm1.0.1 - ccs_config (cesm3_0_beta02) + https://github.com/ESMCI/ccs_config_cesm/tags/ccs_config_cesm1.0.1 + + Removes some obsolete grid definitions + + +cesm + James Edwards 2024-07-25 - cesm3_0_alpha02b - (cesm3_0_beta02) + https://github.com/ESCOMP/cesm/tags/cesm3.0.beta02 - not sure what this should be? + + Add pelayout for B1850MT. Update git-fleximod to v0.8.4 + + +cice + David Bailey 2024-07-02 - cesm_cice6_5_0_12 - components/cice (cesm3_0_beta02) + https://github.com/ESCOMP/CESM_CICE/tags/cesm_cice6_5_0_11 + + This fixes some of the git-fleximod stuff and updates the CICE version + from cice6_5_0_20231221 to cice6_5_0_20240702. + + Also removes the annual restarts from CICE. + + +cmeps + James Edwards 2024-07-25 - cmeps1.0.6 - src/drivers/nuopc/ (cesm3_0_beta02) + https://github.com/ESCOMP/CMEPS/tags/cmeps1.0.6 + + Fix merge error in med_phases_aoflux_mod.F90 + + + James Edwards 2024-07-25 - cmeps1.0.5 - src/drivers/nuopc/ (cesm3_0_beta02) + https://github.com/ESCOMP/CMEPS/tags/cmeps1.0.5 + + Declare cnt variable needed with ESMF_AWARE_THREADING + + ============================================================== Tag name: cesm3_0_alpha02a Originator(s): CSEG diff --git a/ccs_config b/ccs_config index 797acd701..97ee0d2ca 160000 --- a/ccs_config +++ b/ccs_config @@ -1 +1 @@ -Subproject commit 797acd7014f10c8e46e50403cbf0fdf52230a2b5 +Subproject commit 97ee0d2ca1d1a4066a492cedcafe380ad5ccc6ad diff --git a/cime b/cime index 422ddaa77..fcb9c6ec1 160000 --- a/cime +++ b/cime @@ -1 +1 @@ -Subproject commit 422ddaa770a3cea6e83a60c9700ebce77acaceed +Subproject commit fcb9c6ec1e15f2f33995cf247aef3f8ef9f121eb diff --git a/cime_config/config_compsets.xml b/cime_config/config_compsets.xml index 162e5e39c..8b3b6ed71 100644 --- a/cime_config/config_compsets.xml +++ b/cime_config/config_compsets.xml @@ -78,22 +78,15 @@ 1850_CAM60_CLM50%SP_CICE_DOCN%SOM_MOSART_SGLC_SWAV_TEST - - - B1850 - 1850_CAM60_CLM50%BGC-CROP_CICE_MOM6_MOSART_CISM2%GRIS-NOEVOLVE_SWAV_BGC%BDRD - - - - 0001-01-01 - 0001-01-01 - 1850-01-01 - 1955-01-01 - 2005-01-01 - 2013-01-01 + 0001-01-01 + 0001-01-01 + 1850-01-01 + 1955-01-01 + 2005-01-01 + 2013-01-01 diff --git a/cime_config/config_pes.xml b/cime_config/config_pes.xml index 3caf46aab..ddb34f752 100644 --- a/cime_config/config_pes.xml +++ b/cime_config/config_pes.xml @@ -42,16 +42,49 @@ + + + + 5400 + 1816 + 1816 + 3584 + 408 + 4 + 1 + 5400 + + + 1 + 1 + 1 + 1 + 1 + 1 + 1 + 1 + + + 0 + 3584 + 3584 + 0 + 5400 + 0 + 0 + 0 + + 20 ypd/ 7100 pe-hrs/simyr expected 5400 - 939 - 64 - 4397 - 488 - 64 - 64 + 1792 + 1792 + 3608 + 612 + 4 + 1 5400 @@ -68,10 +101,10 @@ 0 0 0 - 1003 - 5400 - 0 - 0 + 1792 + 5404 + 5400 + 5400 0 diff --git a/cime_config/testlist_allactive.xml b/cime_config/testlist_allactive.xml index 6f578b60c..54f6a6118 100644 --- a/cime_config/testlist_allactive.xml +++ b/cime_config/testlist_allactive.xml @@ -1,6 +1,6 @@ - + @@ -8,15 +8,25 @@ - + + + + + + + + + + + - + @@ -26,13 +36,14 @@ - + - + + @@ -40,7 +51,7 @@ - + @@ -74,7 +85,7 @@ - + @@ -82,7 +93,8 @@ - + + @@ -94,13 +106,14 @@ + - + @@ -121,7 +134,7 @@ - + diff --git a/components/cam b/components/cam index ab476f9b7..3e9a281e5 160000 --- a/components/cam +++ b/components/cam @@ -1 +1 @@ -Subproject commit ab476f9b7345cbefdc4cf67ff17f0fe85d8c7387 +Subproject commit 3e9a281e580d1c370d09e591ac9e4926c8e7cd48 diff --git a/components/cdeps b/components/cdeps index 7a522c828..453a9d175 160000 --- a/components/cdeps +++ b/components/cdeps @@ -1 +1 @@ -Subproject commit 7a522c828c32dc35777992653f281ec525509c4a +Subproject commit 453a9d175a5739d9cca5c4ec7b96f45b201decec diff --git a/components/cice b/components/cice index 58d73e91e..f14ec8339 160000 --- a/components/cice +++ b/components/cice @@ -1 +1 @@ -Subproject commit 58d73e91e542177b0d80d2f8d886176eb817893c +Subproject commit f14ec8339bc5bc4a7a0664da5e247b5cfda531a1 diff --git a/components/cism b/components/cism index c05dd5c4f..c84cc9f5b 160000 --- a/components/cism +++ b/components/cism @@ -1 +1 @@ -Subproject commit c05dd5c4fc85327e76523aaea9cfe1e388748928 +Subproject commit c84cc9f5b3103766a35d0a7ddd5e9dbd7deae762 diff --git a/components/clm b/components/clm index a9433779f..e04b7e2ee 160000 --- a/components/clm +++ b/components/clm @@ -1 +1 @@ -Subproject commit a9433779f0ae499d60ad118d2ec331628f0eaaa8 +Subproject commit e04b7e2ee974aaef93117776a96fd7ce1e774b4d diff --git a/components/cmeps b/components/cmeps index 6384ff4e4..452005149 160000 --- a/components/cmeps +++ b/components/cmeps @@ -1 +1 @@ -Subproject commit 6384ff4e4a6bc82a678f9419a43ffbd5d53ac209 +Subproject commit 452005149deea59768410c296b09b8457fd06bcd diff --git a/components/mizuroute b/components/mizuroute index c55bd1ab7..2ff305a02 160000 --- a/components/mizuroute +++ b/components/mizuroute @@ -1 +1 @@ -Subproject commit c55bd1ab746734ea77a00606dde895ab034edf1a +Subproject commit 2ff305a0292cb06789de6cfea7ad3cc0d6173493 diff --git a/components/mom b/components/mom index 2f3c37333..7b33fdbff 160000 --- a/components/mom +++ b/components/mom @@ -1 +1 @@ -Subproject commit 2f3c37333280fc66de1795d48209b7ff267e6e74 +Subproject commit 7b33fdbffe31dfa9db9ec61d310596b580c01dbb diff --git a/components/mosart b/components/mosart index abd2bc4bc..e2ffe0000 160000 --- a/components/mosart +++ b/components/mosart @@ -1 +1 @@ -Subproject commit abd2bc4bc7ee789aba2eb46c834c3d9ee51428b1 +Subproject commit e2ffe00004cc416cfc8bcfae2a949474075c1d1f diff --git a/components/rtm b/components/rtm index 2d6a3c7b2..b3dfcfbba 160000 --- a/components/rtm +++ b/components/rtm @@ -1 +1 @@ -Subproject commit 2d6a3c7b224abcd9af9b8ebb1ca939c5b9e59681 +Subproject commit b3dfcfbba58c151ac5a6ab513b3515ef3deff798 diff --git a/doc/source/cesm_configurations.rst b/doc/source/cesm_configurations.rst index ad9a7e502..88df6a53f 100644 --- a/doc/source/cesm_configurations.rst +++ b/doc/source/cesm_configurations.rst @@ -70,7 +70,7 @@ The CESM2 components can be summarized as follows: .. csv-table:: "CESM2 model components" :header: "Component Generic Type", "Component Generic Name", "Component Name", "Component Type", "Description" - :widths: 12, 10, 10, 10, 60 + :widths: 12, 10, 10, 8, 60 "atmosphere","atm","cam", "active","The `Community Atmosphere Model (CAM) `_ is a global atmospheric general circulation model developed from the NCAR CCM3." "atmosphere","atm","datm", "data", "The `data atmosphere `_ component is a pure data component that reads in atmospheric forcing data" diff --git a/libraries/mct b/libraries/mct deleted file mode 160000 index 82b0071e6..000000000 --- a/libraries/mct +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 82b0071e69d14330b75d23b0bc68543ebea9aadc diff --git a/share b/share index 4b9dc4871..f6f31fd61 160000 --- a/share +++ b/share @@ -1 +1 @@ -Subproject commit 4b9dc4871a259f00f35bb47708d876cb7dcdf75c +Subproject commit f6f31fd61cb8f80aee97311fcca64b3e26b0202c