ESMValTool concatenates fx variables

[mo-gregmunday]

I'm trying to write a recipe which runs for as many CMIP6 models as I can possibly get, for the historical and SSP585 runs. It isn't so important to get 'specific' models, just as many as possible.

In order to do this, I must make sure that a combination of variables exist in for all of those models - including the fx variable, sftlf.

In order to get sftlf and the other variables I want, I have had to use sftlf from wherever I can get it for each model. For some models I can get it from the piControl experiments, for others I can get it for historical, for others I can get it from ssp585.

However, during the concatenation step, which happens because I specify exp: [historical, ssp585], ESMValTool tries to concatenate 2 or more land fraction variables together, which I don't want to happen, and fails because they don't have time coordinates. I'm guessing that this happens because sftlf also falls under 'historical', but I'm not sure.

Any help would be appreciated here!

Also, if there's any better way of selecting all models with data that exists with specific combinations of variables, I'm all ears :)

Cheers!

[ehogan]

@ESMValGroup/technical-lead-development-team would any of you be able to help Greg with his question, please? 😊

[bouweandela]

There are example notebooks available that show how to compose a recipe and discover all available data on ESGF corresponding to some particular facets. My recommendation would be to write a small Python script or a Jupyter notebook based on the examples above to populate the datasets section of your recipe.

Regarding sftlf: if you're using a preprocessor function that requires sftlf, the best matching version of the file will be automatically attached to the main variable. So my recommendation would be to first try and run the recipe without specifying sftlf. If that doesn't result in the solution you want, you can specify exactly which ancillary variable you want in the recipe. Documentation on this feature is available here in the section Defining supplementary variables (ancillary variables and cell measures) of the recipe format documentation and the section Supplementary variables (ancillary variables and cell measures) of the preprocessor documentation.

[bouweandela]

Here are some example scripts that I created in the past:

recipe_easy_ipcc.yml

Find all CMIP6 datasets that provide variable tos for experiments historical, ssp126, and ssp585.

import yaml
from esmvalcore.config import CFG
from esmvalcore.dataset import Dataset, datasets_to_recipe


def find_cmip6():

    historical_template = Dataset(
        project='CMIP6',
        mip='Omon',
        short_name='tos',
        exp='historical',
        dataset='*',
        institute='*',
        ensemble='*',
        grid='*',
        activity='*',
    )
    ssp585_template = historical_template.copy(exp='ssp585')
    ssp126_template = historical_template.copy(exp='ssp126')
    historical = list(historical_template.from_files())
    for dataset in historical:
        print("{dataset} {ensemble}".format(**dataset.facets))
    ssp585 = list(ssp585_template.from_files())
    ssp126 = list(ssp126_template.from_files())
    facets = ['dataset', 'ensemble', 'grid']
    dataset_facets = {tuple((f, v) for f, v in ds.facets.items() if f in facets) for ds in historical}
    for collection in [ssp585, ssp126]:
        dataset_facets &= {tuple((f, v) for f, v in ds.facets.items() if f in facets) for ds in collection}

    template = Dataset(
        project='CMIP6',
        mip='Omon',
        short_name='tos',
        diagnostic='x',
    )
    datasets = [template.copy(**dict(facets)) for facets in sorted(dataset_facets, key=lambda t: tuple((k, v.lower()) for k, v in t))]

    print(f"Found {len(datasets)} datasets")
    return datasets


def main():

    CFG['search_esgf'] = 'always'
    datasets = find_cmip6()

    recipe = datasets_to_recipe(datasets)
    for dataset in recipe['datasets']:
        order = ['dataset', 'ensemble', 'grid']
        dataset = {f: dataset[f] for f in order}
        txt = yaml.safe_dump(dataset, default_flow_style=True, width=120, sort_keys=False).rstrip('\n')
        print(f"- {txt}")


if __name__ == '__main__':
    main()

recipe_impact.yml

Find all CMIP5 and CMIP6 datasets that provide variables tas and pr for experiments historical, and rcp85/ssp585.

import yaml
from esmvalcore.config import CFG
from esmvalcore.dataset import Dataset, datasets_to_recipe


def find_cmip5():

    tas_historical_template = Dataset(
        project='CMIP5',
        mip='Amon',
        short_name='tas',
        exp='historical',
        dataset='*',
        institute='*',
        ensemble='*',
        product='*',
    )
    tas_rcp85_template = tas_historical_template.copy(exp='rcp85')
    pr_historical_template = tas_historical_template.copy(short_name='pr')
    pr_rcp85_template = pr_historical_template.copy(exp='rcp85')
    tas_historical = list(tas_historical_template.from_files())
    tas_rcp85 = list(tas_rcp85_template.from_files())
    pr_historical = list(pr_historical_template.from_files())
    pr_rcp85 = list(pr_rcp85_template.from_files())
    facets = ['dataset', 'ensemble']
    dataset_facets = {tuple((f, v) for f, v in ds.facets.items() if f in facets) for ds in tas_historical}
    for collection in [tas_rcp85, pr_historical, pr_rcp85]:
        dataset_facets &= {tuple((f, v) for f, v in ds.facets.items() if f in facets) for ds in collection}

    template = Dataset(
        project='CMIP5',
        mip='Amon',
        short_name='tas',
        exp=['historical', 'rcp85'],
        diagnostic='x',
    )
    datasets = [template.copy(**dict(facets)) for facets in sorted(dataset_facets, key=lambda t: tuple((k, v.lower()) for k, v in t))]
    print(f"Found {len(datasets)} datasets")
    return datasets


def find_cmip6():

    tas_historical_template = Dataset(
        project='CMIP6',
        mip='Amon',
        short_name='tas',
        exp='historical',
        dataset='*',
        institute='*',
        ensemble='*',
        grid='*',
        activity='*',
    )
    tas_ssp585_template = tas_historical_template.copy(exp='ssp585')
    pr_historical_template = tas_historical_template.copy(short_name='pr')
    pr_ssp585_template = pr_historical_template.copy(exp='ssp585')
    tas_historical = list(tas_historical_template.from_files())
    tas_ssp585 = list(tas_ssp585_template.from_files())
    pr_historical = list(pr_historical_template.from_files())
    pr_ssp585 = list(pr_ssp585_template.from_files())
    facets = ['dataset', 'ensemble', 'grid']
    dataset_facets = {tuple((f, v) for f, v in ds.facets.items() if f in facets) for ds in tas_historical}
    for collection in [tas_ssp585, pr_historical, pr_ssp585]:
        dataset_facets &= {tuple((f, v) for f, v in ds.facets.items() if f in facets) for ds in collection}

    template = Dataset(
        project='CMIP6',
        mip='Amon',
        short_name='tas',
        exp=['historical', 'ssp585'],
        diagnostic='x',
    )
    datasets = [template.copy(**dict(facets)) for facets in sorted(dataset_facets, key=lambda t: tuple((k, v.lower()) for k, v in t))]

    print(f"Found {len(datasets)} datasets")
    return datasets


def main():

    CFG['search_esgf'] = 'always'
    cmip5 = find_cmip5()
    cmip6 = find_cmip6()

    recipe = datasets_to_recipe(cmip5 + cmip6)
    for dataset in recipe['datasets']:
        if dataset['project'] == 'CMIP6':
            order = ['project', 'exp', 'dataset', 'ensemble', 'grid']
        else:
            order = ['project', 'exp', 'dataset', 'ensemble']
        dataset = {f: dataset[f] for f in order}
        txt = yaml.safe_dump(dataset, default_flow_style=True, width=120, sort_keys=False).rstrip('\n')
        print(f"- {txt}")


if __name__ == '__main__':
    main()