From 85253495d7da9de4294c93bc973a94dce0cd46a7 Mon Sep 17 00:00:00 2001 From: Janosh Riebesell Date: Sun, 28 Jan 2024 14:42:20 +0100 Subject: [PATCH] Fix failing `pkg_is_editable` check in PyPI release (#78) * add StrEnums Targets + ModelType used in make_metrics_tables.py * mv data/figshare matbench_discovery update pyproject tool.setuptools.package-data and FIGSHARE_DIR * add Targets col to main metrics table, remove meta cols from other metrics table variants * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * update site/ import paths for figshare URLs * bump package.json deps --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- data/figshare/readme.md | 3 - matbench_discovery/__init__.py | 29 +- .../figshare/1.0.0.json | 0 matbench_discovery/figshare/readme.md | 3 + matbench_discovery/preds.py | 2 +- pyproject.toml | 9 +- scripts/model_figs/make_metrics_tables.py | 62 ++-- scripts/model_figs/per_element_errors.py | 2 + site/package.json | 10 +- ...s-table-first-10k-uip-megnet-combos.svelte | 161 +++++----- site/src/figs/metrics-table-first-10k.svelte | 24 -- .../metrics-table-uip-megnet-combos.svelte | 275 ++++++++---------- ...table-uniq-protos-uip-megnet-combos.svelte | 231 +++++++++++++++ .../src/figs/metrics-table-uniq-protos.svelte | 12 + site/src/figs/metrics-table.svelte | 24 -- site/src/routes/contribute/+page.md | 2 +- .../data/mptrj/MPtrjElemCountsPtable.svelte | 2 +- site/src/routes/data/mptrj/readme.md | 2 +- site/svelte.config.js | 1 + site/tsconfig.json | 4 +- 20 files changed, 508 insertions(+), 350 deletions(-) delete mode 100644 data/figshare/readme.md rename {data => matbench_discovery}/figshare/1.0.0.json (100%) create mode 100644 matbench_discovery/figshare/readme.md create mode 100644 site/src/figs/metrics-table-uniq-protos-uip-megnet-combos.svelte diff --git a/data/figshare/readme.md b/data/figshare/readme.md deleted file mode 100644 index 71be9a27..00000000 --- a/data/figshare/readme.md +++ /dev/null @@ -1,3 +0,0 @@ -# Figshare File URLs - -Files in this directory are auto-generated by `scripts/upload_to_figshare.py`. diff --git a/matbench_discovery/__init__.py b/matbench_discovery/__init__.py index f7c33562..14e7542f 100644 --- a/matbench_discovery/__init__.py +++ b/matbench_discovery/__init__.py @@ -13,7 +13,7 @@ from pymatviz.utils import styled_html_tag pkg_name = "matbench-discovery" -direct_url = Distribution.from_name(pkg_name).read_text("direct_url.json") or "" +direct_url = Distribution.from_name(pkg_name).read_text("direct_url.json") or "{}" pkg_is_editable = json.loads(direct_url).get("dir_info", {}).get("editable", False) PKG_DIR = os.path.dirname(__file__) @@ -25,6 +25,7 @@ SITE_MODELS = f"{ROOT}/site/src/routes/models" SCRIPTS = f"{ROOT}/scripts" # model and date analysis scripts PDF_FIGS = f"{ROOT}/paper/figs" # directory for light-themed PDF figures +FIGSHARE_DIR = f"{PKG_DIR}/figshare" for directory in (SITE_FIGS, SITE_MODELS, PDF_FIGS): os.makedirs(directory, exist_ok=True) @@ -32,7 +33,6 @@ os.makedirs(MP_DIR := f"{DATA_DIR}/mp", exist_ok=True) os.makedirs(WBM_DIR := f"{DATA_DIR}/wbm", exist_ok=True) # JSON files with URLS to data files on figshare -os.makedirs(FIGSHARE_DIR := f"{ROOT}/data/figshare", exist_ok=True) # directory to store model checkpoints downloaded from wandb cloud storage CHECKPOINT_DIR = f"{ROOT}/wandb/checkpoints" @@ -108,15 +108,34 @@ class Task(StrEnum): # version of the PES like CGCNN+P) RP2RE = "RP2RE" # relaxed prototype to relaxed energy IP2RE = "IP2RE" # initial prototype to relaxed energy + IS2E = "IS2E" # initial structure to energy + IS2RE_SR = "IS2RE-SR" # initial structure to relaxed energy after ML relaxation + + +@unique +class Targets(StrEnum): + """Thermodynamic stability prediction task types.""" + + E = "E" + EFS = "EFS" + EFSM = "EFSM" + + +@unique +class ModelType(StrEnum): + """Model types.""" + + GNN = "GNN" + UIP = "UIP-GNN" + BO_GNN = "BO-GNN" + Fingerprint = "Fingerprint" + Transformer = "Transformer" -# load figshare 1.0.0 with open(f"{FIGSHARE_DIR}/1.0.0.json") as file: FIGSHARE_URLS = json.load(file) - # --- start global plot settings - ev_per_atom = styled_html_tag( "(eV/atom)", tag="span", style="font-size: 0.8em; font-weight: lighter;" ) diff --git a/data/figshare/1.0.0.json b/matbench_discovery/figshare/1.0.0.json similarity index 100% rename from data/figshare/1.0.0.json rename to matbench_discovery/figshare/1.0.0.json diff --git a/matbench_discovery/figshare/readme.md b/matbench_discovery/figshare/readme.md new file mode 100644 index 00000000..c9f0c8f4 --- /dev/null +++ b/matbench_discovery/figshare/readme.md @@ -0,0 +1,3 @@ +# Figshare File URLs + +Files in this directory are auto-generated by [`scripts/upload_to_figshare.py`](../../scripts/upload_to_figshare.py). diff --git a/matbench_discovery/preds.py b/matbench_discovery/preds.py index 5de39a13..c417ea72 100644 --- a/matbench_discovery/preds.py +++ b/matbench_discovery/preds.py @@ -57,7 +57,7 @@ class PredFiles(Files): # original MEGNet straight from publication, not re-trained megnet = "megnet/2022-11-18-megnet-wbm-IS2RE.csv.gz" # # CHGNet-relaxed structures fed into MEGNet for formation energy prediction - # chgnet_megnet = "chgnet/2023-03-04-chgnet-wbm-IS2RE.csv.gz" + # chgnet_megnet = "chgnet/2023-03-06-chgnet-0.2.0-wbm-IS2RE.csv.gz" # # M3GNet-relaxed structures fed into MEGNet for formation energy prediction # m3gnet_megnet = "m3gnet/2022-10-31-m3gnet-wbm-IS2RE.csv.gz" # megnet_rs2re = "megnet/2023-08-23-megnet-wbm-RS2RE.csv.gz" diff --git a/pyproject.toml b/pyproject.toml index 946a0e9f..85bac953 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -50,13 +50,12 @@ Package = "https://pypi.org/project/matbench-discovery" test = ["pytest", "pytest-cov"] # how to specify git deps: https://stackoverflow.com/a/73572379 running-models = [ + # aviary commented-out since dep on git repo raises "Invalid value for requires_dist" + # when attempting PyPI publish + # "aviary@git+https://github.com/CompRhys/aviary", "alignn", "chgnet", "jarvis-tools", - # torch needs to install before aviary - "torch", - - "aviary@git+https://github.com/CompRhys/aviary", "m3gnet", "mace-torch", "maml", @@ -71,7 +70,7 @@ include = ["matbench_discovery*"] exclude = ["tests", "tests.*"] [tool.setuptools.package-data] -matbench_discovery = ["../data/figshare/*"] +matbench_discovery = ["figshare/*"] [tool.distutils.bdist_wheel] universal = true diff --git a/scripts/model_figs/make_metrics_tables.py b/scripts/model_figs/make_metrics_tables.py index ea5016f8..75613481 100644 --- a/scripts/model_figs/make_metrics_tables.py +++ b/scripts/model_figs/make_metrics_tables.py @@ -14,7 +14,15 @@ from pymatviz.utils import si_fmt from sklearn.dummy import DummyClassifier -from matbench_discovery import PDF_FIGS, SCRIPTS, SITE_FIGS, Key, Task +from matbench_discovery import ( + PDF_FIGS, + SCRIPTS, + SITE_FIGS, + Key, + ModelType, + Targets, + Task, +) from matbench_discovery.data import DATA_FILES, df_wbm from matbench_discovery.metrics import stable_metrics from matbench_discovery.models import MODEL_METADATA @@ -82,25 +90,28 @@ # %% for each model this ontology dict specifies (training type, test type, model type) +model_type_col, targets_col = "Model Type", "Targets" +ontology_cols = ["Trained", "Task", model_type_col, targets_col] ontology = { - "ALIGNN": (Task.RS2RE, Task.IS2RE, "GNN"), - # "ALIGNN Pretrained": (Task.RS2RE, Task.IS2RE, "GNN"), - "CHGNet": (Task.S2EFSM, "IS2RE-SR", "UIP-GNN"), - "MACE": (Task.S2EFS, "IS2RE-SR", "UIP-GNN"), - "M3GNet": (Task.S2EFS, "IS2RE-SR", "UIP-GNN"), - "MEGNet": (Task.RS2RE, "IS2E", "GNN"), - "MEGNet RS2RE": (Task.RS2RE, "IS2E", "GNN"), - "CGCNN": (Task.RS2RE, "IS2E", "GNN"), - "CGCNN+P": (Task.S2RE, Task.IS2RE, "GNN"), - "Wrenformer": (Task.RP2RE, Task.IP2RE, "Transformer"), - "BOWSR": (Task.RS2RE, "IS2RE-BO", "BO-GNN"), - "Voronoi RF": (Task.RS2RE, "IS2E", "Fingerprint"), - "M3GNet→MEGNet": (Task.S2EFS, "IS2RE-SR", "UIP-GNN"), - "CHGNet→MEGNet": (Task.S2EFSM, "IS2RE-SR", "UIP-GNN"), - "PFP": (Task.S2EFS, Task.IS2RE, "UIP"), - "Dummy": ("", "", ""), + "ALIGNN": (Task.RS2RE, Task.IS2RE, ModelType.GNN, Targets.E), + # "ALIGNN Pretrained": (Task.RS2RE, Task.IS2RE, ModelType.GNN, Targets.E), + "CHGNet": (Task.S2EFSM, Task.IS2RE_SR, ModelType.UIP, Targets.EFSM), + "chgnet_no_relax": (Task.S2EFSM, "IS2RE-STATIC", ModelType.UIP, Targets.EFSM), + "MACE": (Task.S2EFS, Task.IS2RE_SR, ModelType.UIP, Targets.EFS), + "M3GNet": (Task.S2EFS, Task.IS2RE_SR, ModelType.UIP, Targets.EFS), + "MEGNet": (Task.RS2RE, Task.IS2E, ModelType.GNN, Targets.E), + "MEGNet RS2RE": (Task.RS2RE, Task.IS2E, ModelType.GNN, Targets.E), + "CGCNN": (Task.RS2RE, Task.IS2E, ModelType.GNN, Targets.E), + "CGCNN+P": ("S2RE", Task.IS2RE, ModelType.GNN, Targets.E), + "Wrenformer": ("RP2RE", "IP2E", ModelType.Transformer, Targets.E), + "BOWSR": (Task.RS2RE, "IS2RE-BO", ModelType.BO_GNN, Targets.E), + "Voronoi RF": (Task.RS2RE, Task.IS2E, "Fingerprint", Targets.E), + "M3GNet→MEGNet": (Task.S2EFS, Task.IS2RE_SR, ModelType.UIP, Targets.EFS), + "CHGNet→MEGNet": (Task.S2EFSM, Task.IS2RE_SR, ModelType.UIP, Targets.EFSM), + "PFP": (Task.S2EFS, Task.IS2RE, ModelType.UIP, Targets.EFS), + "GNoMe": (Task.S2EFS, Task.IS2RE, ModelType.UIP, Targets.EFS), + "Dummy": ("", "", "", ""), } -ontology_cols = ["Trained", "Deployed", model_type_col := "Model Type"] df_ont = pd.DataFrame(ontology, index=ontology_cols) # RS2RE = relaxed structure to relaxed energy # RP2RE = relaxed prototype to predicted energy @@ -128,10 +139,11 @@ # when setting to True, uncomment the lines chgnet_megnet, m3gnet_megnet, megnet_rs2re # in PredFiles! make_uip_megnet_comparison = False -show_cols = ( - f"F1,DAF,Precision,Accuracy,TPR,TNR,MAE,RMSE,{R2_col}," - f"{train_size_col},{model_type_col}".split(",") -) +meta_cols = [train_size_col, model_type_col, targets_col] +show_cols = [ + *f"F1,DAF,Precision,Accuracy,TPR,TNR,MAE,RMSE,{R2_col}".split(","), + *meta_cols, +] for label, df in ( ("", df_metrics), @@ -154,14 +166,16 @@ # abbreviate long column names df_filtered = df_filtered.rename(columns={"Precision": "Prec", "Accuracy": "Acc"}) - if label == "-first-10k": + if "-first-10k" in label: # hide redundant metrics for first 10k preds (all TPR = 1, TNR = 0) df_filtered = df_filtered.drop(["TPR", "TNR"], axis="columns") + if label != "-uniq-protos": # only show training size and model type once + df_filtered = df_filtered.drop(meta_cols, axis="columns") styler = ( df_filtered.style.format( # render integers without decimal places - {k: "{:,.0f}" for k in "TP FN FP TN".split()}, + {key: "{:,.0f}" for key in "TP FN FP TN".split()}, precision=2, # render floats with 2 decimals na_rep="", # render NaNs as empty string ) diff --git a/scripts/model_figs/per_element_errors.py b/scripts/model_figs/per_element_errors.py index 8613f94c..34653075 100644 --- a/scripts/model_figs/per_element_errors.py +++ b/scripts/model_figs/per_element_errors.py @@ -250,8 +250,10 @@ df_frac_comp * (df_each_err[model].to_numpy()[:, None]), log=True, cbar_title=f"{model} convex hull distance errors (eV/atom)", + cbar_title_kwds=dict(fontsize=16), x_range=(-0.5, 0.5), symbol_pos=(0.1, 0.8), + colormap="viridis", ) img_name = f"ptable-each-error-hists-{model.lower().replace(' ', '-')}" diff --git a/site/package.json b/site/package.json index b742ef4e..97275eb4 100644 --- a/site/package.json +++ b/site/package.json @@ -20,10 +20,10 @@ "@iconify/svelte": "^3.1.6", "@rollup/plugin-yaml": "^4.1.2", "@sveltejs/adapter-static": "^3.0.1", - "@sveltejs/kit": "^2.4.1", - "@sveltejs/vite-plugin-svelte": "^3.0.1", - "@typescript-eslint/eslint-plugin": "^6.19.0", - "@typescript-eslint/parser": "^6.19.0", + "@sveltejs/kit": "^2.5.0", + "@sveltejs/vite-plugin-svelte": "^3.0.2", + "@typescript-eslint/eslint-plugin": "^6.19.1", + "@typescript-eslint/parser": "^6.19.1", "d3-scale-chromatic": "^3.0.0", "elementari": "^0.2.3", "eslint": "^8.56.0", @@ -44,7 +44,7 @@ "svelte-multiselect": "^10.2.0", "svelte-preprocess": "^5.1.3", "svelte-toc": "^0.5.7", - "svelte-zoo": "^0.4.9", + "svelte-zoo": "^0.4.10", "svelte2tsx": "^0.7.0", "tslib": "^2.6.2", "typescript": "5.3.3", diff --git a/site/src/figs/metrics-table-first-10k-uip-megnet-combos.svelte b/site/src/figs/metrics-table-first-10k-uip-megnet-combos.svelte index 6515af3c..e3c198d2 100644 --- a/site/src/figs/metrics-table-first-10k-uip-megnet-combos.svelte +++ b/site/src/figs/metrics-table-first-10k-uip-megnet-combos.svelte @@ -13,115 +13,102 @@ padding: 4px 6px; white-space: nowrap; } -#T_ th { +#T_ th.col_heading { border: 1px solid; border-width: 1px 0; text-align: left; } +#T_ th.row_heading { + font-weight: normal; + padding: 3pt; +} #T__row0_col0, #T__row0_col1, #T__row0_col2, #T__row0_col3, #T__row0_col4, #T__row0_col5, #T__row0_col6 { background-color: #fde725; color: #000000; } #T__row1_col0 { - background-color: #d2e21b; + background-color: #90d743; + color: #000000; +} +#T__row1_col1 { + background-color: #7ad151; color: #000000; } -#T__row1_col1, #T__row1_col2, #T__row1_col3 { - background-color: #cae11f; +#T__row1_col2, #T__row1_col3 { + background-color: #77d153; color: #000000; } #T__row1_col4 { - background-color: #c0df25; + background-color: #a8db34; color: #000000; } #T__row1_col5 { - background-color: #7ad151; + background-color: #81d34d; color: #000000; } #T__row1_col6 { - background-color: #bade28; + background-color: #bddf26; color: #000000; } -#T__row2_col0, #T__row2_col4 { - background-color: #470d60; - color: #f1f1f1; -} -#T__row2_col1, #T__row2_col6 { - background-color: #460a5d; - color: #f1f1f1; -} -#T__row2_col2, #T__row2_col3 { - background-color: #460b5e; - color: #f1f1f1; -} -#T__row2_col5 { - background-color: #471164; - color: #f1f1f1; -} -#T__row3_col0, #T__row3_col1, #T__row3_col2, #T__row3_col3, #T__row4_col2, #T__row4_col3, #T__row5_col6 { +#T__row2_col0, #T__row2_col2, #T__row2_col3, #T__row4_col6 { background-color: #450457; color: #f1f1f1; } -#T__row3_col4 { - background-color: #482173; +#T__row2_col1, #T__row3_col0, #T__row3_col1, #T__row3_col2, #T__row3_col3 { + background-color: #440256; color: #f1f1f1; } -#T__row3_col5 { +#T__row2_col4, #T__row2_col5 { background-color: #482071; color: #f1f1f1; } -#T__row3_col6 { +#T__row2_col6 { background-color: #46337f; color: #f1f1f1; } -#T__row4_col0, #T__row4_col1 { - background-color: #440256; - color: #f1f1f1; -} -#T__row4_col4, #T__row4_col5, #T__row4_col6, #T__row5_col0, #T__row5_col1, #T__row5_col2, #T__row5_col3, #T__row5_col4, #T__row5_col5 { +#T__row3_col4, #T__row3_col5, #T__row3_col6, #T__row4_col0, #T__row4_col1, #T__row4_col2, #T__row4_col3, #T__row4_col4, #T__row4_col5 { background-color: #440154; color: #f1f1f1; } -#T_ :is(td, th):nth-child(6) { border-left: 1px dotted white; } -table { - scrollbar-width: none; /* Firefox */ -} -table::-webkit-scrollbar { - display: none; /* Safari and Chrome */ -} +#T_ :is(td, th):nth-child(6) { border-left: 1px solid white; } + table { + scrollbar-width: none; /* Firefox */ + } + table::-webkit-scrollbar { + display: none; /* Safari and Chrome */ + } - - - - - - - - - - - - + + + + + + + + + - - - - - - - - + + + + + + + + @@ -133,56 +120,44 @@ table::-webkit-scrollbar { - + - - - - - - - - - + + + + + + + + + - + - + - - - - - + + + + + - + - + - - + + - - - - - - - - - - - -
ModelF1DAFPrecisionAccuracyMAERMSER2training sizeModel ClassF1 ↑DAF ↑Prec ↑Acc ↑MAE ↓RMSE ↓R2 ↑Training SizeModel Type
CHGNet0.914.960.830.830.070.100.821,580,3950.945.290.880.880.060.110.811.6M (145.9K) UIP-GNN
0.10 0.16 0.62188,349188.3K (62.8K) UIP-GNN
MEGNet RS2RE0.652.880.480.480.350.37-1.0669,239MEGNet0.642.840.470.470.330.36-0.82133.4K GNN
MEGNetCHGNet→MEGNet 0.642.842.83 0.47 0.470.330.36-0.8269,239GNN0.360.38-1.111.6M (145.9K)UIP-GNN
CHGNet→MEGNetM3GNet→MEGNet 0.642.832.82 0.47 0.47 0.36 0.38-1.111,580,395-1.09188.3K (62.8K) UIP-GNN
M3GNet→MEGNet0.642.820.470.470.360.38-1.09188,349UIP-GNN
diff --git a/site/src/figs/metrics-table-first-10k.svelte b/site/src/figs/metrics-table-first-10k.svelte index ef9a7b1e..40ee4a5e 100644 --- a/site/src/figs/metrics-table-first-10k.svelte +++ b/site/src/figs/metrics-table-first-10k.svelte @@ -246,8 +246,6 @@ MAE ↓ RMSE ↓ R2 ↑ - Training Size - Model Type @@ -260,8 +258,6 @@ 0.06 0.11 0.81 - 1.6M (145.9K) - UIP-GNN MACE @@ -272,8 +268,6 @@ 0.09 0.23 0.22 - 1.6M (145.9K) - UIP-GNN M3GNet @@ -284,8 +278,6 @@ 0.10 0.16 0.62 - 188.3K (62.8K) - UIP-GNN CGCNN @@ -296,8 +288,6 @@ 0.17 0.24 0.27 - 154.7K - GNN ALIGNN @@ -308,8 +298,6 @@ 0.19 0.27 0.14 - 154.7K - GNN CGCNN+P @@ -320,8 +308,6 @@ 0.21 0.29 0.03 - 154.7K - GNN Wrenformer @@ -332,8 +318,6 @@ 0.19 0.25 0.22 - 154.7K - Transformer BOWSR @@ -344,8 +328,6 @@ 0.27 0.33 -1.10 - 133.4K - BO-GNN MEGNet @@ -356,8 +338,6 @@ 0.33 0.36 -0.82 - 133.4K - GNN Voronoi RF @@ -368,8 +348,6 @@ 0.36 0.43 -0.81 - 154.7K - Fingerprint Dummy @@ -380,8 +358,6 @@ 0.12 0.18 0.00 - - diff --git a/site/src/figs/metrics-table-uip-megnet-combos.svelte b/site/src/figs/metrics-table-uip-megnet-combos.svelte index 6ac7d89c..f55402fd 100644 --- a/site/src/figs/metrics-table-uip-megnet-combos.svelte +++ b/site/src/figs/metrics-table-uip-megnet-combos.svelte @@ -13,270 +13,223 @@ padding: 4px 6px; white-space: nowrap; } -#T_ th { +#T_ th.col_heading { border: 1px solid; border-width: 1px 0; text-align: left; } -#T__row0_col0, #T__row0_col1, #T__row0_col2, #T__row0_col3, #T__row0_col5, #T__row0_col6, #T__row0_col7, #T__row0_col8, #T__row1_col4 { +#T_ th.row_heading { + font-weight: normal; + padding: 3pt; +} +#T__row0_col0, #T__row0_col1, #T__row0_col2, #T__row0_col3, #T__row0_col6, #T__row0_col7, #T__row0_col8, #T__row1_col4, #T__row4_col5 { background-color: #fde725; color: #000000; } #T__row0_col4 { - background-color: #287c8e; - color: #f1f1f1; + background-color: #6ccd5a; + color: #000000; } -#T__row1_col0 { - background-color: #9dd93b; +#T__row0_col5 { + background-color: #e2e418; color: #000000; } +#T__row1_col0 { + background-color: #2ab07f; + color: #f1f1f1; +} #T__row1_col1 { - background-color: #3c508b; + background-color: #404688; color: #f1f1f1; } #T__row1_col2 { - background-color: #3d4d8a; + background-color: #3c4f8a; color: #f1f1f1; } #T__row1_col3 { - background-color: #375a8c; + background-color: #3a538b; color: #f1f1f1; } #T__row1_col5 { - background-color: #463480; + background-color: #472c7a; color: #f1f1f1; } #T__row1_col6 { - background-color: #f4e61e; + background-color: #90d743; color: #000000; } #T__row1_col7 { - background-color: #f1e51d; + background-color: #9dd93b; color: #000000; } #T__row1_col8 { - background-color: #f8e621; + background-color: #bade28; color: #000000; } #T__row2_col0 { - background-color: #67cc5c; - color: #000000; -} -#T__row2_col1 { - background-color: #472c7a; + background-color: #2d708e; color: #f1f1f1; } -#T__row2_col2 { - background-color: #443983; - color: #f1f1f1; -} -#T__row2_col3 { - background-color: #414487; +#T__row2_col1, #T__row2_col2, #T__row2_col3, #T__row2_col5, #T__row3_col3, #T__row4_col0, #T__row4_col4, #T__row4_col6, #T__row4_col7, #T__row4_col8 { + background-color: #440154; color: #f1f1f1; } #T__row2_col4 { - background-color: #f6e620; + background-color: #8bd646; color: #000000; } -#T__row2_col5 { - background-color: #482475; - color: #f1f1f1; -} #T__row2_col6 { - background-color: #75d054; - color: #000000; -} -#T__row2_col7 { - background-color: #70cf57; - color: #000000; -} -#T__row2_col8 { - background-color: #95d840; - color: #000000; -} -#T__row3_col0 { - background-color: #1f958b; - color: #f1f1f1; -} -#T__row3_col1, #T__row3_col2, #T__row3_col3, #T__row3_col5, #T__row4_col3, #T__row5_col0, #T__row5_col4, #T__row5_col6, #T__row5_col7, #T__row5_col8 { - background-color: #440154; + background-color: #1fa188; color: #f1f1f1; } -#T__row3_col4 { - background-color: #bddf26; - color: #000000; -} -#T__row3_col6 { - background-color: #35b779; +#T__row2_col7 { + background-color: #1fa187; color: #f1f1f1; } -#T__row3_col7 { - background-color: #2db27d; +#T__row2_col8 { + background-color: #32b67a; color: #f1f1f1; } -#T__row3_col8 { - background-color: #4cc26c; - color: #000000; -} -#T__row4_col0 { - background-color: #218f8d; +#T__row3_col0, #T__row4_col2 { + background-color: #306a8e; color: #f1f1f1; } -#T__row4_col1 { - background-color: #470d60; +#T__row3_col1 { + background-color: #470e61; color: #f1f1f1; } -#T__row4_col2 { +#T__row3_col2 { background-color: #450457; color: #f1f1f1; } -#T__row4_col4, #T__row5_col5 { - background-color: #90d743; +#T__row3_col4 { + background-color: #67cc5c; color: #000000; } -#T__row4_col5 { - background-color: #470e61; +#T__row3_col5 { + background-color: #471164; color: #f1f1f1; } -#T__row4_col6 { - background-color: #2fb47c; - color: #f1f1f1; -} -#T__row4_col7 { - background-color: #29af7f; +#T__row3_col6 { + background-color: #1e9d89; color: #f1f1f1; } -#T__row4_col8 { - background-color: #42be71; +#T__row3_col7 { + background-color: #1fa088; color: #f1f1f1; } -#T__row5_col1 { - background-color: #355f8d; +#T__row3_col8 { + background-color: #2cb17e; color: #f1f1f1; } -#T__row5_col2 { - background-color: #31688e; +#T__row4_col1 { + background-color: #33638d; color: #f1f1f1; } -#T__row5_col3 { - background-color: #21908d; +#T__row4_col3 { + background-color: #218e8d; color: #f1f1f1; } -#T_ :is(td, th):nth-child(8) { border-left: 1px dotted white; } -table { - scrollbar-width: none; /* Firefox */ -} -table::-webkit-scrollbar { - display: none; /* Safari and Chrome */ -} - - - - - - - - - - - - - - - + + + + + + + + + + + - - + + - - - - - - + + + + + + - - + + - - + + - - - + + + - - - - - - + + + + + + - - - - - + + + + + - + - + - + - - - + + + - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + +
ModelF1DAFPrecisionAccuracyTPRTNRMAERMSER2training sizeModel ClassF1 ↑DAF ↑Prec ↑Acc ↑TPR ↑TNR ↑MAE ↓RMSE ↓R2 ↑Training SizeModel Type
CHGNet0.583.060.613.04 0.52 0.840.660.880.070.110.611,580,3950.740.860.060.100.691.6M (145.9K) UIP-GNN
M3GNet0.572.670.582.65 0.45 0.800.770.810.790.80 0.070.110.60188,3490.120.58188.3K (62.8K) UIP-GNN
MEGNet RS2RE0.572.610.450.800.77M3GNet→MEGNet0.552.540.430.790.75 0.800.080.130.4769,239GNN0.090.140.36188.3K (62.8K)UIP-GNN
M3GNet→MEGNetCHGNet→MEGNet 0.552.542.56 0.43 0.790.750.74 0.80 0.090.140.36188,3490.150.341.6M (145.9K) UIP-GNN
CHGNet→MEGNet0.552.560.430.790.740.800.090.150.341,580,395UIP-GNN
MEGNet0.512.700.460.810.570.860.130.20-0.2869,239GNNMEGNet0.512.700.460.810.570.860.130.20-0.28133.4KGNN
diff --git a/site/src/figs/metrics-table-uniq-protos-uip-megnet-combos.svelte b/site/src/figs/metrics-table-uniq-protos-uip-megnet-combos.svelte new file mode 100644 index 00000000..3f221c84 --- /dev/null +++ b/site/src/figs/metrics-table-uniq-protos-uip-megnet-combos.svelte @@ -0,0 +1,231 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
ModelF1 ↑DAF ↑Prec ↑Acc ↑TPR ↑TNR ↑MAE ↓RMSE ↓R2 ↑Training SizeModel Type
CHGNet0.613.320.510.850.760.870.060.100.691.6M (145.9K)UIP-GNN
M3GNet0.572.840.440.810.800.810.070.120.58188.3K (62.8K)UIP-GNN
M3GNet→MEGNet0.542.740.420.800.760.810.090.140.38188.3K (62.8K)UIP-GNN
CHGNet→MEGNet0.542.760.420.800.750.810.090.150.361.6M (145.9K)UIP-GNN
MEGNet0.512.920.450.830.580.870.130.21-0.25133.4KGNN
diff --git a/site/src/figs/metrics-table-uniq-protos.svelte b/site/src/figs/metrics-table-uniq-protos.svelte index d2261d2a..82026983 100644 --- a/site/src/figs/metrics-table-uniq-protos.svelte +++ b/site/src/figs/metrics-table-uniq-protos.svelte @@ -338,6 +338,7 @@ R2 ↑ Training Size Model Type + Targets @@ -354,6 +355,7 @@ 0.66 1.6M (145.9K) UIP-GNN + EFS CHGNet @@ -368,6 +370,7 @@ 0.69 1.6M (145.9K) UIP-GNN + EFSM M3GNet @@ -382,6 +385,7 @@ 0.58 188.3K (62.8K) UIP-GNN + EFS ALIGNN @@ -396,6 +400,7 @@ 0.30 154.7K GNN + E MEGNet @@ -410,6 +415,7 @@ -0.25 133.4K GNN + E CGCNN @@ -424,6 +430,7 @@ -0.60 154.7K GNN + E CGCNN+P @@ -438,6 +445,7 @@ 0.02 154.7K GNN + E Wrenformer @@ -452,6 +460,7 @@ -0.04 154.7K Transformer + E BOWSR @@ -466,6 +475,7 @@ 0.15 133.4K BO-GNN + E Voronoi RF @@ -480,6 +490,7 @@ -0.33 154.7K Fingerprint + E Dummy @@ -494,6 +505,7 @@ 0.00 + diff --git a/site/src/figs/metrics-table.svelte b/site/src/figs/metrics-table.svelte index 93ad49d8..388d01d1 100644 --- a/site/src/figs/metrics-table.svelte +++ b/site/src/figs/metrics-table.svelte @@ -344,8 +344,6 @@ MAE ↓ RMSE ↓ R2 ↑ - Training Size - Model Type @@ -360,8 +358,6 @@ 0.06 0.10 0.66 - 1.6M (145.9K) - UIP-GNN CHGNet @@ -374,8 +370,6 @@ 0.06 0.10 0.69 - 1.6M (145.9K) - UIP-GNN M3GNet @@ -388,8 +382,6 @@ 0.07 0.12 0.58 - 188.3K (62.8K) - UIP-GNN ALIGNN @@ -402,8 +394,6 @@ 0.09 0.15 0.27 - 154.7K - GNN MEGNet @@ -416,8 +406,6 @@ 0.13 0.20 -0.28 - 133.4K - GNN CGCNN @@ -430,8 +418,6 @@ 0.14 0.23 -0.62 - 154.7K - GNN CGCNN+P @@ -444,8 +430,6 @@ 0.11 0.18 0.03 - 154.7K - GNN Wrenformer @@ -458,8 +442,6 @@ 0.10 0.18 -0.04 - 154.7K - Transformer BOWSR @@ -472,8 +454,6 @@ 0.12 0.16 0.14 - 133.4K - BO-GNN Voronoi RF @@ -486,8 +466,6 @@ 0.14 0.21 -0.31 - 154.7K - Fingerprint Dummy @@ -500,8 +478,6 @@ 0.12 0.18 0.00 - - diff --git a/site/src/routes/contribute/+page.md b/site/src/routes/contribute/+page.md index f3fb70b6..53dd8fc6 100644 --- a/site/src/routes/contribute/+page.md +++ b/site/src/routes/contribute/+page.md @@ -1,6 +1,6 @@ # MPtrj Target Distributions diff --git a/site/svelte.config.js b/site/svelte.config.js index 5ac0f5f8..8337b6fa 100644 --- a/site/svelte.config.js +++ b/site/svelte.config.js @@ -105,6 +105,7 @@ export default { alias: { $site: `.`, $root: `..`, + $pkg: `../matbench_discovery`, $models: `src/routes/models`, $figs: `src/figs`, }, diff --git a/site/tsconfig.json b/site/tsconfig.json index ed06dd89..80cd5a98 100644 --- a/site/tsconfig.json +++ b/site/tsconfig.json @@ -12,6 +12,6 @@ "forceConsistentCasingInFileNames": true, "resolveJsonModule": true, - "allowSyntheticDefaultImports": true - } + "allowSyntheticDefaultImports": true, + }, }