From 3d6a3609545bc83c30207224a2e350f35cc0e8a7 Mon Sep 17 00:00:00 2001 From: Jingyang <jingyang.zhang@duke.edu> Date: Mon, 26 Jun 2023 23:01:38 -0400 Subject: [PATCH 1/2] update instructions for contrib. to leaderboard --- CONTRIBUTING.md | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 76e16ca9..1568a092 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,3 +47,22 @@ From the repository folder ```shell pre-commit install ``` + +## Contributing to OpenOOD leaderboard + +We welcome new entries submitted to the leaderboard. Please follow the instructions below to submit your results. + +1. Evaluate your model/method with OpenOOD's benchmark and evaluator such that the comparison is fair. + +2. Report your new results by opening an issue. Remember to specify the following information: + +- **`Training`**: The training method of your model, e.g., `CrossEntropy`. +- **`Postprocessor`**: The postprocessor of your model, e.g., `MSP`, `ReAct`, etc. +- **`Near-OOD AUROC`**: The AUROC score of your model on the near-OOD split. +- **`Far-OOD AUROC`**: The AUROC score of your model on the far-OOD split. +- **`ID Accuracy`**: The accuracy of your model on the ID test data. +- **`Outlier Data`**: Whether your model uses the outlier data for training. +- **`Model Arch.`**: The architecture of your base classifier, e.g., `ResNet18`. +- **`Additional Description`**: Any additional description of your model, e.g., `100 epochs`, `torchvision pretrained`, etc. + +3. Ideally, send us a copy of your model checkpoint so that we can verify your results on our end. You can either upload the checkpoint to a cloud storage and share the link in the issue, or send us an email at [jz288@duke.edu](mailto:jz288@duke.edu). From fb76ad5bed65cfa3b7b8d348d6423acbbee6a0e8 Mon Sep 17 00:00:00 2001 From: Jingyang <jingyang.zhang@duke.edu> Date: Mon, 26 Jun 2023 23:02:00 -0400 Subject: [PATCH 2/2] make cider_preprocessor more concise --- openood/preprocessors/cider_preprocessor.py | 2 ++ openood/preprocessors/utils.py | 7 ++----- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/openood/preprocessors/cider_preprocessor.py b/openood/preprocessors/cider_preprocessor.py index 34b383cc..10c6c684 100644 --- a/openood/preprocessors/cider_preprocessor.py +++ b/openood/preprocessors/cider_preprocessor.py @@ -44,6 +44,8 @@ def __init__(self, config: Config): tvs_trans.Normalize(mean=self.mean, std=self.std), ]) + self.transform = TwoCropTransform(self.transform) + def setup(self, **kwargs): pass diff --git a/openood/preprocessors/utils.py b/openood/preprocessors/utils.py index 23d26b11..1bbd0633 100644 --- a/openood/preprocessors/utils.py +++ b/openood/preprocessors/utils.py @@ -1,7 +1,7 @@ from openood.utils import Config from .base_preprocessor import BasePreprocessor -from .cider_preprocessor import CiderPreprocessor, TwoCropTransform +from .cider_preprocessor import CiderPreprocessor from .csi_preprocessor import CSIPreprocessor from .cutpaste_preprocessor import CutPastePreprocessor from .draem_preprocessor import DRAEMPreprocessor @@ -31,10 +31,7 @@ def get_preprocessor(config: Config, split): } if split == 'train': - if config.preprocessor.name == 'cider': - return TwoCropTransform(CiderPreprocessor(config)) - else: - return train_preprocessors[config.preprocessor.name](config) + return train_preprocessors[config.preprocessor.name](config) else: try: return test_preprocessors[config.preprocessor.name](config)