From 55684ca845b96821a18accbbfde2f504681f471d Mon Sep 17 00:00:00 2001 From: KevinMusgrave Date: Mon, 30 May 2022 02:57:05 -0400 Subject: [PATCH 01/14] Added .github/workflows --- .github/workflows/base_test_workflow.yml | 32 ++++++++++++++++++++++ .github/workflows/run_linter.yml | 28 +++++++++++++++++++ .github/workflows/test_adapters.yml | 15 ++++++++++ .github/workflows/test_containers.yml | 15 ++++++++++ .github/workflows/test_datasets.yml | 15 ++++++++++ .github/workflows/test_frameworks.yml | 15 ++++++++++ .github/workflows/test_hooks.yml | 15 ++++++++++ .github/workflows/test_inference.yml | 15 ++++++++++ .github/workflows/test_layers.yml | 15 ++++++++++ .github/workflows/test_meta_validators.yml | 15 ++++++++++ .github/workflows/test_models.yml | 15 ++++++++++ .github/workflows/test_utils.yml | 15 ++++++++++ .github/workflows/test_validators.yml | 15 ++++++++++ .github/workflows/test_weighters.yml | 15 ++++++++++ 14 files changed, 240 insertions(+) create mode 100644 .github/workflows/base_test_workflow.yml create mode 100644 .github/workflows/run_linter.yml create mode 100644 .github/workflows/test_adapters.yml create mode 100644 .github/workflows/test_containers.yml create mode 100644 .github/workflows/test_datasets.yml create mode 100644 .github/workflows/test_frameworks.yml create mode 100644 .github/workflows/test_hooks.yml create mode 100644 .github/workflows/test_inference.yml create mode 100644 .github/workflows/test_layers.yml create mode 100644 .github/workflows/test_meta_validators.yml create mode 100644 .github/workflows/test_models.yml create mode 100644 .github/workflows/test_utils.yml create mode 100644 .github/workflows/test_validators.yml create mode 100644 .github/workflows/test_weighters.yml diff --git a/.github/workflows/base_test_workflow.yml b/.github/workflows/base_test_workflow.yml new file mode 100644 index 00000000..7413cc6b --- /dev/null +++ b/.github/workflows/base_test_workflow.yml @@ -0,0 +1,32 @@ +# This workflow will install Python dependencies, run tests and lint with a single version of Python +# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions + +name: base_test_workflow + +on: + workflow_call: + inputs: + module-to-test: + required: true + type: string + +jobs: + build: + + runs-on: ubuntu-latest + strategy: + matrix: + python-version: [3.8] + + steps: + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + pip install .[ignite,lightning,record-keeper] + - name: Run unit tests + run: | + TEST_DTYPES=float32,float64 TEST_DEVICE=cpu python -m unittest discover -t . -s tests/${{ inputs.module-to-test }} diff --git a/.github/workflows/run_linter.yml b/.github/workflows/run_linter.yml new file mode 100644 index 00000000..13b98a6c --- /dev/null +++ b/.github/workflows/run_linter.yml @@ -0,0 +1,28 @@ +name: run_linter + +on: + pull_request: + branches: [ master, dev ] + paths: + - 'src/**' + - 'tests/**' + - '.github/workflows/**' + +jobs: + build: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v2 + - name: Set up Python 3.8 + uses: actions/setup-python@v2 + with: + python-version: 3.8 + - name: Install dependencies + run: | + pip install flake8 + - name: Lint with flake8 + run: | + # stop the build if there are Python syntax errors or undefined names + flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics + # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide + flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics --per-file-ignores="__init__.py:F401" diff --git a/.github/workflows/test_adapters.yml b/.github/workflows/test_adapters.yml new file mode 100644 index 00000000..b15c1193 --- /dev/null +++ b/.github/workflows/test_adapters.yml @@ -0,0 +1,15 @@ +name: adapters + +on: + pull_request: + branches: [ master, dev ] + paths: + - 'src/**' + - 'tests/**' + - '.github/workflows/**' + +jobs: + call-base-test-workflow: + uses: ./.github/workflows/base_test_workflow.yml + with: + module-to-test: adapters \ No newline at end of file diff --git a/.github/workflows/test_containers.yml b/.github/workflows/test_containers.yml new file mode 100644 index 00000000..2ea5592d --- /dev/null +++ b/.github/workflows/test_containers.yml @@ -0,0 +1,15 @@ +name: containers + +on: + pull_request: + branches: [ master, dev ] + paths: + - 'src/**' + - 'tests/**' + - '.github/workflows/**' + +jobs: + call-base-test-workflow: + uses: ./.github/workflows/base_test_workflow.yml + with: + module-to-test: containers \ No newline at end of file diff --git a/.github/workflows/test_datasets.yml b/.github/workflows/test_datasets.yml new file mode 100644 index 00000000..359ea8e5 --- /dev/null +++ b/.github/workflows/test_datasets.yml @@ -0,0 +1,15 @@ +name: datasets + +on: + pull_request: + branches: [ master, dev ] + paths: + - 'src/**' + - 'tests/**' + - '.github/workflows/**' + +jobs: + call-base-test-workflow: + uses: ./.github/workflows/base_test_workflow.yml + with: + module-to-test: datasets \ No newline at end of file diff --git a/.github/workflows/test_frameworks.yml b/.github/workflows/test_frameworks.yml new file mode 100644 index 00000000..0c83f922 --- /dev/null +++ b/.github/workflows/test_frameworks.yml @@ -0,0 +1,15 @@ +name: frameworks + +on: + pull_request: + branches: [ master, dev ] + paths: + - 'src/**' + - 'tests/**' + - '.github/workflows/**' + +jobs: + call-base-test-workflow: + uses: ./.github/workflows/base_test_workflow.yml + with: + module-to-test: frameworks \ No newline at end of file diff --git a/.github/workflows/test_hooks.yml b/.github/workflows/test_hooks.yml new file mode 100644 index 00000000..58175597 --- /dev/null +++ b/.github/workflows/test_hooks.yml @@ -0,0 +1,15 @@ +name: hooks + +on: + pull_request: + branches: [ master, dev ] + paths: + - 'src/**' + - 'tests/**' + - '.github/workflows/**' + +jobs: + call-base-test-workflow: + uses: ./.github/workflows/base_test_workflow.yml + with: + module-to-test: hooks \ No newline at end of file diff --git a/.github/workflows/test_inference.yml b/.github/workflows/test_inference.yml new file mode 100644 index 00000000..9213c8a6 --- /dev/null +++ b/.github/workflows/test_inference.yml @@ -0,0 +1,15 @@ +name: inference + +on: + pull_request: + branches: [ master, dev ] + paths: + - 'src/**' + - 'tests/**' + - '.github/workflows/**' + +jobs: + call-base-test-workflow: + uses: ./.github/workflows/base_test_workflow.yml + with: + module-to-test: inference \ No newline at end of file diff --git a/.github/workflows/test_layers.yml b/.github/workflows/test_layers.yml new file mode 100644 index 00000000..3f4c3158 --- /dev/null +++ b/.github/workflows/test_layers.yml @@ -0,0 +1,15 @@ +name: layers + +on: + pull_request: + branches: [ master, dev ] + paths: + - 'src/**' + - 'tests/**' + - '.github/workflows/**' + +jobs: + call-base-test-workflow: + uses: ./.github/workflows/base_test_workflow.yml + with: + module-to-test: layers \ No newline at end of file diff --git a/.github/workflows/test_meta_validators.yml b/.github/workflows/test_meta_validators.yml new file mode 100644 index 00000000..0979c829 --- /dev/null +++ b/.github/workflows/test_meta_validators.yml @@ -0,0 +1,15 @@ +name: meta_validators + +on: + pull_request: + branches: [ master, dev ] + paths: + - 'src/**' + - 'tests/**' + - '.github/workflows/**' + +jobs: + call-base-test-workflow: + uses: ./.github/workflows/base_test_workflow.yml + with: + module-to-test: meta_validators \ No newline at end of file diff --git a/.github/workflows/test_models.yml b/.github/workflows/test_models.yml new file mode 100644 index 00000000..1405527f --- /dev/null +++ b/.github/workflows/test_models.yml @@ -0,0 +1,15 @@ +name: models + +on: + pull_request: + branches: [ master, dev ] + paths: + - 'src/**' + - 'tests/**' + - '.github/workflows/**' + +jobs: + call-base-test-workflow: + uses: ./.github/workflows/base_test_workflow.yml + with: + module-to-test: models \ No newline at end of file diff --git a/.github/workflows/test_utils.yml b/.github/workflows/test_utils.yml new file mode 100644 index 00000000..3766e576 --- /dev/null +++ b/.github/workflows/test_utils.yml @@ -0,0 +1,15 @@ +name: utils + +on: + pull_request: + branches: [ master, dev ] + paths: + - 'src/**' + - 'tests/**' + - '.github/workflows/**' + +jobs: + call-base-test-workflow: + uses: ./.github/workflows/base_test_workflow.yml + with: + module-to-test: utils \ No newline at end of file diff --git a/.github/workflows/test_validators.yml b/.github/workflows/test_validators.yml new file mode 100644 index 00000000..acfb6c70 --- /dev/null +++ b/.github/workflows/test_validators.yml @@ -0,0 +1,15 @@ +name: validators + +on: + pull_request: + branches: [ master, dev ] + paths: + - 'src/**' + - 'tests/**' + - '.github/workflows/**' + +jobs: + call-base-test-workflow: + uses: ./.github/workflows/base_test_workflow.yml + with: + module-to-test: validators \ No newline at end of file diff --git a/.github/workflows/test_weighters.yml b/.github/workflows/test_weighters.yml new file mode 100644 index 00000000..73add1ae --- /dev/null +++ b/.github/workflows/test_weighters.yml @@ -0,0 +1,15 @@ +name: weighters + +on: + pull_request: + branches: [ master, dev ] + paths: + - 'src/**' + - 'tests/**' + - '.github/workflows/**' + +jobs: + call-base-test-workflow: + uses: ./.github/workflows/base_test_workflow.yml + with: + module-to-test: weighters \ No newline at end of file From 2e26a9a543b5c9a25dab1a4061ba0f9a47a94088 Mon Sep 17 00:00:00 2001 From: KevinMusgrave Date: Mon, 30 May 2022 02:58:37 -0400 Subject: [PATCH 02/14] Fix branch name in workflows --- .github/workflows/run_linter.yml | 2 +- .github/workflows/test_adapters.yml | 2 +- .github/workflows/test_containers.yml | 2 +- .github/workflows/test_datasets.yml | 2 +- .github/workflows/test_frameworks.yml | 2 +- .github/workflows/test_hooks.yml | 2 +- .github/workflows/test_inference.yml | 2 +- .github/workflows/test_layers.yml | 2 +- .github/workflows/test_meta_validators.yml | 2 +- .github/workflows/test_models.yml | 2 +- .github/workflows/test_utils.yml | 2 +- .github/workflows/test_validators.yml | 2 +- .github/workflows/test_weighters.yml | 2 +- 13 files changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/run_linter.yml b/.github/workflows/run_linter.yml index 13b98a6c..eecb6d40 100644 --- a/.github/workflows/run_linter.yml +++ b/.github/workflows/run_linter.yml @@ -2,7 +2,7 @@ name: run_linter on: pull_request: - branches: [ master, dev ] + branches: [ main, dev ] paths: - 'src/**' - 'tests/**' diff --git a/.github/workflows/test_adapters.yml b/.github/workflows/test_adapters.yml index b15c1193..899e6edb 100644 --- a/.github/workflows/test_adapters.yml +++ b/.github/workflows/test_adapters.yml @@ -2,7 +2,7 @@ name: adapters on: pull_request: - branches: [ master, dev ] + branches: [ main, dev ] paths: - 'src/**' - 'tests/**' diff --git a/.github/workflows/test_containers.yml b/.github/workflows/test_containers.yml index 2ea5592d..936d5211 100644 --- a/.github/workflows/test_containers.yml +++ b/.github/workflows/test_containers.yml @@ -2,7 +2,7 @@ name: containers on: pull_request: - branches: [ master, dev ] + branches: [ main, dev ] paths: - 'src/**' - 'tests/**' diff --git a/.github/workflows/test_datasets.yml b/.github/workflows/test_datasets.yml index 359ea8e5..7e175d10 100644 --- a/.github/workflows/test_datasets.yml +++ b/.github/workflows/test_datasets.yml @@ -2,7 +2,7 @@ name: datasets on: pull_request: - branches: [ master, dev ] + branches: [ main, dev ] paths: - 'src/**' - 'tests/**' diff --git a/.github/workflows/test_frameworks.yml b/.github/workflows/test_frameworks.yml index 0c83f922..2cff026d 100644 --- a/.github/workflows/test_frameworks.yml +++ b/.github/workflows/test_frameworks.yml @@ -2,7 +2,7 @@ name: frameworks on: pull_request: - branches: [ master, dev ] + branches: [ main, dev ] paths: - 'src/**' - 'tests/**' diff --git a/.github/workflows/test_hooks.yml b/.github/workflows/test_hooks.yml index 58175597..bf9755d6 100644 --- a/.github/workflows/test_hooks.yml +++ b/.github/workflows/test_hooks.yml @@ -2,7 +2,7 @@ name: hooks on: pull_request: - branches: [ master, dev ] + branches: [ main, dev ] paths: - 'src/**' - 'tests/**' diff --git a/.github/workflows/test_inference.yml b/.github/workflows/test_inference.yml index 9213c8a6..9d3ef7f9 100644 --- a/.github/workflows/test_inference.yml +++ b/.github/workflows/test_inference.yml @@ -2,7 +2,7 @@ name: inference on: pull_request: - branches: [ master, dev ] + branches: [ main, dev ] paths: - 'src/**' - 'tests/**' diff --git a/.github/workflows/test_layers.yml b/.github/workflows/test_layers.yml index 3f4c3158..43950efe 100644 --- a/.github/workflows/test_layers.yml +++ b/.github/workflows/test_layers.yml @@ -2,7 +2,7 @@ name: layers on: pull_request: - branches: [ master, dev ] + branches: [ main, dev ] paths: - 'src/**' - 'tests/**' diff --git a/.github/workflows/test_meta_validators.yml b/.github/workflows/test_meta_validators.yml index 0979c829..e5acc505 100644 --- a/.github/workflows/test_meta_validators.yml +++ b/.github/workflows/test_meta_validators.yml @@ -2,7 +2,7 @@ name: meta_validators on: pull_request: - branches: [ master, dev ] + branches: [ main, dev ] paths: - 'src/**' - 'tests/**' diff --git a/.github/workflows/test_models.yml b/.github/workflows/test_models.yml index 1405527f..2e53f54a 100644 --- a/.github/workflows/test_models.yml +++ b/.github/workflows/test_models.yml @@ -2,7 +2,7 @@ name: models on: pull_request: - branches: [ master, dev ] + branches: [ main, dev ] paths: - 'src/**' - 'tests/**' diff --git a/.github/workflows/test_utils.yml b/.github/workflows/test_utils.yml index 3766e576..61191772 100644 --- a/.github/workflows/test_utils.yml +++ b/.github/workflows/test_utils.yml @@ -2,7 +2,7 @@ name: utils on: pull_request: - branches: [ master, dev ] + branches: [ main, dev ] paths: - 'src/**' - 'tests/**' diff --git a/.github/workflows/test_validators.yml b/.github/workflows/test_validators.yml index acfb6c70..b8757d44 100644 --- a/.github/workflows/test_validators.yml +++ b/.github/workflows/test_validators.yml @@ -2,7 +2,7 @@ name: validators on: pull_request: - branches: [ master, dev ] + branches: [ main, dev ] paths: - 'src/**' - 'tests/**' diff --git a/.github/workflows/test_weighters.yml b/.github/workflows/test_weighters.yml index 73add1ae..030a2b46 100644 --- a/.github/workflows/test_weighters.yml +++ b/.github/workflows/test_weighters.yml @@ -2,7 +2,7 @@ name: weighters on: pull_request: - branches: [ master, dev ] + branches: [ main, dev ] paths: - 'src/**' - 'tests/**' From ee06449ab086a71a94043e893d1a8145b0c810ee Mon Sep 17 00:00:00 2001 From: KevinMusgrave Date: Mon, 30 May 2022 03:12:55 -0400 Subject: [PATCH 03/14] Use isclose in test_adda --- tests/hooks/test_adda.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/hooks/test_adda.py b/tests/hooks/test_adda.py index 1c4b7eff..c8837d8e 100644 --- a/tests/hooks/test_adda.py +++ b/tests/hooks/test_adda.py @@ -150,7 +150,7 @@ def test_adda(self): ) total_loss = sum(total_loss) / len(total_loss) - self.assertTrue(total_loss == losses["g_loss"]["total"]) + self.assertTrue(np.isclose(total_loss.item(), losses["g_loss"]["total"])) g_opts.zero_grad() total_loss.backward() g_opts.step() From 9353669347715973ca38d31fa5be9812830a8214 Mon Sep 17 00:00:00 2001 From: KevinMusgrave Date: Mon, 30 May 2022 03:25:55 -0400 Subject: [PATCH 04/14] Add manual_seed to hook tests --- tests/hooks/test_adabn.py | 1 + tests/hooks/test_adda.py | 2 +- tests/hooks/test_aligners.py | 1 + tests/hooks/test_atdoc.py | 1 + tests/hooks/test_bsp.py | 1 + tests/hooks/test_cdan.py | 1 + tests/hooks/test_classification.py | 2 ++ tests/hooks/test_conditions.py | 1 + tests/hooks/test_dann.py | 1 + tests/hooks/test_diversity.py | 1 + tests/hooks/test_domain.py | 1 + tests/hooks/test_domain_confusion.py | 1 + tests/hooks/test_entropy.py | 1 + tests/hooks/test_features.py | 4 ++++ tests/hooks/test_gvb.py | 1 + tests/hooks/test_itl.py | 1 + tests/hooks/test_mcd.py | 2 ++ tests/hooks/test_reducers.py | 1 + tests/hooks/test_rtn.py | 4 ++++ tests/hooks/test_symnets.py | 4 ++++ tests/hooks/test_vada.py | 2 ++ 21 files changed, 33 insertions(+), 1 deletion(-) diff --git a/tests/hooks/test_adabn.py b/tests/hooks/test_adabn.py index 2bd9bda7..377dcd60 100644 --- a/tests/hooks/test_adabn.py +++ b/tests/hooks/test_adabn.py @@ -36,6 +36,7 @@ def forward(self, x, domain): class TestAdaBN(unittest.TestCase): def test_adabn_hook(self): + torch.manual_seed(530) src_imgs = torch.randn(100, 32, 8, 8) target_imgs = torch.randn(100, 32, 8, 8) src_domain = torch.zeros(100) diff --git a/tests/hooks/test_adda.py b/tests/hooks/test_adda.py index c8837d8e..1c4b7eff 100644 --- a/tests/hooks/test_adda.py +++ b/tests/hooks/test_adda.py @@ -150,7 +150,7 @@ def test_adda(self): ) total_loss = sum(total_loss) / len(total_loss) - self.assertTrue(np.isclose(total_loss.item(), losses["g_loss"]["total"])) + self.assertTrue(total_loss == losses["g_loss"]["total"]) g_opts.zero_grad() total_loss.backward() g_opts.step() diff --git a/tests/hooks/test_aligners.py b/tests/hooks/test_aligners.py index 9295ad3b..16addd59 100644 --- a/tests/hooks/test_aligners.py +++ b/tests/hooks/test_aligners.py @@ -33,6 +33,7 @@ def test_equivalent_adapter(G, C, data, aligner_hook, loss_fn): class TestAligners(unittest.TestCase): def test_aligner_plus_classifier_hook(self): + torch.manual_seed(10) for loss_fn in [MMDLoss, CORALLoss]: for joint in [False, True]: if loss_fn == CORALLoss and joint: diff --git a/tests/hooks/test_atdoc.py b/tests/hooks/test_atdoc.py index 5a0bb69f..120bc0c4 100644 --- a/tests/hooks/test_atdoc.py +++ b/tests/hooks/test_atdoc.py @@ -22,6 +22,7 @@ def get_data(dataset_size, feature_dim, num_classes, batch_size): class TestATDOC(unittest.TestCase): def test_atdoc_hook(self): + torch.manual_seed(922) dataset_size = 10000 feature_dim = 128 num_classes = 10 diff --git a/tests/hooks/test_bsp.py b/tests/hooks/test_bsp.py index 767ea4aa..20866013 100644 --- a/tests/hooks/test_bsp.py +++ b/tests/hooks/test_bsp.py @@ -10,6 +10,7 @@ class TestBSP(unittest.TestCase): def test_bsp_hook(self): + torch.manual_seed(453094) h = BSPHook() src_imgs = torch.randn(100, 32) target_imgs = torch.randn(100, 32) diff --git a/tests/hooks/test_cdan.py b/tests/hooks/test_cdan.py index ec4f3bad..627a9a7a 100644 --- a/tests/hooks/test_cdan.py +++ b/tests/hooks/test_cdan.py @@ -87,6 +87,7 @@ def get_correct_domain_losses( class TestCDAN(unittest.TestCase): def test_cdan_domain_hooks(self): + torch.manual_seed(109483) for fc_out_size in [15, 32, 16]: bad_fc_size = fc_out_size in [15, 32] for softmax in [False, True]: diff --git a/tests/hooks/test_classification.py b/tests/hooks/test_classification.py index 6a029b92..4064ed60 100644 --- a/tests/hooks/test_classification.py +++ b/tests/hooks/test_classification.py @@ -9,6 +9,7 @@ class TestClassification(unittest.TestCase): def test_softmax_hook(self): + torch.manual_seed(8112) h = SoftmaxHook(apply_to=["src_imgs_features"]) src_imgs_features = torch.randn(100, 10) outputs, losses = h(locals()) @@ -21,6 +22,7 @@ def test_softmax_hook(self): ) def test_closs_hook(self): + torch.manual_seed(24242) for detach_features in [True, False]: h = CLossHook(detach_features=detach_features) src_imgs = torch.randn(100, 32) diff --git a/tests/hooks/test_conditions.py b/tests/hooks/test_conditions.py index 43646369..4c59e6de 100644 --- a/tests/hooks/test_conditions.py +++ b/tests/hooks/test_conditions.py @@ -10,6 +10,7 @@ class TestConditions(unittest.TestCase): def test_strong_d_hook(self): + torch.manual_seed(3420) for threshold in [0.1, 0.4, 0.6]: h = StrongDHook(threshold=threshold) src_domain = torch.zeros(10) diff --git a/tests/hooks/test_dann.py b/tests/hooks/test_dann.py index f4439131..969e39f9 100644 --- a/tests/hooks/test_dann.py +++ b/tests/hooks/test_dann.py @@ -44,6 +44,7 @@ def test_equivalent_adapter(G, D, C, data, post_g): class TestDANN(unittest.TestCase): def test_dann(self): + torch.manual_seed(65498) for post_g in [None, [BSPHook()], [BNMHook()], [MCCHook()], [AFNHook()]]: for hook_cls in [DANNHook, DANNEHook, CDANNEHook, DANNSoftmaxLogitsHook]: for detach_reducer in [False, True]: diff --git a/tests/hooks/test_diversity.py b/tests/hooks/test_diversity.py index f7c5d965..6c0d1dbc 100644 --- a/tests/hooks/test_diversity.py +++ b/tests/hooks/test_diversity.py @@ -10,6 +10,7 @@ class TestDiversity(unittest.TestCase): def test_diversity_hook(self): + torch.manual_seed(1123) h = TargetDiversityHook() src_imgs = torch.randn(100, 32) target_imgs = torch.randn(100, 32) diff --git a/tests/hooks/test_domain.py b/tests/hooks/test_domain.py index 1e4b5c81..3f241a10 100644 --- a/tests/hooks/test_domain.py +++ b/tests/hooks/test_domain.py @@ -9,6 +9,7 @@ class TestDomain(unittest.TestCase): def test_domain_loss_hook(self): + torch.manual_seed(493929) for detach_features in [False, True]: for use_logits in [True, False]: if use_logits: diff --git a/tests/hooks/test_domain_confusion.py b/tests/hooks/test_domain_confusion.py index dc8ba41e..c3c0e863 100644 --- a/tests/hooks/test_domain_confusion.py +++ b/tests/hooks/test_domain_confusion.py @@ -9,6 +9,7 @@ class TestDomainConfusion(unittest.TestCase): def test_domain_confusion_hook(self): + torch.manual_seed(9999) ( G, C, diff --git a/tests/hooks/test_entropy.py b/tests/hooks/test_entropy.py index 1a16c46e..a3e46484 100644 --- a/tests/hooks/test_entropy.py +++ b/tests/hooks/test_entropy.py @@ -10,6 +10,7 @@ class TestEntropy(unittest.TestCase): def test_entropy_hook(self): + torch.manual_seed(1332) h = TargetEntropyHook() src_imgs = torch.randn(100, 32) target_imgs = torch.randn(100, 32) diff --git a/tests/hooks/test_features.py b/tests/hooks/test_features.py index 5e2cb7c6..77abca52 100644 --- a/tests/hooks/test_features.py +++ b/tests/hooks/test_features.py @@ -15,6 +15,7 @@ class TestFeatures(unittest.TestCase): def test_features_hook_logits_hook(self): + torch.manual_seed(144) for hook_type in ["features", "logits"]: G = Net(32, 8) if hook_type == "features": @@ -81,6 +82,7 @@ def test_features_hook_logits_hook(self): self.assertTrue(G.count == 3) def test_features_and_logits_hook(self): + torch.manual_seed(145) G = Net(32, 8) C = Net(8, 2) src_imgs = torch.randn(100, 32) @@ -135,6 +137,7 @@ def test_features_and_logits_hook(self): self.assertTrue(C.count == 5) def test_d_logits_hook(self): + torch.manual_seed(146) D = Net(32, 2) src_imgs_features = torch.randn(100, 32) target_imgs_features = torch.randn(100, 32) @@ -171,6 +174,7 @@ def test_d_logits_hook(self): outputs, losses = hook(locals()) def test_multiple_detach_modes(self): + torch.manual_seed(147) G = Net(32, 8) C = Net(8, 2) src_imgs = torch.randn(100, 32) diff --git a/tests/hooks/test_gvb.py b/tests/hooks/test_gvb.py index 70fbca9b..2e5ea56b 100644 --- a/tests/hooks/test_gvb.py +++ b/tests/hooks/test_gvb.py @@ -61,6 +61,7 @@ def get_models_and_data(): class TestGVB(unittest.TestCase): def test_gvb_hook(self): + torch.manual_seed(509) for hook_cls in [GVBHook, GVBEHook]: for detach_reducer in [False, True]: if detach_reducer and hook_cls is not GVBEHook: diff --git a/tests/hooks/test_itl.py b/tests/hooks/test_itl.py index b1dc033a..1c3cf65b 100644 --- a/tests/hooks/test_itl.py +++ b/tests/hooks/test_itl.py @@ -10,6 +10,7 @@ class TestITL(unittest.TestCase): def test_ist_loss_hook(self): + torch.manual_seed(334) h = ISTLossHook() ( G, diff --git a/tests/hooks/test_mcd.py b/tests/hooks/test_mcd.py index 876ec098..8e4cc0e0 100644 --- a/tests/hooks/test_mcd.py +++ b/tests/hooks/test_mcd.py @@ -49,6 +49,7 @@ def get_disc_loss(loss_fn, target_logits): class TestMCD(unittest.TestCase): def test_multiple_c_loss_hook(self): + torch.manual_seed(435) for detach in [False, True]: h = MultipleCLossHook(num_c=3, detach_features=detach) batch_size = 32 @@ -91,6 +92,7 @@ def test_multiple_c_loss_hook(self): ) def test_mcd_loss_hook(self): + torch.manual_seed(242) for detach in [False, True]: h = MCDLossHook(detach_features=detach) batch_size = 32 diff --git a/tests/hooks/test_reducers.py b/tests/hooks/test_reducers.py index 2dc0606b..636935b0 100644 --- a/tests/hooks/test_reducers.py +++ b/tests/hooks/test_reducers.py @@ -14,6 +14,7 @@ class TestReducers(unittest.TestCase): def test_entropy_reducer(self): + torch.manual_seed(1335) ( G, C, diff --git a/tests/hooks/test_rtn.py b/tests/hooks/test_rtn.py index 2b3a3756..90645536 100644 --- a/tests/hooks/test_rtn.py +++ b/tests/hooks/test_rtn.py @@ -49,6 +49,7 @@ def test_equivalent_adapter(G, C, R, feature_combiner, data): class TestRTN(unittest.TestCase): def test_residual_hook(self): + torch.manual_seed(5409) src_imgs = torch.randn(100, 32) target_imgs = torch.randn(100, 32) G = Net(32, 16) @@ -87,6 +88,7 @@ def test_residual_hook(self): ) def test_rtn_logits_hook(self): + torch.manual_seed(9829) src_imgs = torch.randn(100, 32) src_labels = torch.randint(0, 10, size=(100,)) target_imgs = torch.randn(100, 32) @@ -125,6 +127,7 @@ def test_rtn_logits_hook(self): self.assertTrue(torch.isclose(correct_c_loss, torch.mean(losses["c_loss"]))) def test_rtn_aligner_hook(self): + torch.manual_seed(2345) src_imgs = torch.randn(100, 32) target_imgs = torch.randn(100, 32) G = Net(32, 16) @@ -163,6 +166,7 @@ def test_rtn_aligner_hook(self): self.assertTrue(confusion_loss == losses["features_confusion_loss"]) def test_rtn_hook(self): + torch.manual_seed(3233) ( G, C, diff --git a/tests/hooks/test_symnets.py b/tests/hooks/test_symnets.py index 598f8ffd..3a053f3d 100644 --- a/tests/hooks/test_symnets.py +++ b/tests/hooks/test_symnets.py @@ -69,6 +69,7 @@ def get_correct_domain_loss(G, C, src_imgs, target_imgs, num_classes, domain, ha class TestSymNets(unittest.TestCase): def test_symnets_entropy_hook(self): + torch.manual_seed(32454) G, C, src_imgs, target_imgs, batch_size, num_classes = get_model_and_data() h = SymNetsEntropyHook() @@ -85,6 +86,7 @@ def test_symnets_entropy_hook(self): self.assertTrue(torch.isclose(losses["symnets_entropy_loss"], correct_entropy)) def test_symnets_domain_loss_hook(self): + torch.manual_seed(54498) G, C, src_imgs, target_imgs, batch_size, num_classes = get_model_and_data() TestG = copy.deepcopy(G) TestC = copy.deepcopy(C) @@ -121,6 +123,7 @@ def test_symnets_domain_loss_hook(self): self.assertTrue(torch.isclose(losses[h.loss_keys[0]], correct_loss)) def test_symnets_category_loss_hook(self): + torch.manual_seed(349587) G, C, src_imgs, target_imgs, batch_size, num_classes = get_model_and_data() src_labels = torch.randint(0, num_classes, size=(batch_size,)) h = SymNetsCategoryLossHook() @@ -139,6 +142,7 @@ def test_symnets_category_loss_hook(self): self.assertTrue(torch.isclose(losses["symnets_category_loss"], loss)) def test_symnets(self): + torch.manual_seed(29345) G, C, src_imgs, target_imgs, batch_size, num_classes = get_model_and_data() originalG = copy.deepcopy(G) diff --git a/tests/hooks/test_vada.py b/tests/hooks/test_vada.py index 1e825478..c04fab20 100644 --- a/tests/hooks/test_vada.py +++ b/tests/hooks/test_vada.py @@ -10,6 +10,7 @@ class TestVADA(unittest.TestCase): def test_vat_hook(self): + torch.manual_seed(87948) src_imgs = torch.randn(100, 32) target_imgs = torch.randn(100, 32) G = Net(32, 16) @@ -36,6 +37,7 @@ def test_vat_hook(self): self.assertTrue(losses.keys() == {"src_vat_loss", "target_vat_loss"}) def test_vat_plus_entropy_hook(self): + torch.manual_seed(27391) src_imgs = torch.randn(100, 32) target_imgs = torch.randn(100, 32) G = Net(32, 16) From 19e220a1cb0ae595a9ca8c1abe0b4669ddd76b29 Mon Sep 17 00:00:00 2001 From: KevinMusgrave Date: Mon, 30 May 2022 04:04:37 -0400 Subject: [PATCH 05/14] Some changes to hopefully make tests pass more consistently --- tests/hooks/test_adda.py | 2 +- tests/layers/test_ist_loss.py | 4 +++- tests/layers/test_neighborhood_aggregation.py | 10 +++++++--- tests/layers/test_silhouette_score.py | 2 +- 4 files changed, 12 insertions(+), 6 deletions(-) diff --git a/tests/hooks/test_adda.py b/tests/hooks/test_adda.py index 1c4b7eff..bfefd17d 100644 --- a/tests/hooks/test_adda.py +++ b/tests/hooks/test_adda.py @@ -150,7 +150,7 @@ def test_adda(self): ) total_loss = sum(total_loss) / len(total_loss) - self.assertTrue(total_loss == losses["g_loss"]["total"]) + self.assertAlmostEqual(total_loss.item(), losses["g_loss"]["total"]) g_opts.zero_grad() total_loss.backward() g_opts.step() diff --git a/tests/layers/test_ist_loss.py b/tests/layers/test_ist_loss.py index aa112159..a2e2e65c 100644 --- a/tests/layers/test_ist_loss.py +++ b/tests/layers/test_ist_loss.py @@ -77,7 +77,9 @@ def test_ist_loss(self): ] * torch.log(mean_preds[1]) correct_loss += -div - self.assertTrue(torch.isclose(loss, correct_loss)) + self.assertAlmostEqual( + loss.item(), correct_loss.item(), places=6 + ) loss.backward() grad1 = x.grad.clone() diff --git a/tests/layers/test_neighborhood_aggregation.py b/tests/layers/test_neighborhood_aggregation.py index 1fc1ad14..025ac645 100644 --- a/tests/layers/test_neighborhood_aggregation.py +++ b/tests/layers/test_neighborhood_aggregation.py @@ -1,5 +1,6 @@ import unittest +import numpy as np import torch from pytorch_adapt.layers import ConfidenceWeights, NeighborhoodAggregation @@ -14,7 +15,7 @@ def get_correct_pseudo_labels(features, feat_memory, pred_memory, idx, k): dis[di, idx[di]] = torch.max(dis) _, p1 = torch.sort(dis, dim=1) - w = torch.zeros(features.size(0), feat_memory.size(0)).cuda() + w = torch.zeros(features.size(0), feat_memory.size(0), device=pred_memory.device) for wi in range(w.size(0)): for wj in range(k): w[wi][p1[wi, wj]] = 1 / k @@ -42,7 +43,9 @@ def get_correct_memory_update( class TestNeighborhoodAggregation(unittest.TestCase): def test_neighborhood_aggregation(self): - dataset_size = 100000 + torch.manual_seed(209843) + np.random.seed(34298) + dataset_size = 10000 feature_dim = 2048 num_classes = 123 for k in [5, 11, 29]: @@ -59,7 +62,8 @@ def test_neighborhood_aggregation(self): for i in range(10): features = torch.randn(batch_size, feature_dim, device=TEST_DEVICE) logits = torch.randn(batch_size, num_classes, device=TEST_DEVICE) - idx = torch.randint(0, dataset_size, size=(64,)) + idx = np.random.choice(dataset_size, size=(64,), replace=False) + idx = torch.from_numpy(idx) pseudo_labels, neighbor_preds = na( features, logits, update=True, idx=idx diff --git a/tests/layers/test_silhouette_score.py b/tests/layers/test_silhouette_score.py index e730b3dc..bf99d948 100644 --- a/tests/layers/test_silhouette_score.py +++ b/tests/layers/test_silhouette_score.py @@ -14,7 +14,7 @@ class TestSilhouetteScore(unittest.TestCase): def test_silhouette_score(self): fn = SilhouetteScore() for embedding_size in [10, 100]: - for num_embeddings in [1000, 10000]: + for num_embeddings in [1000, 2000]: for num_classes in [1, 12, 123, num_embeddings]: features = torch.randn( num_embeddings, embedding_size, device=TEST_DEVICE From a43cc29b396cc38d09e9f61a12340937f44d97ba Mon Sep 17 00:00:00 2001 From: KevinMusgrave Date: Mon, 30 May 2022 04:45:42 -0400 Subject: [PATCH 06/14] Use CustomKNN in some validator tests to avoid depending on faiss --- tests/hooks/test_adda.py | 4 +++- tests/validators/test_deep_embedded_validator.py | 6 +++++- tests/validators/test_knn_validator.py | 13 +++++++++---- tests/validators/test_per_class_validator.py | 9 +++++++-- tests/validators/test_target_knn_validator.py | 6 +++++- tests/validators/utils.py | 6 ++++++ 6 files changed, 35 insertions(+), 9 deletions(-) create mode 100644 tests/validators/utils.py diff --git a/tests/hooks/test_adda.py b/tests/hooks/test_adda.py index bfefd17d..edf8957f 100644 --- a/tests/hooks/test_adda.py +++ b/tests/hooks/test_adda.py @@ -150,7 +150,9 @@ def test_adda(self): ) total_loss = sum(total_loss) / len(total_loss) - self.assertAlmostEqual(total_loss.item(), losses["g_loss"]["total"]) + self.assertAlmostEqual( + total_loss.item(), losses["g_loss"]["total"], places=6 + ) g_opts.zero_grad() total_loss.backward() g_opts.step() diff --git a/tests/validators/test_deep_embedded_validator.py b/tests/validators/test_deep_embedded_validator.py index 1075b355..d62525cd 100644 --- a/tests/validators/test_deep_embedded_validator.py +++ b/tests/validators/test_deep_embedded_validator.py @@ -100,6 +100,7 @@ def get_correct_score(cls, src_train, target_train, src_val): class TestDeepEmbeddedValidator(unittest.TestCase): def test_deep_embedded_validator(self): + self.longMessage = True torch.manual_seed(1234) np.random.seed(1234) validator = DeepEmbeddedValidator( @@ -133,7 +134,10 @@ def test_deep_embedded_validator(self): correct_score = get_correct_score(self, src_train, target_train, src_val) correct_score = -correct_score - self.assertTrue(np.isclose(score, correct_score, rtol=0.2)) + self.assertTrue( + np.isclose(score, correct_score, rtol=0.2), + msg=f"score={score}, correct_score={correct_score}", + ) shutil.rmtree(TEST_FOLDER) diff --git a/tests/validators/test_knn_validator.py b/tests/validators/test_knn_validator.py index f790b2cd..8f696928 100644 --- a/tests/validators/test_knn_validator.py +++ b/tests/validators/test_knn_validator.py @@ -7,13 +7,15 @@ from pytorch_adapt.validators.knn_validator import BatchedAccuracyCalculator from .. import TEST_DEVICE +from .utils import get_knn_func class TestKNNValidator(unittest.TestCase): def test_knn_validator(self): torch.cuda.empty_cache() - knn_validator = ScoreHistory(KNNValidator()) - cluster_validator = ScoreHistory(KNNValidator(metric="AMI")) + knn_func = get_knn_func() + knn_validator = ScoreHistory(KNNValidator(knn_func=knn_func)) + cluster_validator = ScoreHistory(KNNValidator(metric="AMI", knn_func=knn_func)) for epoch in [1, 2]: for validator in [knn_validator, cluster_validator]: dataset_size = 10000 @@ -46,6 +48,7 @@ def test_knn_validator(self): def test_batched_vs_regular(self): metric = "mean_average_precision" + knn_func = get_knn_func() for batch_size in [None, 10, 99, 128, 500, 512]: for dataset_size in [10, 100, 1000, 10000]: src_features = torch.randn(dataset_size, 64, device=TEST_DEVICE) @@ -56,8 +59,10 @@ def test_batched_vs_regular(self): src_train = {"features": src_features, "domain": src_domain} target_train = {"features": target_features, "domain": target_domain} - v1 = KNNValidator(metric=metric) - v2 = KNNValidator(batch_size=batch_size, metric=metric) + v1 = KNNValidator(metric=metric, knn_func=knn_func) + v2 = KNNValidator( + batch_size=batch_size, metric=metric, knn_func=knn_func + ) if batch_size is not None: self.assertTrue(isinstance(v1.acc_fn, AccuracyCalculator)) diff --git a/tests/validators/test_per_class_validator.py b/tests/validators/test_per_class_validator.py index 14115664..ff39802c 100644 --- a/tests/validators/test_per_class_validator.py +++ b/tests/validators/test_per_class_validator.py @@ -13,6 +13,7 @@ from pytorch_adapt.validators.per_class_validator import get_common_labels from .. import TEST_DEVICE +from .utils import get_knn_func def get_data(dataset_size, domain, label_range=None): @@ -58,10 +59,14 @@ class TestPerClassValidator(unittest.TestCase): def test_per_class_validator(self): torch.manual_seed(204) dataset_size = 256 + knn_func = get_knn_func() inner_validators = [ - KNNValidator(key_map={"src_val": "src_train"}, metric="AMI"), KNNValidator( - key_map={"src_val": "src_train", "target_val": "target_train"} + key_map={"src_val": "src_train"}, metric="AMI", knn_func=knn_func + ), + KNNValidator( + key_map={"src_val": "src_train", "target_val": "target_train"}, + knn_func=knn_func, ), MMDValidator(mmd_kwargs={"mmd_type": "quadratic"}), SNDValidator(), diff --git a/tests/validators/test_target_knn_validator.py b/tests/validators/test_target_knn_validator.py index adba1a67..0707603b 100644 --- a/tests/validators/test_target_knn_validator.py +++ b/tests/validators/test_target_knn_validator.py @@ -5,6 +5,7 @@ from pytorch_adapt.validators import TargetKNNValidator from .. import TEST_DEVICE +from .utils import get_knn_func def target_label_fn(x): @@ -26,9 +27,12 @@ def test_target_knn_validator(self): "target_train": {"features": target_features, "labels": target_labels}, } + knn_func = get_knn_func() for add_target_to_ref in [False, True]: validator = TargetKNNValidator( - add_target_to_ref=add_target_to_ref, target_label_fn=target_label_fn + add_target_to_ref=add_target_to_ref, + target_label_fn=target_label_fn, + knn_func=knn_func, ) score = validator(**data) diff --git a/tests/validators/utils.py b/tests/validators/utils.py new file mode 100644 index 00000000..7dd20478 --- /dev/null +++ b/tests/validators/utils.py @@ -0,0 +1,6 @@ +from pytorch_metric_learning.distances import LpDistance +from pytorch_metric_learning.utils.inference import CustomKNN + + +def get_knn_func(): + return CustomKNN(LpDistance(normalize_embeddings=False)) From 07e820438f968d28756ca1ae6d7d08d1350c1a44 Mon Sep 17 00:00:00 2001 From: KevinMusgrave Date: Mon, 30 May 2022 05:32:05 -0400 Subject: [PATCH 07/14] Some more fixes to the validator tests --- tests/validators/test_deep_embedded_validator.py | 2 +- tests/validators/test_knn_validator.py | 6 ++++-- tests/validators/test_per_class_validator.py | 6 ++---- tests/validators/test_snd_validator.py | 1 + tests/validators/utils.py | 5 +++++ 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tests/validators/test_deep_embedded_validator.py b/tests/validators/test_deep_embedded_validator.py index d62525cd..85db99bd 100644 --- a/tests/validators/test_deep_embedded_validator.py +++ b/tests/validators/test_deep_embedded_validator.py @@ -135,7 +135,7 @@ def test_deep_embedded_validator(self): correct_score = get_correct_score(self, src_train, target_train, src_val) correct_score = -correct_score self.assertTrue( - np.isclose(score, correct_score, rtol=0.2), + np.isclose(score, correct_score, rtol=0.5), msg=f"score={score}, correct_score={correct_score}", ) diff --git a/tests/validators/test_knn_validator.py b/tests/validators/test_knn_validator.py index 8f696928..68f64ac3 100644 --- a/tests/validators/test_knn_validator.py +++ b/tests/validators/test_knn_validator.py @@ -7,7 +7,7 @@ from pytorch_adapt.validators.knn_validator import BatchedAccuracyCalculator from .. import TEST_DEVICE -from .utils import get_knn_func +from .utils import get_knn_func, kmeans_func class TestKNNValidator(unittest.TestCase): @@ -15,7 +15,9 @@ def test_knn_validator(self): torch.cuda.empty_cache() knn_func = get_knn_func() knn_validator = ScoreHistory(KNNValidator(knn_func=knn_func)) - cluster_validator = ScoreHistory(KNNValidator(metric="AMI", knn_func=knn_func)) + cluster_validator = ScoreHistory( + KNNValidator(metric="AMI", knn_func=knn_func, kmeans_func=kmeans_func) + ) for epoch in [1, 2]: for validator in [knn_validator, cluster_validator]: dataset_size = 10000 diff --git a/tests/validators/test_per_class_validator.py b/tests/validators/test_per_class_validator.py index ff39802c..af1b22ef 100644 --- a/tests/validators/test_per_class_validator.py +++ b/tests/validators/test_per_class_validator.py @@ -58,12 +58,10 @@ def get_correct_score(data, validator): class TestPerClassValidator(unittest.TestCase): def test_per_class_validator(self): torch.manual_seed(204) + np.random.seed(204) dataset_size = 256 knn_func = get_knn_func() inner_validators = [ - KNNValidator( - key_map={"src_val": "src_train"}, metric="AMI", knn_func=knn_func - ), KNNValidator( key_map={"src_val": "src_train", "target_val": "target_train"}, knn_func=knn_func, @@ -84,7 +82,7 @@ def test_per_class_validator(self): if isinstance(v, MMDValidator): self.assertTrue(np.isclose(score, correct_score, rtol=0.05)) else: - self.assertTrue(score == correct_score) + self.assertEqual(score, correct_score) def test_common_labels(self): def label_fn(x): diff --git a/tests/validators/test_snd_validator.py b/tests/validators/test_snd_validator.py index 986cc438..5b22c529 100644 --- a/tests/validators/test_snd_validator.py +++ b/tests/validators/test_snd_validator.py @@ -81,6 +81,7 @@ def test_snd_validator_with_framework(self): adapter = wrapper_type( Classifier(models=models, optimizers=optimizers), validator=ScoreHistory(SNDValidator()), + device=TEST_DEVICE, ) score, _ = adapter.run( {"train": train_dataset, "target_train": target_train}, diff --git a/tests/validators/utils.py b/tests/validators/utils.py index 7dd20478..e3325561 100644 --- a/tests/validators/utils.py +++ b/tests/validators/utils.py @@ -1,6 +1,11 @@ from pytorch_metric_learning.distances import LpDistance from pytorch_metric_learning.utils.inference import CustomKNN +from sklearn.cluster import KMeans def get_knn_func(): return CustomKNN(LpDistance(normalize_embeddings=False)) + + +def kmeans_func(query, num_clusters): + return KMeans(n_clusters=num_clusters).fit_predict(query.cpu().numpy()) From 64fba681f9dcb3c0385af9a21b21c28d5e6b1e1b Mon Sep 17 00:00:00 2001 From: KevinMusgrave Date: Mon, 30 May 2022 05:49:39 -0400 Subject: [PATCH 08/14] Pass kwargs down to load_state_dict_from_url --- src/pytorch_adapt/__init__.py | 2 +- src/pytorch_adapt/models/pretrained.py | 54 +++++++++++++++++------- src/pytorch_adapt/models/utils.py | 6 +-- tests/models/test_pretrained_download.py | 8 ++-- 4 files changed, 47 insertions(+), 23 deletions(-) diff --git a/src/pytorch_adapt/__init__.py b/src/pytorch_adapt/__init__.py index 8a352f99..e1a95f5f 100644 --- a/src/pytorch_adapt/__init__.py +++ b/src/pytorch_adapt/__init__.py @@ -1 +1 @@ -__version__ = "0.0.73" +__version__ = "0.0.74" diff --git a/src/pytorch_adapt/models/pretrained.py b/src/pytorch_adapt/models/pretrained.py index b162d0a0..7c60ac97 100644 --- a/src/pytorch_adapt/models/pretrained.py +++ b/src/pytorch_adapt/models/pretrained.py @@ -3,7 +3,7 @@ from .utils import download_weights -def mnistG(pretrained=False, progress=True): +def mnistG(pretrained=False, progress=True, **kwargs): """ Returns: An [```MNISTFeatures```][pytorch_adapt.models.MNISTFeatures] model @@ -12,11 +12,15 @@ def mnistG(pretrained=False, progress=True): model = MNISTFeatures() url = "https://cornell.box.com/shared/static/tdx0ts24e273j7mf3r2ox7a12xh4fdfy" h = "68ee79452f1d5301be2329dfa542ac6fa18de99e09d6540838606d9d700b09c8" - filename = f"mnistG-{h[:8]}.pth" - return download_weights(model, url, pretrained, progress, filename) + file_name = f"mnistG-{h[:8]}.pth" + return download_weights( + model, url, pretrained, progress=progress, file_name=file_name, **kwargs + ) -def mnistC(num_classes=10, in_size=1200, h=256, pretrained=False, progress=True): +def mnistC( + num_classes=10, in_size=1200, h=256, pretrained=False, progress=True, **kwargs +): """ Returns: A [```Classifier```][pytorch_adapt.models.Classifier] model @@ -25,11 +29,13 @@ def mnistC(num_classes=10, in_size=1200, h=256, pretrained=False, progress=True) model = Classifier(num_classes=num_classes, in_size=in_size, h=h) url = "https://cornell.box.com/shared/static/j4zrogronmievq1csulrkai7zjm27gcq" h = "ac7b5a13df2ef3522b6550a147eb44dde8ff4fead3ddedc540d9fe63c9d597c1" - filename = f"mnistC-{h[:8]}.pth" - return download_weights(model, url, pretrained, progress, filename) + file_name = f"mnistC-{h[:8]}.pth" + return download_weights( + model, url, pretrained, progress=progress, file_name=file_name, **kwargs + ) -def resnet50(pretrained=False, progress=True): +def resnet50(pretrained=False, progress=True, **kwargs): import timm model = timm.create_model("resnet50", pretrained=False, num_classes=0) @@ -38,8 +44,10 @@ def resnet50(pretrained=False, progress=True): # So the model for all 3 domains is the same url = "https://cornell.box.com/shared/static/1oxb5xk5dq3od1d3gprigznxmqb3wgr1" h = "a567ecd6ea5addf29ccfa8bf706be78a35adff7cd5cf5a3a99d89b19807454ae" - filename = f"resnet50MusgraveUDA-{h[:8]}.pth" - return download_weights(model, url, pretrained, progress, filename) + file_name = f"resnet50MusgraveUDA-{h[:8]}.pth" + return download_weights( + model, url, pretrained, progress=progress, file_name=file_name, **kwargs + ) def office31G(*args, **kwargs): @@ -51,7 +59,13 @@ def office31G(*args, **kwargs): def office31C( - domain=None, num_classes=31, in_size=2048, h=256, pretrained=False, progress=True + domain=None, + num_classes=31, + in_size=2048, + h=256, + pretrained=False, + progress=True, + **kwargs, ): """ Returns: @@ -79,8 +93,10 @@ def office31C( "dslr": "fc0acd7a71eb5f12d4af619e5c63bcc42e5a23441bbd105fe0f7a37c26f37d80", "webcam": "b2bb55978380fa9ca6452cba30e0ac2a19b7166d8348bcc1554fdabd185e4cdd", }[domain] - filename = f"office31C{domain}-{h[:8]}.pth" - return download_weights(model, url, pretrained, progress, filename) + file_name = f"office31C{domain}-{h[:8]}.pth" + return download_weights( + model, url, pretrained, progress=progress, file_name=file_name, **kwargs + ) def officehomeG(*args, **kwargs): @@ -92,7 +108,13 @@ def officehomeG(*args, **kwargs): def officehomeC( - domain=None, num_classes=65, in_size=2048, h=256, pretrained=False, progress=True + domain=None, + num_classes=65, + in_size=2048, + h=256, + pretrained=False, + progress=True, + **kwargs, ): """ Returns: @@ -122,5 +144,7 @@ def officehomeC( "product": "472ff36fdf13ec6c1fa1236d1d0800e2a5cf2e3d366b6b63ff5807dff6a761d8", "real": "f0c8d6e941d4f488ff2438eb5cccdc59e78f35961e48f03d2186752e5878c697", }[domain] - filename = f"officehomeC{domain}-{h[:8]}.pth" - return download_weights(model, url, pretrained, progress, filename) + file_name = f"officehomeC{domain}-{h[:8]}.pth" + return download_weights( + model, url, pretrained, progress=progress, file_name=file_name, **kwargs + ) diff --git a/src/pytorch_adapt/models/utils.py b/src/pytorch_adapt/models/utils.py index e5cac9d0..2eef0af5 100644 --- a/src/pytorch_adapt/models/utils.py +++ b/src/pytorch_adapt/models/utils.py @@ -1,10 +1,8 @@ from torch.hub import load_state_dict_from_url -def download_weights(model, url, pretrained, progress, file_name): +def download_weights(model, url, pretrained, **kwargs): if pretrained: - state_dict = load_state_dict_from_url( - url, progress=progress, check_hash=True, file_name=file_name - ) + state_dict = load_state_dict_from_url(url, check_hash=True, **kwargs) model.load_state_dict(state_dict) return model diff --git a/tests/models/test_pretrained_download.py b/tests/models/test_pretrained_download.py index cdd17d5b..c6f26187 100644 --- a/tests/models/test_pretrained_download.py +++ b/tests/models/test_pretrained_download.py @@ -9,17 +9,19 @@ officehomeG, ) +from .. import TEST_DEVICE + class TestPretrainedDownload(unittest.TestCase): def test_pretrained_download(self): for x in [mnistC, mnistG, office31G, officehomeG]: - x(pretrained=True) + x(pretrained=True, map_location=TEST_DEVICE) for domain in ["amazon", "dslr", "webcam"]: - office31C(domain=domain, pretrained=True) + office31C(domain=domain, pretrained=True, map_location=TEST_DEVICE) for domain in ["art", "clipart", "product", "real"]: - officehomeC(domain=domain, pretrained=True) + officehomeC(domain=domain, pretrained=True, map_location=TEST_DEVICE) def test_not_pretrained(self): office31C(pretrained=False) From c6a462d1781709c8f82b8df7bc7e562cf319a388 Mon Sep 17 00:00:00 2001 From: KevinMusgrave Date: Mon, 30 May 2022 05:54:20 -0400 Subject: [PATCH 09/14] Add timm optional dependency --- .github/workflows/base_test_workflow.yml | 2 +- setup.py | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/base_test_workflow.yml b/.github/workflows/base_test_workflow.yml index 7413cc6b..26eaba35 100644 --- a/.github/workflows/base_test_workflow.yml +++ b/.github/workflows/base_test_workflow.yml @@ -26,7 +26,7 @@ jobs: python-version: ${{ matrix.python-version }} - name: Install dependencies run: | - pip install .[ignite,lightning,record-keeper] + pip install .[ignite,lightning,record-keeper,timm] - name: Run unit tests run: | TEST_DTYPES=float32,float64 TEST_DEVICE=cpu python -m unittest discover -t . -s tests/${{ inputs.module-to-test }} diff --git a/setup.py b/setup.py index 7cb429ff..cfda8064 100644 --- a/setup.py +++ b/setup.py @@ -12,6 +12,7 @@ extras_require_ignite = ["pytorch-ignite == 0.5.0.dev20220221"] extras_require_lightning = ["pytorch-lightning"] extras_require_record_keeper = ["record-keeper >= 0.9.31"] +extras_require_timm = ["timm"] setuptools.setup( @@ -41,5 +42,6 @@ "ignite": extras_require_ignite, "lightning": extras_require_lightning, "record-keeper": extras_require_record_keeper, + "timm": extras_require_timm, }, ) From eb0d1540856dcd798bc31329812f78b6886cebaf Mon Sep 17 00:00:00 2001 From: KevinMusgrave Date: Mon, 30 May 2022 06:35:07 -0400 Subject: [PATCH 10/14] Update base_test_workflow --- .github/workflows/base_test_workflow.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/base_test_workflow.yml b/.github/workflows/base_test_workflow.yml index 26eaba35..6a1564d1 100644 --- a/.github/workflows/base_test_workflow.yml +++ b/.github/workflows/base_test_workflow.yml @@ -1,6 +1,3 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python -# For more information see: https://help.github.com/actions/language-and-framework-guides/using-python-with-github-actions - name: base_test_workflow on: @@ -27,6 +24,7 @@ jobs: - name: Install dependencies run: | pip install .[ignite,lightning,record-keeper,timm] + pip install --upgrade protobuf<=3.20.1 - name: Run unit tests run: | TEST_DTYPES=float32,float64 TEST_DEVICE=cpu python -m unittest discover -t . -s tests/${{ inputs.module-to-test }} From 356bc8f4460362e30401e366b7d224f677039e39 Mon Sep 17 00:00:00 2001 From: KevinMusgrave Date: Mon, 30 May 2022 06:39:22 -0400 Subject: [PATCH 11/14] Use strict equality --- .github/workflows/base_test_workflow.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/base_test_workflow.yml b/.github/workflows/base_test_workflow.yml index 6a1564d1..7d43333c 100644 --- a/.github/workflows/base_test_workflow.yml +++ b/.github/workflows/base_test_workflow.yml @@ -24,7 +24,7 @@ jobs: - name: Install dependencies run: | pip install .[ignite,lightning,record-keeper,timm] - pip install --upgrade protobuf<=3.20.1 + pip install --upgrade protobuf==3.20.1 - name: Run unit tests run: | TEST_DTYPES=float32,float64 TEST_DEVICE=cpu python -m unittest discover -t . -s tests/${{ inputs.module-to-test }} From e54983a7b36e1d7342748b65a8cfdddef9f4aed9 Mon Sep 17 00:00:00 2001 From: KevinMusgrave Date: Mon, 30 May 2022 07:14:47 -0400 Subject: [PATCH 12/14] Set device correctly in frameworks tests --- tests/frameworks/test_ignite.py | 7 +++---- tests/frameworks/test_lightning.py | 8 +++++++- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/tests/frameworks/test_ignite.py b/tests/frameworks/test_ignite.py index 87109dee..c70faa83 100644 --- a/tests/frameworks/test_ignite.py +++ b/tests/frameworks/test_ignite.py @@ -21,7 +21,7 @@ from pytorch_adapt.utils import common_functions as c_f from pytorch_adapt.validators import AccuracyValidator, EntropyValidator, ScoreHistory -from .. import TEST_FOLDER +from .. import TEST_DEVICE, TEST_FOLDER class NanModel(nn.Module): @@ -57,7 +57,6 @@ def helper( with_checkpoint_fn=False, use_nan_model=False, ): - device = torch.device("cuda") datasets = {} for k in ["src_train", "target_train"]: datasets[k] = EmbeddingDataset( @@ -72,7 +71,7 @@ def helper( datasets["src_train"], datasets["target_train"] ) - C = nn.Linear(32, 16, device=device) + C = nn.Linear(32, 16, device=TEST_DEVICE) if use_nan_model: C = NanModel(C) models = Models({"G": nn.Identity(), "C": C}) @@ -103,7 +102,7 @@ def helper( logger=logger, log_freq=1, with_pbars=False, - device=device, + device=TEST_DEVICE, ) return adapter, datasets diff --git a/tests/frameworks/test_lightning.py b/tests/frameworks/test_lightning.py index c848f098..da576834 100644 --- a/tests/frameworks/test_lightning.py +++ b/tests/frameworks/test_lightning.py @@ -2,6 +2,7 @@ import unittest import pytorch_lightning as pl +import torch from pytorch_adapt.adapters import DANN from pytorch_adapt.datasets import DataloaderCreator @@ -9,6 +10,7 @@ from pytorch_adapt.frameworks.utils import filter_datasets from pytorch_adapt.validators import IMValidator +from .. import TEST_DEVICE from ..adapters.utils import get_datasets, get_gcd @@ -19,7 +21,11 @@ def test_lightning(self): adapter = DANN(models) validator = IMValidator() adapter = Lightning(adapter, validator=validator) - trainer = pl.Trainer(gpus=1, max_epochs=2, log_every_n_steps=1) + + accelerator = "cpu" if TEST_DEVICE == torch.device("cpu") else "gpu" + trainer = pl.Trainer( + devices=1, accelerator=accelerator, max_epochs=2, log_every_n_steps=1 + ) datasets = filter_datasets(datasets, validator) dataloaders = DataloaderCreator(num_workers=2)(**datasets) trainer.fit(adapter, dataloaders["train"], dataloaders["target_train"]) From 2b31723c126d794940b41b901be71ffd97e5cc3b Mon Sep 17 00:00:00 2001 From: KevinMusgrave Date: Mon, 30 May 2022 07:24:24 -0400 Subject: [PATCH 13/14] Use seed in test_combined_source_and_target --- tests/datasets/test_combined_source_and_target.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/datasets/test_combined_source_and_target.py b/tests/datasets/test_combined_source_and_target.py index a032e455..82b925be 100644 --- a/tests/datasets/test_combined_source_and_target.py +++ b/tests/datasets/test_combined_source_and_target.py @@ -9,7 +9,7 @@ class TestCombinedSourceAndTarget(unittest.TestCase): def test_combined(self): - + np.random.seed(3429) for target_dataset_size in [99, 199]: src_dataset_size = 117 @@ -45,5 +45,5 @@ def test_combined(self): self.assertTrue(len(bincount) == src_dataset_size) ideal_bincount = total_len // src_dataset_size self.assertTrue( - all(np.isclose(x, ideal_bincount, rtol=5e-2) for x in bincount) + all(np.isclose(x, ideal_bincount, rtol=0.1) for x in bincount) ) From 1e80cf3a9923eae75345cc76b186b86cdcff64d5 Mon Sep 17 00:00:00 2001 From: KevinMusgrave Date: Mon, 30 May 2022 08:15:30 -0400 Subject: [PATCH 14/14] Move model to TEST_DEVICE in test_snd_validator --- tests/validators/test_snd_validator.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/validators/test_snd_validator.py b/tests/validators/test_snd_validator.py index 5b22c529..e2a44e93 100644 --- a/tests/validators/test_snd_validator.py +++ b/tests/validators/test_snd_validator.py @@ -75,7 +75,9 @@ def test_snd_validator_with_framework(self): ) target_train = TargetDataset(train_datasets[2]) - C = torch.nn.Sequential(torch.nn.Linear(128, 10), torch.nn.Softmax(dim=1)) + C = torch.nn.Sequential( + torch.nn.Linear(128, 10), torch.nn.Softmax(dim=1) + ).to(TEST_DEVICE) models = Models({"G": C, "C": torch.nn.Identity()}) optimizers = Optimizers((torch.optim.Adam, {"lr": 0})) adapter = wrapper_type(