From 8929b8e49fe6f085b19601b5520b7cf3e58c2bc0 Mon Sep 17 00:00:00 2001 From: Martin Schrimpf Date: Fri, 12 Jul 2019 17:29:17 -0400 Subject: [PATCH] use pytest.mark instead of pytest.config https://docs.pytest.org/en/latest/deprecations.html#pytest-config-global --- .travis.yml | 4 ++-- tests/README.md | 16 ++++++++++++++ tests/activations/test___init__.py | 23 ++++++++++----------- tests/brain_transformation/test_behavior.py | 3 +-- tests/conftest.py | 6 ------ tests/flags.py | 8 ------- 6 files changed, 30 insertions(+), 30 deletions(-) create mode 100644 tests/README.md delete mode 100644 tests/conftest.py delete mode 100644 tests/flags.py diff --git a/.travis.yml b/.travis.yml index c773531..101ff3a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -36,5 +36,5 @@ before_script: - aws --no-sign-request --region=us-east-1 s3 cp s3://brain-score-models/imagenet2012-val.hdf5 ./imagenet2012.hdf5 - conda list # list installed package versions script: -- if [ "$TRAVIS_PULL_REQUEST" = "false"]; then CUDA_VISIBLE_DEVICES= MT_MULTITHREAD=0 MT_IMAGENET_PATH=./imagenet2012.hdf5 pytest --ignore=tf-models --skip-memory-intense; fi -- if [ "$TRAVIS_PULL_REQUEST" = "true" ]; then CUDA_VISIBLE_DEVICES= MT_MULTITHREAD=0 MT_IMAGENET_PATH=./imagenet2012.hdf5 pytest --ignore=tf-models --skip-memory-intense --skip-private; fi +- if [ "$TRAVIS_PULL_REQUEST" = "false"]; then CUDA_VISIBLE_DEVICES= MT_MULTITHREAD=0 MT_IMAGENET_PATH=./imagenet2012.hdf5 pytest --ignore=tf-models -m "not memory_intense"; fi +- if [ "$TRAVIS_PULL_REQUEST" = "true" ]; then CUDA_VISIBLE_DEVICES= MT_MULTITHREAD=0 MT_IMAGENET_PATH=./imagenet2012.hdf5 pytest --ignore=tf-models -m "not memory_intense and not private_access"; fi diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000..e0520dc --- /dev/null +++ b/tests/README.md @@ -0,0 +1,16 @@ +# Unit Tests +## Markers +Unit tests have various markers that denote possible issues in the travis build: + +* **private_access**: tests that require access to a private ressource, such as assemblies on S3 (travis pull request builds can not have private access) +* **memory_intense**: tests requiring more memory than is available in the travis sandbox (currently 3 GB, https://docs.travis-ci.com/user/common-build-problems/#my-build-script-is-killed-without-any-error) + +Use the following syntax to mark a test: +``` +@pytest.mark.memory_intense +def test_something(...): + assert False +``` + +To skip a specific marker, run e.g. `pytest -m "not memory_intense"`. +To skip multiple markers, run e.g. `pytest -m "not private_access and not memory_intense"`. diff --git a/tests/activations/test___init__.py b/tests/activations/test___init__.py index 282ac9d..33dca69 100644 --- a/tests/activations/test___init__.py +++ b/tests/activations/test___init__.py @@ -9,7 +9,6 @@ from model_tools.activations import KerasWrapper, PytorchWrapper, TensorflowSlimWrapper from model_tools.activations.core import flatten from model_tools.activations.pca import LayerPCA -from tests.flags import memory_intense def unique_preserved_order(a): @@ -131,10 +130,10 @@ def tfslim_vgg16(): @pytest.mark.parametrize(["pca_components", "logits"], [(None, True), (None, False), (5, False)]) @pytest.mark.parametrize(["model_ctr", "layers"], [ pytest.param(pytorch_custom, ['linear', 'relu2']), - pytest.param(pytorch_alexnet, ['features.12', 'classifier.5'], marks=memory_intense), - pytest.param(keras_vgg19, ['block3_pool'], marks=memory_intense), - pytest.param(tfslim_custom, ['my_model/pool2'], marks=memory_intense), - pytest.param(tfslim_vgg16, ['vgg_16/pool5'], marks=memory_intense), + pytest.param(pytorch_alexnet, ['features.12', 'classifier.5'], marks=pytest.mark.memory_intense), + pytest.param(keras_vgg19, ['block3_pool'], marks=pytest.mark.memory_intense), + pytest.param(tfslim_custom, ['my_model/pool2'], marks=pytest.mark.memory_intense), + pytest.param(tfslim_vgg16, ['vgg_16/pool5'], marks=pytest.mark.memory_intense), ]) def test_from_image_path(model_ctr, layers, image_name, pca_components, logits): stimuli_paths = [os.path.join(os.path.dirname(__file__), image_name)] @@ -160,10 +159,10 @@ def test_from_image_path(model_ctr, layers, image_name, pca_components, logits): @pytest.mark.parametrize("pca_components", [None, 5]) @pytest.mark.parametrize(["model_ctr", "layers"], [ pytest.param(pytorch_custom, ['linear', 'relu2']), - pytest.param(pytorch_alexnet, ['features.12', 'classifier.5'], marks=memory_intense), - pytest.param(keras_vgg19, ['block3_pool'], marks=memory_intense), - pytest.param(tfslim_custom, ['my_model/pool2'], marks=memory_intense), - pytest.param(tfslim_vgg16, ['vgg_16/pool5'], marks=memory_intense), + pytest.param(pytorch_alexnet, ['features.12', 'classifier.5'], marks=pytest.mark.memory_intense), + pytest.param(keras_vgg19, ['block3_pool'], marks=pytest.mark.memory_intense), + pytest.param(tfslim_custom, ['my_model/pool2'], marks=pytest.mark.memory_intense), + pytest.param(tfslim_vgg16, ['vgg_16/pool5'], marks=pytest.mark.memory_intense), ]) def test_from_stimulus_set(model_ctr, layers, pca_components): image_names = ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png'] @@ -185,7 +184,7 @@ def test_from_stimulus_set(model_ctr, layers, pca_components): assert len(activations['neuroid']) == pca_components * len(layers) -@memory_intense +@pytest.mark.memory_intense @pytest.mark.parametrize("pca_components", [None, 1000]) def test_exact_activations(pca_components): activations = test_from_image_path(model_ctr=pytorch_alexnet_resize, layers=['features.12', 'classifier.5'], @@ -195,7 +194,7 @@ def test_exact_activations(pca_components): assert (activations == target).all() -@memory_intense +@pytest.mark.memory_intense @pytest.mark.parametrize(["model_ctr", "internal_layers"], [ (pytorch_alexnet, ['features.12', 'classifier.5']), (keras_vgg19, ['block3_pool']), @@ -212,7 +211,7 @@ def test_mixed_layer_logits(model_ctr, internal_layers): assert unique_preserved_order(activations['layer'])[-1] == 'logits' -@memory_intense +@pytest.mark.memory_intense @pytest.mark.parametrize(["model_ctr", "expected_identifier"], [ (pytorch_custom, 'MyModel'), (pytorch_alexnet, 'AlexNet'), diff --git a/tests/brain_transformation/test_behavior.py b/tests/brain_transformation/test_behavior.py index e807fb2..174f40d 100644 --- a/tests/brain_transformation/test_behavior.py +++ b/tests/brain_transformation/test_behavior.py @@ -11,7 +11,6 @@ from brainscore.model_interface import BrainModel from model_tools.activations import PytorchWrapper from model_tools.brain_transformation import ModelCommitment, ProbabilitiesMapping -from tests.flags import private_access def pytorch_custom(): @@ -75,7 +74,7 @@ def test_creates_probabilities(self): probabilities.sel(image_id='rgb1', choice='label2') == approx(1) -@private_access +@pytest.mark.private_access class TestI2N: @pytest.mark.parametrize(['model', 'expected_score'], [ diff --git a/tests/conftest.py b/tests/conftest.py deleted file mode 100644 index dd12ef1..0000000 --- a/tests/conftest.py +++ /dev/null @@ -1,6 +0,0 @@ -def pytest_addoption(parser): - """Add command-line flags for pytest.""" - parser.addoption("--skip-private", action="store_true", - help="do not run tests requiring private access") - parser.addoption("--skip-memory-intense", action="store_true", - help="do not run memory intense tests") diff --git a/tests/flags.py b/tests/flags.py deleted file mode 100644 index 32bdb6a..0000000 --- a/tests/flags.py +++ /dev/null @@ -1,8 +0,0 @@ -import pytest - -private_access = pytest.mark.skipif( - pytest.config.getoption("--skip-private"), - reason="set to not run tests that require private s3 access") -memory_intense = pytest.mark.skipif( - pytest.config.getoption("--skip-memory-intense"), - reason="set to not run tests requiring a lot of memory (more than what travis offers)")