Merge pull request #7 from brain-score/pytest-mark

use pytest.mark instead of pytest.config

.travis.yml

@@ -36,5 +36,5 @@ before_script:
 - aws --no-sign-request --region=us-east-1 s3 cp s3://brain-score-models/imagenet2012-val.hdf5 ./imagenet2012.hdf5
 - conda list # list installed package versions
 script:
-- if [ "$TRAVIS_PULL_REQUEST" = "false"];  then CUDA_VISIBLE_DEVICES= MT_MULTITHREAD=0 MT_IMAGENET_PATH=./imagenet2012.hdf5 pytest --ignore=tf-models --skip-memory-intense; fi
-- if [ "$TRAVIS_PULL_REQUEST" = "true" ];  then CUDA_VISIBLE_DEVICES= MT_MULTITHREAD=0 MT_IMAGENET_PATH=./imagenet2012.hdf5 pytest --ignore=tf-models --skip-memory-intense --skip-private; fi
+- if [ "$TRAVIS_PULL_REQUEST" = "false"];  then CUDA_VISIBLE_DEVICES= MT_MULTITHREAD=0 MT_IMAGENET_PATH=./imagenet2012.hdf5 pytest --ignore=tf-models -m "not memory_intense"; fi
+- if [ "$TRAVIS_PULL_REQUEST" = "true" ];  then CUDA_VISIBLE_DEVICES= MT_MULTITHREAD=0 MT_IMAGENET_PATH=./imagenet2012.hdf5 pytest --ignore=tf-models -m "not memory_intense and not private_access"; fi

tests/README.md

@@ -0,0 +1,16 @@
+# Unit Tests
+## Markers
+Unit tests have various markers that denote possible issues in the travis build:
+
+* **private_access**: tests that require access to a private ressource, such as assemblies on S3 (travis pull request builds can not have private access)
+* **memory_intense**: tests requiring more memory than is available in the travis sandbox (currently 3 GB, https://docs.travis-ci.com/user/common-build-problems/#my-build-script-is-killed-without-any-error)
+
+Use the following syntax to mark a test:
+```
+@pytest.mark.memory_intense
+def test_something(...):
+    assert False
+```
+
+To skip a specific marker, run e.g. `pytest -m "not memory_intense"`.
+To skip multiple markers, run e.g. `pytest -m "not private_access and not memory_intense"`.

tests/activations/test___init__.py

@@ -9,7 +9,6 @@
 from model_tools.activations import KerasWrapper, PytorchWrapper, TensorflowSlimWrapper
 from model_tools.activations.core import flatten
 from model_tools.activations.pca import LayerPCA
-from tests.flags import memory_intense
 
 
 def unique_preserved_order(a):
@@ -131,10 +130,10 @@ def tfslim_vgg16():
 @pytest.mark.parametrize(["pca_components", "logits"], [(None, True), (None, False), (5, False)])
 @pytest.mark.parametrize(["model_ctr", "layers"], [
     pytest.param(pytorch_custom, ['linear', 'relu2']),
-    pytest.param(pytorch_alexnet, ['features.12', 'classifier.5'], marks=memory_intense),
-    pytest.param(keras_vgg19, ['block3_pool'], marks=memory_intense),
-    pytest.param(tfslim_custom, ['my_model/pool2'], marks=memory_intense),
-    pytest.param(tfslim_vgg16, ['vgg_16/pool5'], marks=memory_intense),
+    pytest.param(pytorch_alexnet, ['features.12', 'classifier.5'], marks=pytest.mark.memory_intense),
+    pytest.param(keras_vgg19, ['block3_pool'], marks=pytest.mark.memory_intense),
+    pytest.param(tfslim_custom, ['my_model/pool2'], marks=pytest.mark.memory_intense),
+    pytest.param(tfslim_vgg16, ['vgg_16/pool5'], marks=pytest.mark.memory_intense),
 ])
 def test_from_image_path(model_ctr, layers, image_name, pca_components, logits):
     stimuli_paths = [os.path.join(os.path.dirname(__file__), image_name)]
@@ -160,10 +159,10 @@ def test_from_image_path(model_ctr, layers, image_name, pca_components, logits):
 @pytest.mark.parametrize("pca_components", [None, 5])
 @pytest.mark.parametrize(["model_ctr", "layers"], [
     pytest.param(pytorch_custom, ['linear', 'relu2']),
-    pytest.param(pytorch_alexnet, ['features.12', 'classifier.5'], marks=memory_intense),
-    pytest.param(keras_vgg19, ['block3_pool'], marks=memory_intense),
-    pytest.param(tfslim_custom, ['my_model/pool2'], marks=memory_intense),
-    pytest.param(tfslim_vgg16, ['vgg_16/pool5'], marks=memory_intense),
+    pytest.param(pytorch_alexnet, ['features.12', 'classifier.5'], marks=pytest.mark.memory_intense),
+    pytest.param(keras_vgg19, ['block3_pool'], marks=pytest.mark.memory_intense),
+    pytest.param(tfslim_custom, ['my_model/pool2'], marks=pytest.mark.memory_intense),
+    pytest.param(tfslim_vgg16, ['vgg_16/pool5'], marks=pytest.mark.memory_intense),
 ])
 def test_from_stimulus_set(model_ctr, layers, pca_components):
     image_names = ['rgb.jpg', 'grayscale.png', 'grayscale2.jpg', 'grayscale_alpha.png']
@@ -185,7 +184,7 @@ def test_from_stimulus_set(model_ctr, layers, pca_components):
         assert len(activations['neuroid']) == pca_components * len(layers)
 
 
-@memory_intense
+@pytest.mark.memory_intense
 @pytest.mark.parametrize("pca_components", [None, 1000])
 def test_exact_activations(pca_components):
     activations = test_from_image_path(model_ctr=pytorch_alexnet_resize, layers=['features.12', 'classifier.5'],
@@ -195,7 +194,7 @@ def test_exact_activations(pca_components):
     assert (activations == target).all()
 
 
-@memory_intense
+@pytest.mark.memory_intense
 @pytest.mark.parametrize(["model_ctr", "internal_layers"], [
     (pytorch_alexnet, ['features.12', 'classifier.5']),
     (keras_vgg19, ['block3_pool']),
@@ -212,7 +211,7 @@ def test_mixed_layer_logits(model_ctr, internal_layers):
     assert unique_preserved_order(activations['layer'])[-1] == 'logits'
 
 
-@memory_intense
+@pytest.mark.memory_intense
 @pytest.mark.parametrize(["model_ctr", "expected_identifier"], [
     (pytorch_custom, 'MyModel'),
     (pytorch_alexnet, 'AlexNet'),

tests/brain_transformation/test_behavior.py

@@ -11,7 +11,6 @@
 from brainscore.model_interface import BrainModel
 from model_tools.activations import PytorchWrapper
 from model_tools.brain_transformation import ModelCommitment, ProbabilitiesMapping
-from tests.flags import private_access
 
 
 def pytorch_custom():
@@ -75,7 +74,7 @@ def test_creates_probabilities(self):
                probabilities.sel(image_id='rgb1', choice='label2') == approx(1)
 
 
-@private_access
+@pytest.mark.private_access
 class TestI2N:
     @pytest.mark.parametrize(['model', 'expected_score'],
                              [