Improve the "expected failures" experience (#178)

If we add more tests that we consider more or less optional features,
we need a way for clients to manage them.

xfails seems like it should work for that: Make it possible to
write the expected failures to a file next to the client-under-test
executable. This should make managing a longer xfail list reasonable.

This also means we don't need to repeat the xfails for go-tuf in two
places.

Signed-off-by: Jussi Kukkonen <[email protected]>

.github/workflows/ci.yml

@@ -62,4 +62,3 @@ jobs:
         uses: ./
         with:
           entrypoint: "clients/go-tuf/go-tuf"
-          expected-failures: "test_keytype_and_scheme[rsa/rsassa-pss-sha256]"

Makefile

@@ -62,7 +62,6 @@ PHONY: test-go-tuf
 test-go-tuf: dev build-go-tuf faketime
 	./env/bin/pytest tuf_conformance \
 		--entrypoint "./clients/go-tuf/go-tuf" \
-		--expected-failures "test_keytype_and_scheme[rsa/rsassa-pss-sha256]" \
 		--repository-dump-dir $(DUMP_DIR)
 	@echo Repository dump in $(DUMP_DIR)
 PHONY: build-go-tuf

README.md

@@ -32,6 +32,14 @@ There are two required steps:
               entrypoint: path/to/my/test/executable
     ```
 
+### Expected failures
+
+The test suite also contains tests that are not strictly speaking specification requirements (such as
+tests for specific keytype or hash algorithm suport). Clients can mark tests as "expected failures"
+if they do not intend to support this specific feature.
+
+The tests that are expected to fail can be listed in `<entrypoint>.xfails` file. In the previous
+workflow example the xfails file would be `path/to/my/test/executable.xfails`
 
 ## Development
 

action.yml

@@ -10,10 +10,6 @@ inputs:
       but if you call this action in a job matrix, make sure each call gets a unique name"
     default: ""
     required: false
-  expected-failures:
-    description: "Optional list test names expected to fail"
-    default: ""
-    required: false
 
 runs:
   using: "composite"
@@ -36,7 +32,6 @@ runs:
       id: tuf-conformance
       env:
         ENTRYPOINT: ${{ inputs.entrypoint }}
-        EXPECTED_FAILURES: ${{ inputs.expected-failures }}
         TEST_LOCATION: ${{ github.action_path }}/tuf_conformance
         NAME: ${{ inputs.artifact-name }}
       run: |
@@ -50,7 +45,6 @@ runs:
         # run test suite
         pytest -v "$TEST_LOCATION" \
           --entrypoint "$ENTRYPOINT" \
-          --expected-failures "$EXPECTED_FAILURES" \
           --repository-dump-dir ./test-repositories \
       shell: bash
 

clients/go-tuf/go-tuf.xfails

@@ -0,0 +1 @@
+test_keytype_and_scheme[rsa/rsassa-pss-sha256]

tuf_conformance/conftest.py

@@ -16,13 +16,6 @@ def pytest_addoption(parser: pytest.Parser) -> None:
         required=True,
         type=str,
     )
-    parser.addoption(
-        "--expected-failures",
-        action="store",
-        help="Optional space delimited list of test names expected to fail",
-        required=False,
-        type=str,
-    )
     parser.addoption(
         "--repository-dump-dir",
         action="store",
@@ -90,14 +83,25 @@ def static_client(
     return ClientRunner(entrypoint, static_server, request.node.name)
 
 
+@cache
+def read_xfails(pytestconfig: pytest.Config) -> list[str]:
+    # Find expected failures from .xfails file
+    xfail_file = f"{pytestconfig.getoption('--entrypoint')}.xfails"
+    if not os.path.isabs(xfail_file):
+        xfail_file = os.path.join(pytestconfig.invocation_params.dir, xfail_file)
+
+    try:
+        with open(xfail_file) as f:
+            return f.read().splitlines()
+    except FileNotFoundError:
+        return []
+
+
 @pytest.fixture(autouse=True)
 def conformance_xfail(
     pytestconfig: pytest.Config, request: pytest.FixtureRequest
 ) -> None:
-    xfail_option = pytestconfig.getoption("--expected-failures")
-    if xfail_option is None:
-        return
+    xfails = read_xfails(pytestconfig)
 
-    xfails = xfail_option.split(" ")
     if request.node.originalname in xfails or request.node.name in xfails:
         request.node.add_marker(pytest.mark.xfail(strict=True))

tuf_conformance/test_file_download.py

@@ -187,8 +187,8 @@ def test_download_with_hash_algorithms(
     """Test support of hash algorithms. The specification does not require any
     specific algorithms, and only mentions sha256.
 
-    Clients can use --expected-failures to mark tests that fail because of algorithms
-    they do not intend to support."""
+    Clients can use the .xfail file to mark tests as "expected to fail".
+    """
     init_data, repo = server.new_test(client.test_name)
 
     # Create a legitimate test artifact