Implement e2e result checking for test case 1 (#83)

* Implement e2e result checking for test case 1

.gitignore

@@ -10,6 +10,7 @@
 __pycache__/*
 .cache/*
 .*.swp
+.swp
 */.ipynb_checkpoints/*
 .DS_Store
 /tmp

.travis.yml

@@ -26,7 +26,6 @@ jobs:
        - "3.6"
       script:
       - pip3 freeze
-      - python setup.py test
       - make lint
 
     - stage: "Test"
@@ -37,7 +36,6 @@ jobs:
        - "3.7"
       script:
       - pip3 freeze
-      - python setup.py test
       - make lint
 
     - stage: "Test"
@@ -77,7 +75,7 @@ jobs:
       - "3.6"
       script:
       - pip3 freeze
-      - python setup.py test --addopts --runslow
+      - python setup.py test --addopts="-m 'not e2e' --runslow"
 
     - stage: "Test"
       name: "test:unittest:3.7"
@@ -88,7 +86,7 @@ jobs:
       - pip install codecov
       script:
       - pip3 freeze
-      - python setup.py test --addopts --runslow
+      - python setup.py test --addopts="-m 'not e2e' --runslow"
       after_script:
       - codecov
 

e2e-manifests/01-deny-all-traffic-to-an-application.yml

@@ -1,18 +1,25 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: 01-deny-all-traffic-to-an-application
+  labels:
+    illuminatio-e2e: 01-deny-all-traffic-to-an-application
+---
 apiVersion: apps/v1
 kind: Deployment
 metadata:
   labels:
     app: web
+    illuminatio-e2e: 01-deny-all-traffic-to-an-application
   name: web
-  namespace: 01-deny-all
+  namespace: 01-deny-all-traffic-to-an-application
 spec:
   replicas: 1
   selector:
     matchLabels:
       app: web
   template:
     metadata:
-      namespace: 01-deny-all
       labels:
         app: web
     spec:
@@ -26,10 +33,10 @@ spec:
 apiVersion: v1
 kind: Service
 metadata:
-  namespace: 01-deny-all
   labels:
     app: web
   name: web
+  namespace: 01-deny-all-traffic-to-an-application
 spec:
   ports:
   - port: 80
@@ -42,8 +49,8 @@ spec:
 kind: NetworkPolicy
 apiVersion: networking.k8s.io/v1
 metadata:
-  namespace: 01-deny-all
   name: web-deny-all
+  namespace: 01-deny-all-traffic-to-an-application
 spec:
   podSelector:
     matchLabels:

e2e-manifests/expected/01-deny-all-traffic-to-an-application.yml

@@ -0,0 +1,4 @@
+01-deny-all-traffic-to-an-application:app=web:
+  01-deny-all-traffic-to-an-application:app=web:
+    -*:
+      success: true

local_dev/run_e2e_tests.sh

@@ -9,11 +9,7 @@ ILLUMINATIO_IMAGE="${DOCKER_REGISTRY}:5000/illuminatio-runner:dev"
 docker build -t "${ILLUMINATIO_IMAGE}" -f illuminatio-runner.dockerfile .
 
 # Use minikube docker daemon to push to the insecure registry
-DOCKER_REGISTRY=${DOCKER_REGISTRY:-"localhost"}
 
-until docker push "${ILLUMINATIO_IMAGE}"
-do
-  sleep 1
-done
+docker push "${ILLUMINATIO_IMAGE}"
 
 python setup.py test --addopts="-m e2e"

local_dev/start_docker.sh

@@ -19,7 +19,6 @@ minikube start \
     --extra-config=controller-manager.cluster-cidr=192.168.0.0/16 \
     --bootstrapper=kubeadm \
     --host-only-cidr=172.17.17.1/24 \
-    --insecure-registry=localhost:5000 \
     --kubernetes-version="${KUBERNETES_VERSION}"
 
 # Setup the minikube docker registry and calico

pytest.ini

@@ -0,0 +1,4 @@
+[pytest]
+markers =
+    e2e: marks tests as e2e tests, (require a running kubernetes cluster)
+    slow

requirements.txt

@@ -1,7 +1,7 @@
 requests>=2.20.0
 google-auth==1.5.1
 python-dateutil==2.7.3
-pyyaml==4.2b4
+pyyaml==5.1.2
 oauthlib==3.0.0
 kubernetes==10.0.1
 click==6.7

tests/test_e2e.py

@@ -1,37 +1,84 @@
-import subprocess
 import pytest
+import subprocess
+import tempfile
+import yaml
 from kubernetes import client, config, utils
+from tests.utils import wait_for_deployments_ready
 
+E2E_INPUT_MANIFEST = "e2e-manifests/{}.yml"
+E2E_EXPECTED_YAML = "e2e-manifests/expected/{}.yml"
+E2E_RUNNER_IMAGE = "localhost:5000/illuminatio-runner:dev"
 
-@pytest.mark.e2e
-def test_deny_all_traffic_to_an_application():
-    namespace = "01-deny-all"
+
+@pytest.fixture
+def load_kube_config():
     config.load_kube_config()
-    k8s_client = client.ApiClient()
-    corev1 = client.CoreV1Api()
-
-    corev1.create_namespace(client.V1Namespace(
-        metadata=client.V1ObjectMeta(
-            name=namespace,
-            labels={"illuminatio-e2e": namespace})))
-    utils.create_from_yaml(k8s_client,
-                           "e2e-manifests/01-deny-all-traffic-to-an-application.yml",
-                           namespace=namespace)
-
-    # ToDo add sleep or wait until all resources are up otherwise we have a race condition
-    # ToDo handle execptions
-    res = subprocess.run(["illuminatio", "run", "--runner-image=localhost:5000/illuminatio-runner:dev"],
-                         capture_output=True,
-                         timeout=60)
-
-    assert res.returncode == 0
-    # ToDo evaluate result
-    print(res.stdout)
-
-    # Clean up
-    res = subprocess.run(["illuminatio", "clean"], capture_output=True)
-    assert res.returncode == 0
-    print(res.stdout)
-
-    # Clean up
-    corev1.delete_namespace(name=namespace)
+
+
+@pytest.fixture
+def api_client(load_kube_config):
+    return client.ApiClient()
+
+
+@pytest.fixture
+def core_v1(load_kube_config):
+    return client.CoreV1Api()
+
+
+@pytest.fixture
+def apps_v1(load_kube_config):
+    return client.AppsV1Api()
+
+
+@pytest.fixture(autouse=True)
+def clean_cluster(core_v1):
+    yield  # below code is executed after test(s)
+    # delete e2e namespaces created in test setup
+    e2e_namespaces = core_v1.list_namespace(label_selector="illuminatio-e2e")
+    for namespace in e2e_namespaces.items:
+        core_v1.delete_namespace(name=namespace.metadata.name)
+    # delete illuminatio resources
+    subprocess.check_output(["illuminatio", "clean"])
+
+
+@pytest.mark.parametrize(
+    "e2e_test_case",
+    [
+        "01-deny-all-traffic-to-an-application"
+        ],
+    )
+@pytest.mark.e2e
+def test__e2e__clean_setup__results_are_expected(e2e_test_case, api_client, apps_v1):
+    # get input and expected from test case name
+    input_manifest = E2E_INPUT_MANIFEST.format(e2e_test_case)
+    expected_yaml = E2E_EXPECTED_YAML.format(e2e_test_case)
+    # create resources to test with
+    utils.create_from_yaml(api_client,
+                           input_manifest)
+    # wait for test resources to be ready
+    wait_for_deployments_ready(e2e_test_case, api=apps_v1)
+    # run illuminatio, with yaml output for later comparison
+    tmp_dir = tempfile.TemporaryDirectory()
+    result_file_name = f"{tmp_dir.name}/result.yaml"
+    with open(result_file_name, "w") as result_file:
+        cmd = ["illuminatio", "run", "--runner-image", f"{E2E_RUNNER_IMAGE}", "-o", f"{result_file.name}"]
+        subprocess.check_output(cmd, timeout=120)
+    # load contents of result and expected
+    result = None
+    with open(result_file_name, "r") as stream:
+        result = yaml.safe_load(stream)
+    assert result is not None, f"Could not load result from {result_file_name}"
+    expected = None
+    with open(expected_yaml, "r") as stream:
+        expected = yaml.safe_load(stream)
+    assert expected is not None,  f"Could not load expected from {expected_yaml}"
+    # assert that the correct cases have been generated and results match
+    assert "cases" in result
+    try:
+        assert expected == result["cases"]
+    except AssertionError as e:
+        print("Generated cases did not match expected. Generated:\n")
+        print(yaml.dump(result["cases"]))
+        print("Expected:\n")
+        print(yaml.dump(expected))
+        raise e

tests/test_test_case.py

@@ -203,7 +203,7 @@ def test_mergeInDict_twoCasesSameFromHostSameToHost_returnsDict():
 def test_toYaml_oneTestCase_returnsExpectedYaml():
     testHost = ClusterHost("namespc", {"label": "val"})
     case = NetworkTestCase(LocalHost(), testHost, 80, False)
-    expected = "localhost:\n  namespc:label=val: ['-80']\n"
+    expected = "localhost:\n  namespc:label=val:\n  - '-80'\n"
     assert to_yaml([case]) == expected
 
 

tests/utils.py

@@ -0,0 +1,32 @@
+import pytest
+import time
+import kubernetes as k8s
+
+
+def wait_for_deployments_ready(e2_test_case,
+                               api: k8s.client.AppsV1Api,
+                               max_tries=30,
+                               sleep_time=5):
+    """
+    Checks e2_test_case's namespaces for Deployments and waits until all are fully ready
+    """
+    label_selector = f"illuminatio-e2e={e2_test_case}"
+    tries = 0
+    print(f"Ensure that Pods of Deployments with labels {label_selector} are ready")
+    while tries <= max_tries:
+        try:
+            deployments = api.list_deployment_for_all_namespaces(label_selector=label_selector)
+            if all([_deployment_ready(d) for d in deployments.items]):
+                return
+        except k8s.client.rest.ApiException as api_exception:
+            print(api_exception)
+        time.sleep(sleep_time)
+        tries += 1
+    waited_time = tries * sleep_time
+    pytest.fail(f"Deployments in {label_selector} have not come up in {waited_time}s")
+
+
+def _deployment_ready(deployment):
+    ready = deployment.status.ready_replicas or 0
+    scheduled = deployment.status.replicas or 0
+    return scheduled > 0 and scheduled == ready