diff --git a/.github/workflows/test-test.yaml b/.github/workflows/test-test.yaml
index 3ba531d8..5051d361 100644
--- a/.github/workflows/test-test.yaml
+++ b/.github/workflows/test-test.yaml
@@ -11,7 +11,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Check
+      - name: Test
         uses: ./terraform-test
         with:
           path: tests/workflows/test-test/local
@@ -23,7 +23,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Check
+      - name: Test
         uses: ./terraform-test
         with:
           path: tests/workflows/test-test/local
@@ -36,7 +36,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Check
+      - name: Test
         uses: ./terraform-test
         with:
           path: tests/workflows/test-test/local
@@ -52,7 +52,7 @@ jobs:
       - name: Checkout
         uses: actions/checkout@v3
 
-      - name: Check
+      - name: Test
         uses: ./terraform-test
         id: failing
         continue-on-error: true
@@ -68,3 +68,8 @@ jobs:
             echo "Test did not fail correctly"
             exit 1
           fi
+
+          if [[ "${{ steps.failing.outcome }}" != "failure" ]]; then
+            echo "Test did not fail correctly"
+            exit 1
+          fi
diff --git a/image/entrypoints/test.sh b/image/entrypoints/test.sh
index e23959be..c02dd0c4 100755
--- a/image/entrypoints/test.sh
+++ b/image/entrypoints/test.sh
@@ -16,7 +16,7 @@ function set-test-args() {
     fi
 
     if [[ -n "$INPUT_TEST_PATH" ]]; then
-        TEST_ARGS="$TEST_ARGS -test-directory=$(relative_to "$INPUT_PATH" "$INPUT_TEST_PATH")"
+        TEST_ARGS="$TEST_ARGS -test-directory=$INPUT_TEST_PATH"
     fi
 
     if [[ -n "$INPUT_TEST_FILTER" ]]; then
@@ -32,4 +32,7 @@ PLAN_ARGS=""
 set-variable-args
 
 debug_log $TOOL_COMMAND_NAME test -no-color $TEST_ARGS '$PLAN_ARGS'  # don't expand PLAN_ARGS
-$TOOL_COMMAND_NAME test -no-color $TEST_ARGS $PLAN_ARGS
+if ! (cd "$INPUT_PATH" && $TOOL_COMMAND_NAME test -no-color $TEST_ARGS $PLAN_ARGS); then
+    set_output failure-reason tests-failed
+    exit 1
+fi
diff --git a/tests/workflows/test-test/local/custom-test-dir/a-third.tftest.hcl b/tests/workflows/test-test/local/custom-test-dir/a-third.tftest.hcl
index 0ff95c82..ce9d73b8 100644
--- a/tests/workflows/test-test/local/custom-test-dir/a-third.tftest.hcl
+++ b/tests/workflows/test-test/local/custom-test-dir/a-third.tftest.hcl
@@ -1,3 +1,12 @@
+run "require_provider" {
+  # This will require terraform to be initialised correctly to bring in the time provider
+  command = apply
+
+  module = {
+    source = "./sleep"
+  }
+}
+
 run "test_third" {
   command = apply
 
diff --git a/tests/workflows/test-test/local/sleep/main.tf b/tests/workflows/test-test/local/sleep/main.tf
new file mode 100644
index 00000000..e5913152
--- /dev/null
+++ b/tests/workflows/test-test/local/sleep/main.tf
@@ -0,0 +1,3 @@
+resource "time_sleep" "this" {
+  create_duration = "1s"
+}
\ No newline at end of file