diff --git a/cwltest/utils.py b/cwltest/utils.py
index d4c0570..012e286 100644
--- a/cwltest/utils.py
+++ b/cwltest/utils.py
@@ -422,6 +422,23 @@ def run_test_plain(
             " ".join([shlex.quote(tc) for tc in test_command]),
         )
         raise
+    except json.JSONDecodeError:
+        logger.error(
+            """Test %s failed: %s""",
+            number,
+            " ".join([shlex.quote(tc) for tc in test_command]),
+        )
+        logger.error(test.get("doc", "").replace("\n", " ").strip())
+        invalid_json_msg = "Output is not a valid JSON document: '%s'" % outstr
+        logger.error(invalid_json_msg)
+        return TestResult(
+            1,
+            outstr,
+            outerr,
+            duration,
+            config.classname,
+            invalid_json_msg,
+        )
     except subprocess.TimeoutExpired:
         logger.error(
             """Test %s timed out: %s""",
diff --git a/tests/test-data/dummy-executor.sh b/tests/test-data/dummy-executor.sh
new file mode 100755
index 0000000..ca3a674
--- /dev/null
+++ b/tests/test-data/dummy-executor.sh
@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "it is not JSON format!"
diff --git a/tests/test-data/empty.yml b/tests/test-data/empty.yml
new file mode 100644
index 0000000..0967ef4
--- /dev/null
+++ b/tests/test-data/empty.yml
@@ -0,0 +1 @@
+{}
diff --git a/tests/test-data/nothing.yml b/tests/test-data/nothing.yml
new file mode 100644
index 0000000..1e860a7
--- /dev/null
+++ b/tests/test-data/nothing.yml
@@ -0,0 +1,10 @@
+- job: empty.yml
+  tool: true.cwl
+  output: {}
+  id: do_nothing
+  doc: Example of doing nothing
+- job: empty.yml
+  tool: true.cwl
+  output: {}
+  id: do_nothing2
+  doc: Example of doing nothing more
diff --git a/tests/test-data/true.cwl b/tests/test-data/true.cwl
new file mode 100644
index 0000000..f8e5e7a
--- /dev/null
+++ b/tests/test-data/true.cwl
@@ -0,0 +1,5 @@
+class: CommandLineTool
+cwlVersion: v1.0
+inputs: {}
+outputs: {}
+baseCommand: ["true"]
diff --git a/tests/test_invalid_outputs.py b/tests/test_invalid_outputs.py
new file mode 100644
index 0000000..d57182d
--- /dev/null
+++ b/tests/test_invalid_outputs.py
@@ -0,0 +1,17 @@
+from pathlib import Path
+
+import schema_salad.ref_resolver
+
+from .util import get_data, run_with_mock_cwl_runner
+
+
+def test_invalid_outputs(tmp_path: Path) -> None:
+    args = [
+        "--test",
+        schema_salad.ref_resolver.file_uri(get_data("tests/test-data/nothing.yml")),
+    ]
+    error_code, stdout, stderr = run_with_mock_cwl_runner(
+        args, get_data("tests/test-data/dummy-executor.sh")
+    )
+    assert error_code == 1
+    assert "0 tests passed, 2 failures, 0 unsupported features" in stderr
diff --git a/tests/util.py b/tests/util.py
index 3c5cff3..9394612 100644
--- a/tests/util.py
+++ b/tests/util.py
@@ -5,7 +5,7 @@
 import subprocess  # nosec
 from contextlib import ExitStack
 from pathlib import Path
-from typing import List, Tuple
+from typing import List, Optional, Tuple
 
 from cwltest.utils import as_file, files
 
@@ -28,9 +28,12 @@ def get_data(filename: str) -> str:
     return str(filepath.resolve())
 
 
-def run_with_mock_cwl_runner(args: List[str]) -> Tuple[int, str, str]:
+def run_with_mock_cwl_runner(
+    args: List[str], cwl_runner: Optional[str] = None
+) -> Tuple[int, str, str]:
     """Bind a mock cwlref-runner implementation to cwltest."""
-    cwl_runner = get_data("tests/test-data/mock_cwl_runner.py")
+    if cwl_runner is None:
+        cwl_runner = get_data("tests/test-data/mock_cwl_runner.py")
     process = subprocess.Popen(  # nosec
         ["cwltest", "--tool", cwl_runner] + args,
         stdout=subprocess.PIPE,