Handling invalid output object from engines

cwltest/utils.py

@@ -422,6 +422,23 @@ def run_test_plain(
  " ".join([shlex.quote(tc) for tc in test_command]),
  )
  raise
+ except json.JSONDecodeError:
+ logger.error(
+ """Test %s failed: %s""",
+ number,
+ " ".join([shlex.quote(tc) for tc in test_command]),
+ )
+ logger.error(test.get("doc", "").replace("\n", " ").strip())
+ invalid_json_msg = "Output is not a valid JSON document: '%s'" % outstr
+ logger.error(invalid_json_msg)
+ return TestResult(
+ 1,
+ outstr,
+ outerr,
+ duration,
+ config.classname,
+ invalid_json_msg,
+ )
  except subprocess.TimeoutExpired:
  logger.error(
  """Test %s timed out: %s""",

tests/test-data/dummy-executor.sh

@@ -0,0 +1,3 @@
+#!/bin/sh
+
+echo "it is not JSON format!"

tests/test-data/empty.yml

@@ -0,0 +1 @@
+{}

tests/test-data/nothing.yml

@@ -0,0 +1,10 @@
+- job: empty.yml
+ tool: true.cwl
+ output: {}
+ id: do_nothing
+ doc: Example of doing nothing
+- job: empty.yml
+ tool: true.cwl
+ output: {}
+ id: do_nothing2
+ doc: Example of doing nothing more

tests/test-data/true.cwl

@@ -0,0 +1,5 @@
+class: CommandLineTool
+cwlVersion: v1.0
+inputs: {}
+outputs: {}
+baseCommand: ["true"]

tests/test_invalid_outputs.py

@@ -0,0 +1,17 @@
+from pathlib import Path
+
+import schema_salad.ref_resolver
+
+from .util import get_data, run_with_mock_cwl_runner
+
+
+def test_invalid_outputs(tmp_path: Path) -> None:
+ args = [
+ "--test",
+ schema_salad.ref_resolver.file_uri(get_data("tests/test-data/nothing.yml")),
+ ]
+ error_code, stdout, stderr = run_with_mock_cwl_runner(
+ args, get_data("tests/test-data/dummy-executor.sh")
+ )
+ assert error_code == 1
+ assert "0 tests passed, 2 failures, 0 unsupported features" in stderr

tests/util.py

@@ -5,7 +5,7 @@
 import subprocess # nosec
 from contextlib import ExitStack
 from pathlib import Path
-from typing import List, Tuple
+from typing import List, Optional, Tuple
 
 from cwltest.utils import as_file, files
 
@@ -28,9 +28,12 @@ def get_data(filename: str) -> str:
  return str(filepath.resolve())
 
 
-def run_with_mock_cwl_runner(args: List[str]) -> Tuple[int, str, str]:
+def run_with_mock_cwl_runner(
+ args: List[str], cwl_runner: Optional[str] = None
+) -> Tuple[int, str, str]:
  """Bind a mock cwlref-runner implementation to cwltest."""
- cwl_runner = get_data("tests/test-data/mock_cwl_runner.py")
+ if cwl_runner is None:
+ cwl_runner = get_data("tests/test-data/mock_cwl_runner.py")
  process = subprocess.Popen( # nosec
  ["cwltest", "--tool", cwl_runner] + args,
  stdout=subprocess.PIPE,