Allow multiple input test cases (#85)

README.md

@@ -29,11 +29,20 @@ You will get output like so:
 ```bash
 $ invoke test python any
 
-> 🟢 script "./src/python/sort_builtin.py" succeeded
-> 🟢 script "./src/python/sort_bubble_sort.py" succeeded
-> 🟢 script "./src/python/sort_selection_sort.py" succeeded
-> 🟢 script "./src/python/sort_insertion_sort.py" succeeded
-> ✨ script run success ✨
+docker run ...
+ ⏱ ./src/python/sort_builtin.py on ./data/sort_input_0.txt ran for 0.5 seconds
+ 🟢 ./src/python/sort_builtin.py on ./data/sort_input_0.txt succeeded
+docker run ...
+ ⏱ ./src/python/sort_bubble_sort.py on ./data/sort_input_0.txt ran for 0.51 seconds
+ 🟢 ./src/python/sort_bubble_sort.py on ./data/sort_input_0.txt succeeded
+docker run ...
+ ⏱ ./src/python/sort_selection_sort.py on ./data/sort_input_0.txt ran for 0.58 seconds
+ 🟢 ./src/python/sort_selection_sort.py on ./data/sort_input_0.txt succeeded
+docker run ...
+ ⏱ ./src/python/sort_insertion_sort.py on ./data/sort_input_0.txt ran for 0.52 seconds
+ 🟢 ./src/python/sort_insertion_sort.py on ./data/sort_input_0.txt succeeded
+
+✨ script run success ✨
 ```
 
 _(note: I will likely not bother to update the above example when the output changes in minor ways)_

tasks.py

@@ -42,10 +42,12 @@ class TestRunnerContext:
  script_name: str
  docker_pull: str
  docker_run_test: str
+ script_type: str
  script_relative_path: str
  script_name_with_file_type: str
  script_output_file_path: str
  script_output_file_name: str
+ input_file_path: str
  prepared_file_path: str
  snippet_start_line: int
  snippet_end_line: int
@@ -73,23 +75,31 @@ def __init__(self, language) -> None:
  # generate the contexts
  self.ctxs = []
  for script_path in glob.glob(f"{self.base_directory}/src/{language}/*"):
+ # given "src/python/sort_builtin.py" => return "sort"
+ script_type = script_path.split("/")[-1].split("_")[0]
+
  # ignore helpers, metadata files, etc
  if config.get("ignoreFiles") and script_path.split("/")[-1] in config.get(
  "ignoreFiles"
  ):
  continue
+
  # ignore directories, generally compiled code
  if not os.path.isfile(script_path):
  continue
- # generate a context for this particular script
- if ctx := self.generate(language, config, script_path):
- self.ctxs.append(ctx)
+
+ for input_file_path in glob.glob(
+ f"{self.data_folder_path}/{script_type}_input_*.txt"
+ ):
+ # generate a context for this particular script
+ if ctx := self.generate(language, config, script_path, input_file_path):
+ self.ctxs.append(ctx)
 
  @property
  def data(self):
  return [ctx.data for ctx in self.ctxs]
 
- def generate(self, language, config, script_path):
+ def generate(self, language, config, script_path, input_file_path):
  # given "src/python/sort_builtin.py" => split on "/" and return "sort_builtin.py"
  script_path_split_on_slash = script_path.split("/")
  script_name_with_file_type = script_path_split_on_slash[-1]
@@ -106,11 +116,14 @@ def generate(self, language, config, script_path):
 
  # get the path of the file that's been prepared in advance
  # and has the output we would be expecting from out script
- prepared_file_path = f"{self.data_folder_path}/{script_type}_output.txt"
+ prepared_file_path = input_file_path.replace("input", "output")
+
+ # given "data/sort_input_1.txt" => return "1"
+ prepared_file_index = prepared_file_path.split("_")[-1].split(".")[0]
 
  # our scripts write their output files to this path
  script_output_file_name = (
- f"output_{script_type}_via_{language}_{script_name}.txt"
+ f"output_{language}_{script_name}_{prepared_file_index}.txt"
  )
  script_output_file_path = f"{self.data_folder_path}/{script_output_file_name}"
 
@@ -142,7 +155,7 @@ def generate(self, language, config, script_path):
 
  # construct ending call args
  docker_run_test_list += [
- f"-e=INPUT_PATH={self.data_folder_path}/{script_type}_input.txt",
+ f"-e=INPUT_PATH={input_file_path}",
  f"-e=OUTPUT_PATH={script_output_file_path}",
  config["dockerImage"],
  *script_invoker,
@@ -170,14 +183,14 @@ def generate(self, language, config, script_path):
  if snippet_start_line != 0:
  raise Exception(
  f'Found multiple "{self.snippet_start_text}" lines in {script_relative_path}.\n'
- f"The text with found on lines {snippet_start_line - snippet_start_line_offset + 1} and {idx + 1}."
+ f"The lines were {snippet_start_line - snippet_start_line_offset + 1} and {idx + 1}."
  )
  snippet_start_line = idx + 3
  if self.snippet_end_text in line:
  if snippet_end_line != 0:
  raise Exception(
  f'Found multiple "{self.snippet_end_text}" lines in {script_relative_path}.\n'
- f"The text with found on lines {snippet_end_line + snippet_end_line_offset + 1} and {idx + 1}."
+ f"The lines were {snippet_end_line + snippet_end_line_offset + 1} and {idx + 1}."
  )
  snippet_end_line = idx - snippet_end_line_offset
  if snippet_start_line == 0:
@@ -195,10 +208,12 @@ def generate(self, language, config, script_path):
  script_name=script_name,
  docker_pull=docker_pull,
  docker_run_test=docker_run_test,
+ script_type=script_type,
  script_relative_path=script_relative_path,
  script_name_with_file_type=script_name_with_file_type,
  script_output_file_path=script_output_file_path,
  script_output_file_name=script_output_file_name,
+ input_file_path=input_file_path,
  prepared_file_path=prepared_file_path,
  snippet_start_line=snippet_start_line,
  snippet_end_line=snippet_end_line,
@@ -244,19 +259,24 @@ def run_tests(self, input_script):
  # report timing
  # we round the number so humans dont over-index on small differences
  print(
- f'\t⏱ script "{ctx.script_relative_path}" run for {round(end_time - start_time, 2)} seconds'
+ f"\t⏱ {ctx.script_relative_path} on {ctx.input_file_path} "
+ f"ran for {round(end_time - start_time, 2)} seconds"
  )
 
  # check if the script invoke failed
  if output.exited != 0:
  self.set_success_status(False)
- print(f'\t🔴 script "{ctx.script_relative_path}" failed, reason:')
+ print(
+ f"\t🔴 {ctx.script_relative_path} on {ctx.input_file_path} failed, reason:"
+ )
  print(f'\t\t the exit code "{output.exited}" was not 0')
 
  # check if the output file was created
  if not os.path.exists(ctx.script_output_file_path):
  self.set_success_status(False)
- print(f'\t🔴 script "{ctx.script_relative_path}" failed, reason:')
+ print(
+ f"\t🔴 {ctx.script_relative_path} on {ctx.input_file_path} failed, reason:"
+ )
  print(
  f"\t\t the output {ctx.script_output_file_name} file was not created"
  )
@@ -266,14 +286,18 @@ def run_tests(self, input_script):
  ctx.prepared_file_path, ctx.script_output_file_path
  ):
  self.set_success_status(True)
- print(f'\t🟢 script "{ctx.script_relative_path}" succeeded')
+ print(
+ f"\t🟢 {ctx.script_relative_path} on {ctx.input_file_path} succeeded"
+ )
 
  # check if the output file does not match the prepared file
  if os.path.exists(ctx.script_output_file_path) and not filecmp.cmp(
  ctx.prepared_file_path, ctx.script_output_file_path
  ):
  self.set_success_status(False)
- print(f'\t🔴 script "{ctx.script_relative_path}" failed, reason:')
+ print(
+ f"\t🔴 {ctx.script_relative_path} on {ctx.input_file_path} failed, reason:"
+ )
  print(
  f"\t\t output file {ctx.script_output_file_name} has does not match the prepared file"
  )
@@ -370,4 +394,4 @@ def test(ctx: invoke.Context, language, input_script):
 
 @invoke.task
 def clean(ctx: invoke.Context):
- ctx.run("git clean -fdx ./data/*")
+ ctx.run("git clean -fdx ./data/output_*")