Make flow execution output work with JUNIT output format

maestro-cli/src/main/java/maestro/cli/command/RecordCommand.kt

@@ -23,6 +23,7 @@ import maestro.cli.App
 import maestro.cli.CliError
 import maestro.cli.DisableAnsiMixin
 import maestro.cli.api.ApiClient
+import maestro.cli.model.TestExecutionSummary
 import maestro.cli.report.TestDebugReporter
 import maestro.cli.runner.TestRunner
 import maestro.cli.runner.resultview.AnsiResultView
@@ -41,7 +42,7 @@ import java.util.concurrent.Callable
  "Render a beautiful video of your Flow - Great for demos and bug reports"
  ]
 )
-class RecordCommand : Callable<Int> {
+class RecordCommand : Callable<TestExecutionSummary.FlowResult> {
 
  @CommandLine.Mixin
  var disableANSIMixin: DisableAnsiMixin? = null
@@ -64,7 +65,7 @@ class RecordCommand : Callable<Int> {
  )
  private var debugOutput: String? = null
 
- override fun call(): Int {
+ override fun call(): TestExecutionSummary.FlowResult {
  if (!flowFile.exists()) {
  throw CommandLine.ParameterException(
  commandSpec.commandLine(),

maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt

@@ -22,12 +22,14 @@ package maestro.cli.command
 import maestro.cli.App
 import maestro.cli.CliError
 import maestro.cli.DisableAnsiMixin
+import maestro.cli.model.FlowStatus
 import maestro.cli.report.ReportFormat
 import maestro.cli.report.ReporterFactory
 import maestro.cli.report.TestDebugReporter
 import maestro.cli.runner.TestRunner
 import maestro.cli.runner.TestSuiteInteractor
 import maestro.cli.runner.resultview.AnsiResultView
+import maestro.cli.runner.resultview.NoopResultView
 import maestro.cli.runner.resultview.PlainTextResultView
 import maestro.cli.session.MaestroSessionManager
 import maestro.cli.util.PrintUtils
@@ -72,6 +74,12 @@ class TestCommand : Callable<Int> {
  )
  private var format: ReportFormat = ReportFormat.NOOP
 
+ @Option(
+ names = ["--hide-execution"],
+ description = ["Hides the execution flow of the tests."],
+ )
+ private var hideExecution: Boolean = false
+
  @Option(
  names = ["--test-suite-name"],
  description = ["Test suite name"],
@@ -142,8 +150,11 @@ class TestCommand : Callable<Int> {
  return MaestroSessionManager.newSession(parent?.host, parent?.port, deviceId) { session ->
  val maestro = session.maestro
  val device = session.device
+ val resultView = if (!hideExecution) {
+ if (DisableAnsiMixin.ansiEnabled) AnsiResultView() else PlainTextResultView()
+ } else NoopResultView
 
- if (flowFile.isDirectory || format != ReportFormat.NOOP) {
+ if (flowFile.isDirectory) {
  if (continuous) {
  throw CommandLine.ParameterException(
  commandSpec.commandLine(),
@@ -164,12 +175,11 @@ class TestCommand : Callable<Int> {
  .sink()
  .buffer()
  },
+ view = resultView,
  debugOutputPath = debugOutputPath
  )
 
- if (!flattenDebugOutput) {
- TestDebugReporter.deleteOldFiles()
- }
+ TestDebugReporter.deleteOldFiles()
  if (suiteResult.passed) {
  0
  } else {
@@ -178,20 +188,16 @@ class TestCommand : Callable<Int> {
  }
  } else {
  if (continuous) {
- if(!flattenDebugOutput){
- TestDebugReporter.deleteOldFiles()
- }
+ TestDebugReporter.deleteOldFiles()
  TestRunner.runContinuous(maestro, device, flowFile, env)
  } else {
  val resultView = if (DisableAnsiMixin.ansiEnabled) AnsiResultView() else PlainTextResultView()
  val resultSingle = TestRunner.runSingle(maestro, device, flowFile, env, resultView, debugOutputPath)
- if (resultSingle == 1) {
+ if (resultSingle.status != FlowStatus.SUCCESS) {
  printExitDebugMessage()
  }
- if(!flattenDebugOutput){
- TestDebugReporter.deleteOldFiles()
- }
- return@newSession resultSingle
+ TestDebugReporter.deleteOldFiles()
+ return@newSession if (resultSingle.status != FlowStatus.SUCCESS) 1 else 0
  }
  }
  }

maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt

@@ -1,36 +1,29 @@
 package maestro.cli.runner
 
-import com.github.michaelbull.result.Err
-import com.github.michaelbull.result.Ok
-import com.github.michaelbull.result.Result
-import com.github.michaelbull.result.get
-import com.github.michaelbull.result.getOr
-import com.github.michaelbull.result.onFailure
+import com.github.michaelbull.result.*
 import maestro.Maestro
 import maestro.cli.device.Device
+import maestro.cli.model.FlowStatus
+import maestro.cli.model.TestExecutionSummary
 import maestro.cli.report.FlowDebugMetadata
 import maestro.cli.report.TestDebugReporter
 import maestro.cli.runner.resultview.AnsiResultView
 import maestro.cli.runner.resultview.ResultView
 import maestro.cli.runner.resultview.UiState
 import maestro.cli.util.PrintUtils
 import maestro.cli.view.ErrorViewUtils
-import maestro.debuglog.DebugLogStore
-import maestro.debuglog.LogConfig
 import maestro.orchestra.MaestroCommand
 import maestro.orchestra.MaestroInitFlow
 import maestro.orchestra.OrchestraAppState
 import maestro.orchestra.util.Env.withEnv
 import maestro.orchestra.yaml.YamlCommandReader
 import org.slf4j.LoggerFactory
 import java.io.File
-import java.nio.file.Files
 import java.nio.file.Path
-import java.nio.file.Paths
-import java.time.LocalDateTime
-import java.time.format.DateTimeFormatter
 import kotlin.concurrent.thread
-import kotlin.io.path.absolutePathString
+import kotlin.system.measureTimeMillis
+import kotlin.time.Duration.Companion.milliseconds
+import kotlin.time.measureTime
 
 object TestRunner {
 
@@ -43,27 +36,42 @@ object TestRunner {
  env: Map<String, String>,
  resultView: ResultView,
  debugOutputPath: Path
- ): Int {
+ ): TestExecutionSummary.FlowResult {
 
  // debug
  val debug = FlowDebugMetadata()
 
- val result = runCatching(resultView, maestro) {
- val commands = YamlCommandReader.readCommands(flowFile.toPath())
- .withEnv(env)
- MaestroCommandRunner.runCommands(
- maestro,
- device,
- resultView,
- commands,
- debug
- )
- }
+ val result: Result<MaestroCommandRunner.Result, Exception>
+ val time = measureTimeMillis {
+ result = runCatching(resultView, maestro) {
+ val commands = YamlCommandReader.readCommands(flowFile.toPath())
+ .withEnv(env)
+ MaestroCommandRunner.runCommands(
+ maestro,
+ device,
+ resultView,
+ commands,
+ debug
+ )
+ }
+ }.milliseconds
 
  TestDebugReporter.saveFlow(flowFile.name, debug, debugOutputPath)
  if (debug.exception != null) PrintUtils.err("${debug.exception?.message}")
 
- return if (result.get()?.flowSuccess == true) 0 else 1
+ val flowStatus = if (result.get()?.flowSuccess == true) FlowStatus.SUCCESS else FlowStatus.ERROR
+
+ return TestExecutionSummary.FlowResult(
+ name = flowFile.nameWithoutExtension,
+ fileName = flowFile.nameWithoutExtension,
+ status = flowStatus,
+ failure = if (flowStatus == FlowStatus.ERROR) {
+ TestExecutionSummary.Failure(
+ message = result.getError()?.message ?: debug.exception?.message ?: "Unknown error",
+ )
+ } else null,
+ duration = time,
+ )
  }
 
  fun runContinuous(

maestro-cli/src/main/java/maestro/cli/runner/TestSuiteInteractor.kt

@@ -1,27 +1,19 @@
 package maestro.cli.runner
 
 import maestro.Maestro
-import maestro.MaestroException
 import maestro.cli.CliError
 import maestro.cli.device.Device
 import maestro.cli.model.FlowStatus
 import maestro.cli.model.TestExecutionSummary
-import maestro.cli.report.*
+import maestro.cli.report.TestSuiteReporter
+import maestro.cli.runner.resultview.ResultView
 import maestro.cli.util.PrintUtils
-import maestro.cli.util.TimeUtils
-import maestro.cli.view.ErrorViewUtils
 import maestro.cli.view.TestSuiteStatusView
 import maestro.cli.view.TestSuiteStatusView.TestSuiteViewModel
-import maestro.orchestra.Orchestra
-import maestro.orchestra.util.Env.withEnv
 import maestro.orchestra.workspace.WorkspaceExecutionPlanner
-import maestro.orchestra.yaml.YamlCommandReader
 import okio.Sink
 import org.slf4j.LoggerFactory
-import java.io.File
 import java.nio.file.Path
-import kotlin.math.roundToLong
-import kotlin.system.measureTimeMillis
 import kotlin.time.Duration.Companion.seconds
 
 class TestSuiteInteractor(
@@ -35,6 +27,7 @@ class TestSuiteInteractor(
  fun runTestSuite(
  executionPlan: WorkspaceExecutionPlanner.ExecutionPlan,
  reportOut: Sink?,
+ view: ResultView,
  env: Map<String, String>,
  debugOutputPath: Path
  ): TestExecutionSummary {
@@ -52,7 +45,7 @@ class TestSuiteInteractor(
  // first run sequence of flows if present
  val flowSequence = executionPlan.sequence
  for (flow in flowSequence?.flows ?: emptyList()) {
- val result = runFlow(flow.toFile(), env, maestro, debugOutputPath)
+ val result = TestRunner.runSingle(maestro, device, flow.toFile(),env, view, debugOutputPath)
  flowResults.add(result)
 
  if (result.status == FlowStatus.ERROR) {
@@ -67,7 +60,7 @@ class TestSuiteInteractor(
 
  // proceed to run all other Flows
  executionPlan.flowsToRun.forEach { flow ->
- val result = runFlow(flow.toFile(), env, maestro, debugOutputPath)
+ val result = TestRunner.runSingle(maestro, device, flow.toFile(), env, view, debugOutputPath)
 
  if (result.status == FlowStatus.ERROR) {
  passed = false
@@ -115,131 +108,4 @@ class TestSuiteInteractor(
  return summary
  }
 
- private fun runFlow(
- flowFile: File,
- env: Map<String, String>,
- maestro: Maestro,
- debugOutputPath: Path
- ): TestExecutionSummary.FlowResult {
- var flowName: String = flowFile.nameWithoutExtension
- var flowStatus: FlowStatus
- var errorMessage: String? = null
-
- // debug
- val debug = FlowDebugMetadata()
- val debugCommands = debug.commands
- val debugScreenshots = debug.screenshots
-
- fun takeDebugScreenshot(status: CommandStatus): File? {
- val containsFailed = debugScreenshots.any { it.status == CommandStatus.FAILED }
-
- // Avoids duplicate failed images from parent commands
- if (containsFailed && status == CommandStatus.FAILED) {
- return null
- }
-
- val result = kotlin.runCatching {
- val out = File.createTempFile("screenshot-${System.currentTimeMillis()}", ".png")
- .also { it.deleteOnExit() } // save to another dir before exiting
- maestro.takeScreenshot(out, false)
- debugScreenshots.add(
- ScreenshotDebugMetadata(
- screenshot = out,
- timestamp = System.currentTimeMillis(),
- status = status
- )
- )
- out
- }
-
- return result.getOrNull()
- }
-
- val flowTimeMillis = measureTimeMillis {
- try {
- val commands = YamlCommandReader.readCommands(flowFile.toPath())
- .withEnv(env)
-
- val config = YamlCommandReader.getConfig(commands)
-
- val orchestra = Orchestra(
- maestro = maestro,
- onCommandStart = { _, command ->
- logger.info("${command.description()} RUNNING")
- debugCommands[command] = CommandDebugMetadata(
- timestamp = System.currentTimeMillis(),
- status = CommandStatus.RUNNING
- )
- },
- onCommandComplete = { _, command ->
- logger.info("${command.description()} COMPLETED")
- debugCommands[command]?.let {
- it.status = CommandStatus.COMPLETED
- it.calculateDuration()
- }
- },
- onCommandFailed = { _, command, e ->
- logger.info("${command.description()} FAILED")
- if (e is MaestroException) debug.exception = e
- debugCommands[command]?.let {
- it.status = CommandStatus.FAILED
- it.calculateDuration()
- it.error = e
- }
-
- takeDebugScreenshot(CommandStatus.FAILED)
- Orchestra.ErrorResolution.FAIL
- },
- onCommandSkipped = { _, command ->
- logger.info("${command.description()} SKIPPED")
- debugCommands[command]?.let {
- it.status = CommandStatus.SKIPPED
- }
- },
- onCommandReset = { command ->
- logger.info("${command.description()} PENDING")
- debugCommands[command]?.let {
- it.status = CommandStatus.PENDING
- }
- },
- )
-
- config?.name?.let {
- flowName = it
- }
-
- val flowSuccess = orchestra.runFlow(commands)
- flowStatus = if (flowSuccess) FlowStatus.SUCCESS else FlowStatus.ERROR
- } catch (e: Exception) {
- logger.error("Failed to complete flow", e)
- flowStatus = FlowStatus.ERROR
- errorMessage = ErrorViewUtils.exceptionToMessage(e)
- }
- }
- val flowDuration = TimeUtils.durationInSeconds(flowTimeMillis)
-
- TestDebugReporter.saveFlow(flowName, debug, debugOutputPath)
-
- TestSuiteStatusView.showFlowCompletion(
- TestSuiteViewModel.FlowResult(
- name = flowName,
- status = flowStatus,
- duration = flowDuration,
- error = debug.exception?.message,
- )
- )
-
- return TestExecutionSummary.FlowResult(
- name = flowName,
- fileName = flowFile.nameWithoutExtension,
- status = flowStatus,
- failure = if (flowStatus == FlowStatus.ERROR) {
- TestExecutionSummary.Failure(
- message = errorMessage ?: debug.exception?.message ?: "Unknown error",
- )
- } else null,
- duration = flowDuration,
- )
- }
-
 }