Skip to content

Commit

Permalink
Make flow execution output work with JUNIT output format
Browse files Browse the repository at this point in the history
  • Loading branch information
@NyCodeGHG
NyCodeGHG committed Jun 7, 2024
1 parent 5f994da commit e0caf0b
Show file tree
Hide file tree
Showing 5 changed files with 65 additions and 179 deletions.
5 changes: 3 additions & 2 deletions maestro-cli/src/main/java/maestro/cli/command/RecordCommand.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,7 @@ import maestro.cli.App
import maestro.cli.CliError
import maestro.cli.DisableAnsiMixin
import maestro.cli.api.ApiClient
import maestro.cli.model.TestExecutionSummary
import maestro.cli.report.TestDebugReporter
import maestro.cli.runner.TestRunner
import maestro.cli.runner.resultview.AnsiResultView
Expand All @@ -41,7 +42,7 @@ import java.util.concurrent.Callable
"Render a beautiful video of your Flow - Great for demos and bug reports"
]
)
class RecordCommand : Callable<Int> {
class RecordCommand : Callable<TestExecutionSummary.FlowResult> {

@CommandLine.Mixin
var disableANSIMixin: DisableAnsiMixin? = null
Expand All @@ -64,7 +65,7 @@ class RecordCommand : Callable<Int> {
)
private var debugOutput: String? = null

override fun call(): Int {
override fun call(): TestExecutionSummary.FlowResult {
if (!flowFile.exists()) {
throw CommandLine.ParameterException(
commandSpec.commandLine(),
Expand Down
30 changes: 18 additions & 12 deletions maestro-cli/src/main/java/maestro/cli/command/TestCommand.kt
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,12 +22,14 @@ package maestro.cli.command
import maestro.cli.App
import maestro.cli.CliError
import maestro.cli.DisableAnsiMixin
import maestro.cli.model.FlowStatus
import maestro.cli.report.ReportFormat
import maestro.cli.report.ReporterFactory
import maestro.cli.report.TestDebugReporter
import maestro.cli.runner.TestRunner
import maestro.cli.runner.TestSuiteInteractor
import maestro.cli.runner.resultview.AnsiResultView
import maestro.cli.runner.resultview.NoopResultView
import maestro.cli.runner.resultview.PlainTextResultView
import maestro.cli.session.MaestroSessionManager
import maestro.cli.util.PrintUtils
Expand Down Expand Up @@ -72,6 +74,12 @@ class TestCommand : Callable<Int> {
)
private var format: ReportFormat = ReportFormat.NOOP

@Option(
names = ["--hide-execution"],
description = ["Hides the execution flow of the tests."],
)
private var hideExecution: Boolean = false

@Option(
names = ["--test-suite-name"],
description = ["Test suite name"],
Expand Down Expand Up @@ -142,8 +150,11 @@ class TestCommand : Callable<Int> {
return MaestroSessionManager.newSession(parent?.host, parent?.port, deviceId) { session ->
val maestro = session.maestro
val device = session.device
val resultView = if (!hideExecution) {
if (DisableAnsiMixin.ansiEnabled) AnsiResultView() else PlainTextResultView()
} else NoopResultView

if (flowFile.isDirectory || format != ReportFormat.NOOP) {
if (flowFile.isDirectory) {
if (continuous) {
throw CommandLine.ParameterException(
commandSpec.commandLine(),
Expand All @@ -164,12 +175,11 @@ class TestCommand : Callable<Int> {
.sink()
.buffer()
},
view = resultView,
debugOutputPath = debugOutputPath
)

if (!flattenDebugOutput) {
TestDebugReporter.deleteOldFiles()
}
TestDebugReporter.deleteOldFiles()
if (suiteResult.passed) {
0
} else {
Expand All @@ -178,20 +188,16 @@ class TestCommand : Callable<Int> {
}
} else {
if (continuous) {
if(!flattenDebugOutput){
TestDebugReporter.deleteOldFiles()
}
TestDebugReporter.deleteOldFiles()
TestRunner.runContinuous(maestro, device, flowFile, env)
} else {
val resultView = if (DisableAnsiMixin.ansiEnabled) AnsiResultView() else PlainTextResultView()
val resultSingle = TestRunner.runSingle(maestro, device, flowFile, env, resultView, debugOutputPath)
if (resultSingle == 1) {
if (resultSingle.status != FlowStatus.SUCCESS) {
printExitDebugMessage()
}
if(!flattenDebugOutput){
TestDebugReporter.deleteOldFiles()
}
return@newSession resultSingle
TestDebugReporter.deleteOldFiles()
return@newSession if (resultSingle.status != FlowStatus.SUCCESS) 1 else 0
}
}
}
Expand Down
60 changes: 34 additions & 26 deletions maestro-cli/src/main/java/maestro/cli/runner/TestRunner.kt
View file
Original file line number Diff line number Diff line change
@@ -1,36 +1,29 @@
package maestro.cli.runner

import com.github.michaelbull.result.Err
import com.github.michaelbull.result.Ok
import com.github.michaelbull.result.Result
import com.github.michaelbull.result.get
import com.github.michaelbull.result.getOr
import com.github.michaelbull.result.onFailure
import com.github.michaelbull.result.*
import maestro.Maestro
import maestro.cli.device.Device
import maestro.cli.model.FlowStatus
import maestro.cli.model.TestExecutionSummary
import maestro.cli.report.FlowDebugMetadata
import maestro.cli.report.TestDebugReporter
import maestro.cli.runner.resultview.AnsiResultView
import maestro.cli.runner.resultview.ResultView
import maestro.cli.runner.resultview.UiState
import maestro.cli.util.PrintUtils
import maestro.cli.view.ErrorViewUtils
import maestro.debuglog.DebugLogStore
import maestro.debuglog.LogConfig
import maestro.orchestra.MaestroCommand
import maestro.orchestra.MaestroInitFlow
import maestro.orchestra.OrchestraAppState
import maestro.orchestra.util.Env.withEnv
import maestro.orchestra.yaml.YamlCommandReader
import org.slf4j.LoggerFactory
import java.io.File
import java.nio.file.Files
import java.nio.file.Path
import java.nio.file.Paths
import java.time.LocalDateTime
import java.time.format.DateTimeFormatter
import kotlin.concurrent.thread
import kotlin.io.path.absolutePathString
import kotlin.system.measureTimeMillis
import kotlin.time.Duration.Companion.milliseconds
import kotlin.time.measureTime

object TestRunner {

Expand All @@ -43,27 +36,42 @@ object TestRunner {
env: Map<String, String>,
resultView: ResultView,
debugOutputPath: Path
): Int {
): TestExecutionSummary.FlowResult {

// debug
val debug = FlowDebugMetadata()

val result = runCatching(resultView, maestro) {
val commands = YamlCommandReader.readCommands(flowFile.toPath())
.withEnv(env)
MaestroCommandRunner.runCommands(
maestro,
device,
resultView,
commands,
debug
)
}
val result: Result<MaestroCommandRunner.Result, Exception>
val time = measureTimeMillis {
result = runCatching(resultView, maestro) {
val commands = YamlCommandReader.readCommands(flowFile.toPath())
.withEnv(env)
MaestroCommandRunner.runCommands(
maestro,
device,
resultView,
commands,
debug
)
}
}.milliseconds

TestDebugReporter.saveFlow(flowFile.name, debug, debugOutputPath)
if (debug.exception != null) PrintUtils.err("${debug.exception?.message}")

return if (result.get()?.flowSuccess == true) 0 else 1
val flowStatus = if (result.get()?.flowSuccess == true) FlowStatus.SUCCESS else FlowStatus.ERROR

return TestExecutionSummary.FlowResult(
name = flowFile.nameWithoutExtension,
fileName = flowFile.nameWithoutExtension,
status = flowStatus,
failure = if (flowStatus == FlowStatus.ERROR) {
TestExecutionSummary.Failure(
message = result.getError()?.message ?: debug.exception?.message ?: "Unknown error",
)
} else null,
duration = time,
)
}

fun runContinuous(
Expand Down
144 changes: 5 additions & 139 deletions maestro-cli/src/main/java/maestro/cli/runner/TestSuiteInteractor.kt
View file
Original file line number Diff line number Diff line change
@@ -1,27 +1,19 @@
package maestro.cli.runner

import maestro.Maestro
import maestro.MaestroException
import maestro.cli.CliError
import maestro.cli.device.Device
import maestro.cli.model.FlowStatus
import maestro.cli.model.TestExecutionSummary
import maestro.cli.report.*
import maestro.cli.report.TestSuiteReporter
import maestro.cli.runner.resultview.ResultView
import maestro.cli.util.PrintUtils
import maestro.cli.util.TimeUtils
import maestro.cli.view.ErrorViewUtils
import maestro.cli.view.TestSuiteStatusView
import maestro.cli.view.TestSuiteStatusView.TestSuiteViewModel
import maestro.orchestra.Orchestra
import maestro.orchestra.util.Env.withEnv
import maestro.orchestra.workspace.WorkspaceExecutionPlanner
import maestro.orchestra.yaml.YamlCommandReader
import okio.Sink
import org.slf4j.LoggerFactory
import java.io.File
import java.nio.file.Path
import kotlin.math.roundToLong
import kotlin.system.measureTimeMillis
import kotlin.time.Duration.Companion.seconds

class TestSuiteInteractor(
Expand All @@ -35,6 +27,7 @@ class TestSuiteInteractor(
fun runTestSuite(
executionPlan: WorkspaceExecutionPlanner.ExecutionPlan,
reportOut: Sink?,
view: ResultView,
env: Map<String, String>,
debugOutputPath: Path
): TestExecutionSummary {
Expand All @@ -52,7 +45,7 @@ class TestSuiteInteractor(
// first run sequence of flows if present
val flowSequence = executionPlan.sequence
for (flow in flowSequence?.flows ?: emptyList()) {
val result = runFlow(flow.toFile(), env, maestro, debugOutputPath)
val result = TestRunner.runSingle(maestro, device, flow.toFile(),env, view, debugOutputPath)
flowResults.add(result)

if (result.status == FlowStatus.ERROR) {
Expand All @@ -67,7 +60,7 @@ class TestSuiteInteractor(

// proceed to run all other Flows
executionPlan.flowsToRun.forEach { flow ->
val result = runFlow(flow.toFile(), env, maestro, debugOutputPath)
val result = TestRunner.runSingle(maestro, device, flow.toFile(), env, view, debugOutputPath)

if (result.status == FlowStatus.ERROR) {
passed = false
Expand Down Expand Up @@ -115,131 +108,4 @@ class TestSuiteInteractor(
return summary
}

private fun runFlow(
flowFile: File,
env: Map<String, String>,
maestro: Maestro,
debugOutputPath: Path
): TestExecutionSummary.FlowResult {
var flowName: String = flowFile.nameWithoutExtension
var flowStatus: FlowStatus
var errorMessage: String? = null

// debug
val debug = FlowDebugMetadata()
val debugCommands = debug.commands
val debugScreenshots = debug.screenshots

fun takeDebugScreenshot(status: CommandStatus): File? {
val containsFailed = debugScreenshots.any { it.status == CommandStatus.FAILED }

// Avoids duplicate failed images from parent commands
if (containsFailed && status == CommandStatus.FAILED) {
return null
}

val result = kotlin.runCatching {
val out = File.createTempFile("screenshot-${System.currentTimeMillis()}", ".png")
.also { it.deleteOnExit() } // save to another dir before exiting
maestro.takeScreenshot(out, false)
debugScreenshots.add(
ScreenshotDebugMetadata(
screenshot = out,
timestamp = System.currentTimeMillis(),
status = status
)
)
out
}

return result.getOrNull()
}

val flowTimeMillis = measureTimeMillis {
try {
val commands = YamlCommandReader.readCommands(flowFile.toPath())
.withEnv(env)

val config = YamlCommandReader.getConfig(commands)

val orchestra = Orchestra(
maestro = maestro,
onCommandStart = { _, command ->
logger.info("${command.description()} RUNNING")
debugCommands[command] = CommandDebugMetadata(
timestamp = System.currentTimeMillis(),
status = CommandStatus.RUNNING
)
},
onCommandComplete = { _, command ->
logger.info("${command.description()} COMPLETED")
debugCommands[command]?.let {
it.status = CommandStatus.COMPLETED
it.calculateDuration()
}
},
onCommandFailed = { _, command, e ->
logger.info("${command.description()} FAILED")
if (e is MaestroException) debug.exception = e
debugCommands[command]?.let {
it.status = CommandStatus.FAILED
it.calculateDuration()
it.error = e
}

takeDebugScreenshot(CommandStatus.FAILED)
Orchestra.ErrorResolution.FAIL
},
onCommandSkipped = { _, command ->
logger.info("${command.description()} SKIPPED")
debugCommands[command]?.let {
it.status = CommandStatus.SKIPPED
}
},
onCommandReset = { command ->
logger.info("${command.description()} PENDING")
debugCommands[command]?.let {
it.status = CommandStatus.PENDING
}
},
)

config?.name?.let {
flowName = it
}

val flowSuccess = orchestra.runFlow(commands)
flowStatus = if (flowSuccess) FlowStatus.SUCCESS else FlowStatus.ERROR
} catch (e: Exception) {
logger.error("Failed to complete flow", e)
flowStatus = FlowStatus.ERROR
errorMessage = ErrorViewUtils.exceptionToMessage(e)
}
}
val flowDuration = TimeUtils.durationInSeconds(flowTimeMillis)

TestDebugReporter.saveFlow(flowName, debug, debugOutputPath)

TestSuiteStatusView.showFlowCompletion(
TestSuiteViewModel.FlowResult(
name = flowName,
status = flowStatus,
duration = flowDuration,
error = debug.exception?.message,
)
)

return TestExecutionSummary.FlowResult(
name = flowName,
fileName = flowFile.nameWithoutExtension,
status = flowStatus,
failure = if (flowStatus == FlowStatus.ERROR) {
TestExecutionSummary.Failure(
message = errorMessage ?: debug.exception?.message ?: "Unknown error",
)
} else null,
duration = flowDuration,
)
}

}
Loading

0 comments on commit e0caf0b

Please sign in to comment.