Implement slowness checking and --reference flag to use a command as a reference when printing or exporting summary

src/benchmark/relative_speed.rs

@@ -8,46 +8,61 @@ pub struct BenchmarkResultWithRelativeSpeed<'a> {
  pub result: &'a BenchmarkResult,
  pub relative_speed: Scalar,
  pub relative_speed_stddev: Option<Scalar>,
- pub is_fastest: bool,
+ pub is_reference: bool,
+ pub is_faster: bool,
 }
 
 pub fn compare_mean_time(l: &BenchmarkResult, r: &BenchmarkResult) -> Ordering {
  l.mean.partial_cmp(&r.mean).unwrap_or(Ordering::Equal)
 }
 
-pub fn compute(results: &[BenchmarkResult]) -> Option<Vec<BenchmarkResultWithRelativeSpeed>> {
- let fastest: &BenchmarkResult = results
+pub fn fastest(results: &[BenchmarkResult]) -> &BenchmarkResult {
+ results
  .iter()
  .min_by(|&l, &r| compare_mean_time(l, r))
- .expect("at least one benchmark result");
+ .expect("at least one benchmark result")
+}
 
- if fastest.mean == 0.0 {
+pub fn compute<'a>(
+ reference: &'a BenchmarkResult,
+ results: &'a [BenchmarkResult],
+) -> Option<Vec<BenchmarkResultWithRelativeSpeed<'a>>> {
+ if reference.mean == 0.0 {
  return None;
  }
 
  Some(
  results
  .iter()
  .map(|result| {
- let ratio = result.mean / fastest.mean;
+ let is_reference = result == reference;
+ let is_faster = result.mean >= reference.mean;
 
- // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulas
- // Covariance asssumed to be 0, i.e. variables are assumed to be independent
- let ratio_stddev = match (result.stddev, fastest.stddev) {
- (Some(result_stddev), Some(fastest_stddev)) => Some(
- ratio
- * ((result_stddev / result.mean).powi(2)
- + (fastest_stddev / fastest.mean).powi(2))
- .sqrt(),
- ),
- _ => None,
+ let ratio = if is_faster {
+ result.mean / reference.mean
+ } else {
+ reference.mean / result.mean
  };
 
+ // https://en.wikipedia.org/wiki/Propagation_of_uncertainty#Example_formulae
+ // Covariance asssumed to be 0, i.e. variables are assumed to be independent
+ let ratio_stddev =
+ (result.stddev)
+ .zip(reference.stddev)
+ .map(|(result_stddev, fastest_stddev)| {
+ let (a, b) = (
+ (result_stddev / result.mean),
+ (fastest_stddev / reference.mean),
+ );
+ ratio * (a.powi(2) + b.powi(2)).sqrt()
+ });
+
  BenchmarkResultWithRelativeSpeed {
  result,
  relative_speed: ratio,
  relative_speed_stddev: ratio_stddev,
- is_fastest: result == fastest,
+ is_reference,
+ is_faster,
  }
  })
  .collect(),
@@ -83,7 +98,7 @@ fn test_compute_relative_speed() {
  create_result("cmd3", 5.0),
  ];
 
- let annotated_results = compute(&results).unwrap();
+ let annotated_results = compute(fastest(&results), &results).unwrap();
 
  assert_relative_eq!(1.5, annotated_results[0].relative_speed);
  assert_relative_eq!(1.0, annotated_results[1].relative_speed);
@@ -94,7 +109,7 @@ fn test_compute_relative_speed() {
 fn test_compute_relative_speed_for_zero_times() {
  let results = vec![create_result("cmd1", 1.0), create_result("cmd2", 0.0)];
 
- let annotated_results = compute(&results);
+ let annotated_results = compute(fastest(&results), &results);
 
  assert!(annotated_results.is_none());
 }

src/benchmark/scheduler.rs

@@ -4,7 +4,7 @@ use super::benchmark_result::BenchmarkResult;
 use super::executor::{Executor, MockExecutor, RawExecutor, ShellExecutor};
 use super::{relative_speed, Benchmark};
 
-use crate::command::Commands;
+use crate::command::{Command, Commands};
 use crate::export::ExportManager;
 use crate::options::{ExecutorKind, Options, OutputStyleOption};
 
@@ -40,7 +40,12 @@ impl<'a> Scheduler<'a> {
 
  executor.calibrate()?;
 
- for (number, cmd) in self.commands.iter().enumerate() {
+ let reference = self
+ .options
+ .reference_command
+ .as_ref()
+ .map(|cmd| Command::new(None, &cmd));
+ for (number, cmd) in reference.iter().chain(self.commands.iter()).enumerate() {
  self.results
  .push(Benchmark::new(number, cmd, self.options, &*executor).run()?);
 
@@ -62,24 +67,36 @@ impl<'a> Scheduler<'a> {
  return;
  }
 
- if let Some(mut annotated_results) = relative_speed::compute(&self.results) {
- annotated_results.sort_by(|l, r| relative_speed::compare_mean_time(l.result, r.result));
+ let reference = self
+ .options
+ .reference_command
+ .as_ref()
+ .map(|_| &self.results[0])
+ .unwrap_or_else(|| relative_speed::fastest(&self.results));
+
+ if let Some(mut annotated_results) = relative_speed::compute(reference, &self.results) {
+ if self.options.reference_command.is_none() {
+ annotated_results
+ .sort_by(|l, r| relative_speed::compare_mean_time(l.result, r.result));
+ }
 
- let fastest = &annotated_results[0];
+ let reference = &annotated_results[0];
  let others = &annotated_results[1..];
 
  println!("{}", "Summary".bold());
- println!(" '{}' ran", fastest.result.command.cyan());
+ println!(" '{}' ran", reference.result.command.cyan());
 
  for item in others {
+ let comparator = if item.is_faster { "faster" } else { "slower" };
  println!(
- "{}{} times faster than '{}'",
+ "{}{} times {} than '{}'",
  format!("{:8.2}", item.relative_speed).bold().green(),
  if let Some(stddev) = item.relative_speed_stddev {
  format!(" ± {}", format!("{:.2}", stddev).green())
  } else {
  "".into()
  },
+ comparator,
  &item.result.command.magenta()
  );
  }

src/cli.rs

@@ -68,6 +68,15 @@ fn build_command() -> Command<'static> {
  .help("Perform exactly NUM runs for each command. If this option is not specified, \
  hyperfine automatically determines the number of runs."),
  )
+ .arg(
+ Arg::new("reference")
+ .long("reference")
+ .short('R')
+ .takes_value(true)
+ .number_of_values(1)
+ .value_name("CMD")
+ .help("The reference command to measure the results against."),
+ )
  .arg(
  Arg::new("setup")
  .long("setup")

src/export/markup.rs

@@ -52,7 +52,7 @@ pub trait MarkupExporter {
  let min_str = format_duration_value(measurement.min, Some(unit)).0;
  let max_str = format_duration_value(measurement.max, Some(unit)).0;
  let rel_str = format!("{:.2}", entry.relative_speed);
- let rel_stddev_str = if entry.is_fastest {
+ let rel_stddev_str = if entry.is_reference {
  "".into()
  } else if let Some(stddev) = entry.relative_speed_stddev {
  format!(" ± {:.2}", stddev)
@@ -104,7 +104,7 @@ fn determine_unit_from_results(results: &[BenchmarkResult]) -> Unit {
 impl<T: MarkupExporter> Exporter for T {
  fn serialize(&self, results: &[BenchmarkResult], unit: Option<Unit>) -> Result<Vec<u8>> {
  let unit = unit.unwrap_or_else(|| determine_unit_from_results(results));
- let entries = relative_speed::compute(results);
+ let entries = relative_speed::compute(relative_speed::fastest(results), results);
  if entries.is_none() {
  return Err(anyhow!(
  "Relative speed comparison is not available for markup exporter."

src/options.rs

@@ -178,6 +178,9 @@ pub struct Options {
  /// Whether or not to ignore non-zero exit codes
  pub command_failure_action: CmdFailureAction,
 
+ /// Command to run before each *batch* of timing runs, i.e. before each individual benchmark
+ pub reference_command: Option<String>,
+
  /// Command(s) to run before each timing run
  pub preparation_command: Option<Vec<String>>,
 
@@ -207,6 +210,7 @@ impl Default for Options {
  warmup_count: 0,
  min_benchmarking_time: 3.0,
  command_failure_action: CmdFailureAction::RaiseError,
+ reference_command: None,
  preparation_command: None,
  setup_command: None,
  cleanup_command: None,
@@ -260,6 +264,8 @@ impl Options {
  (None, None) => {}
  };
 
+ options.reference_command = matches.value_of("reference").map(String::from);
+
  options.setup_command = matches.value_of("setup").map(String::from);
 
  options.preparation_command = matches