analytics: support command and test-bot analytics.

These are used to analyse which commands are used and the
success/failure rate of official taps using `brew test-bot`.

.github/workflows/tests.yml

@@ -414,6 +414,8 @@ jobs:
  runs-on: macos-13
  - name: test default formula (macOS 14 arm64)
  runs-on: macos-14
+ env:
+ HOMEBREW_TEST_BOT_ANALYTICS: 1
  steps:
  - name: Set up Homebrew
  id: set-up-homebrew

Library/Homebrew/brew.rb

@@ -84,9 +84,12 @@
  end
 
  if internal_cmd || Commands.external_ruby_v2_cmd_path(cmd)
- cmd_class = Homebrew::AbstractCommand.command(T.must(cmd))
+ cmd = T.must(cmd)
+ cmd_class = Homebrew::AbstractCommand.command(cmd)
  if cmd_class
- cmd_class.new.run
+ command_instance = cmd_class.new
+ Utils::Analytics.report_command_run(command_instance)
+ command_instance.run
  else
  Homebrew.public_send Commands.method_name(cmd)
  end

Library/Homebrew/test/utils/analytics_spec.rb

@@ -144,6 +144,45 @@
  end
  end
 
+ describe "::report_command_run" do
+ let(:command) { "audit" }
+ let(:options) { "--tap=" }
+ let(:command_instance) do
+ require "dev-cmd/audit"
+ Homebrew::DevCmd::Audit.new(["--tap=homebrew/core"])
+ end
+
+ it "passes to the influxdb method" do
+ ENV.delete("HOMEBREW_NO_ANALYTICS_THIS_RUN")
+ ENV.delete("HOMEBREW_NO_ANALYTICS")
+ ENV["HOMEBREW_ANALYTICS_DEBUG"] = "true"
+ expect(described_class).to receive(:report_influx).with(
+ :command_run,
+ hash_including(command:),
+ hash_including(options:),
+ ).once
+ described_class.report_command_run(command_instance)
+ end
+ end
+
+ describe "::report_test_bot_test" do
+ let(:command) { "install wget" }
+ let(:passed) { true }
+
+ it "passes to the influxdb method" do
+ ENV.delete("HOMEBREW_NO_ANALYTICS_THIS_RUN")
+ ENV.delete("HOMEBREW_NO_ANALYTICS")
+ ENV["HOMEBREW_ANALYTICS_DEBUG"] = "true"
+ ENV["HOMEBREW_TEST_BOT_ANALYTICS"] = "true"
+ expect(described_class).to receive(:report_influx).with(
+ :test_bot_test,
+ hash_including(passed:),
+ hash_including(command:),
+ ).once
+ described_class.report_test_bot_test(command, passed)
+ end
+ end
+
  specify "::table_output" do
  results = { ack: 10, wget: 100 }
  expect { described_class.table_output("install", "30", results) }

Library/Homebrew/utils/analytics.rb

@@ -99,10 +99,62 @@ def report_build_error(exception)
  return unless tap
  return unless tap.should_report_analytics?
 
- options = exception.options.to_a.map(&:to_s).join(" ")
+ options = exception.options.to_a.compact.map(&:to_s).sort.uniq.join(" ")
  report_package_event(:build_error, package_name: formula.name, tap_name: tap.name, options:)
  end
 
+ sig { params(command_instance: Homebrew::AbstractCommand).void }
+ def report_command_run(command_instance)
+ return if not_this_run? || disabled?
+
+ command = command_instance.class.command_name
+
+ options_array = command_instance.args.options_only.to_a.compact
+
+ # Strip out any flag values to reduce cardinality and preserve privacy.
+ options_array.map! { |option| option.sub(/=.*/, "=") }
+ options = options_array.sort.uniq.join(" ")
+
+ # Tags must have low cardinality.
+ tags = {
+ command:,
+ ci: ENV["CI"].present?,
+ devcmdrun: config_true?(:devcmdrun),
+ developer: Homebrew::EnvConfig.developer?,
+ }
+
+ # Fields can have high cardinality.
+ fields = { options: }
+
+ report_influx(:command_run, tags, fields)
+ end
+
+ sig { params(step_command_short: String, passed: T::Boolean).void }
+ def report_test_bot_test(step_command_short, passed)
+ return if not_this_run? || disabled?
+ return if ENV["HOMEBREW_TEST_BOT_ANALYTICS"].blank?
+
+ # ensure passed is a boolean
+ passed = passed ? true : false
+
+ # Tags must have low cardinality.
+ tags = {
+ passed:,
+ arch: HOMEBREW_PHYSICAL_PROCESSOR,
+ os: HOMEBREW_SYSTEM,
+ }
+
+ # Strip out any flag values to reduce cardinality and preserve privacy.
+ command = step_command_short.split
+ .map { |arg| arg.sub(/=.*/, "=") }
+ .join(" ")
+
+ # Fields can have high cardinality.
+ fields = { command: }
+
+ report_influx(:test_bot_test, tags, fields)
+ end
+
  def influx_message_displayed?
  config_true?(:influxanalyticsmessage)
  end

docs/Analytics.md

@@ -8,14 +8,15 @@ Homebrew is provided free of charge and run entirely by volunteers in their spar
 
 - If a formula is widely used and is failing often it will enable us to prioritise fixing that formula over others.
 - Collecting the OS version allows us to decide which versions of macOS to prioritise for support and identify build failures that occur only on single versions.
+- If a command is not widely used, it can be deprecated.
 
 ## How Long?
 
 Homebrew's anonymous analytics has a 365 day retention period in InfluxDB.
 
 ## What?
 
-Homebrew's analytics record some shared information for every event:
+Homebrew's analytics record some shared information for every formula or cask event:
 
 - Whether the data is being sent from CI, e.g. `true` if the CI environment variable is set.
 - Whether you are using the default install prefix (e.g. `/opt/homebrew`) or a custom one (e.g. `/home/mike/.brew`). If your prefix is custom, it will be sent as `custom-prefix` to preserve anonymity.
@@ -33,6 +34,8 @@ Homebrew's analytics records the following different events:
 - The `install_on_request` event category and the Homebrew formula from a non-private GitHub tap you have requested to install (e.g. when explicitly named with a `brew install`) plus options. This allows us to differentiate the formulae that users intend to install from those pulled in as dependencies.
 - The `cask_install` event category and the Homebrew cask from a non-private GitHub tap you install as the action. This allows us to identify which casks where work should be prioritised, as well as how to handle possible deprecation or removal of any.
 - The `build_error` event category and the Homebrew formula plus options that failed to install as the action, e.g. `wget --HEAD`. This allows us to identify formulae that may need fixing. The details or logs of the build error are not sent.
+- The `command_run` event category and the command and flags you run as the action. This allows us to identify which commands and flags are most commonly used and which are not used at all.
+- The `test_bot_test` event category is only used in Homebrew's CI environment and is used to record the results of running tests on pull requests.
 
 You can also view all the information that is sent by Homebrew's analytics by setting `HOMEBREW_ANALYTICS_DEBUG=1` in your environment. Please note this will also stop any analytics from being sent.