From f4dabd938457ca2de640451d14d1946e3bf7d168 Mon Sep 17 00:00:00 2001 From: Mike McQuaid Date: Thu, 25 Apr 2024 17:29:06 +0100 Subject: [PATCH] analytics: support command and test-bot analytics. These are used to analyse which commands are used and the success/failure rate of official taps using `brew test-bot`. --- .github/workflows/tests.yml | 2 + Library/Homebrew/brew.rb | 7 ++- Library/Homebrew/test/utils/analytics_spec.rb | 39 +++++++++++++++ Library/Homebrew/utils/analytics.rb | 49 ++++++++++++++++++- docs/Analytics.md | 5 +- 5 files changed, 98 insertions(+), 4 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d7c7e1dae039fb..5932afcf1acb04 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -414,6 +414,8 @@ jobs: runs-on: macos-13 - name: test default formula (macOS 14 arm64) runs-on: macos-14 + env: + HOMEBREW_TEST_BOT_ANALYTICS: 1 steps: - name: Set up Homebrew id: set-up-homebrew diff --git a/Library/Homebrew/brew.rb b/Library/Homebrew/brew.rb index fd877a3610c767..b30c87bae1894d 100644 --- a/Library/Homebrew/brew.rb +++ b/Library/Homebrew/brew.rb @@ -84,9 +84,12 @@ end if internal_cmd || Commands.external_ruby_v2_cmd_path(cmd) - cmd_class = Homebrew::AbstractCommand.command(T.must(cmd)) + cmd = T.must(cmd) + cmd_class = Homebrew::AbstractCommand.command(cmd) if cmd_class - cmd_class.new.run + command_instance = cmd_class.new + Utils::Analytics.report_command_run(command_instance) + command_instance.run else Homebrew.public_send Commands.method_name(cmd) end diff --git a/Library/Homebrew/test/utils/analytics_spec.rb b/Library/Homebrew/test/utils/analytics_spec.rb index 5caaf2fa704f33..4428b1423acab5 100644 --- a/Library/Homebrew/test/utils/analytics_spec.rb +++ b/Library/Homebrew/test/utils/analytics_spec.rb @@ -144,6 +144,45 @@ end end + describe "::report_command_run" do + let(:command) { "audit" } + let(:options) { "--tap=" } + let(:command_instance) do + require "dev-cmd/audit" + Homebrew::DevCmd::Audit.new(["--tap=homebrew/core"]) + end + + it "passes to the influxdb method" do + ENV.delete("HOMEBREW_NO_ANALYTICS_THIS_RUN") + ENV.delete("HOMEBREW_NO_ANALYTICS") + ENV["HOMEBREW_ANALYTICS_DEBUG"] = "true" + expect(described_class).to receive(:report_influx).with( + :command_run, + hash_including(command:), + hash_including(options:), + ).once + described_class.report_command_run(command_instance) + end + end + + describe "::report_test_bot_test" do + let(:command) { "install wget" } + let(:passed) { true } + + it "passes to the influxdb method" do + ENV.delete("HOMEBREW_NO_ANALYTICS_THIS_RUN") + ENV.delete("HOMEBREW_NO_ANALYTICS") + ENV["HOMEBREW_ANALYTICS_DEBUG"] = "true" + ENV["HOMEBREW_TEST_BOT_ANALYTICS"] = "true" + expect(described_class).to receive(:report_influx).with( + :test_bot_test, + hash_including(passed:), + hash_including(command:), + ).once + described_class.report_test_bot_test(command, passed) + end + end + specify "::table_output" do results = { ack: 10, wget: 100 } expect { described_class.table_output("install", "30", results) } diff --git a/Library/Homebrew/utils/analytics.rb b/Library/Homebrew/utils/analytics.rb index 51bdebaa455f96..17a3958d9a618b 100644 --- a/Library/Homebrew/utils/analytics.rb +++ b/Library/Homebrew/utils/analytics.rb @@ -99,10 +99,57 @@ def report_build_error(exception) return unless tap return unless tap.should_report_analytics? - options = exception.options.to_a.map(&:to_s).join(" ") + options = exception.options.to_a.compact.map(&:to_s).sort.uniq.join(" ") report_package_event(:build_error, package_name: formula.name, tap_name: tap.name, options:) end + sig { params(command_instance: Homebrew::AbstractCommand).void } + def report_command_run(command_instance) + return if not_this_run? || disabled? + + command = command_instance.class.command_name + + options_array = command_instance.args.options_only.to_a.compact + + # Strip out any flag values to reduce cardinality. + options_array.map! { |option| option.sub(/=.*/, "=") } + options = options_array.sort.uniq.join(" ") + + # Tags must have low cardinality. + tags = { + command:, + ci: ENV["CI"].present?, + devcmdrun: config_true?(:devcmdrun), + developer: Homebrew::EnvConfig.developer?, + } + + # Fields can have high cardinality. + fields = { options: } + + report_influx(:command_run, tags, fields) + end + + sig { params(command: String, passed: T::Boolean).void } + def report_test_bot_test(command, passed) + return if not_this_run? || disabled? + return if ENV["HOMEBREW_TEST_BOT_ANALYTICS"].blank? + + # ensure passed is a boolean + passed = passed ? true : false + + # Tags must have low cardinality. + tags = { + passed:, + arch: HOMEBREW_PHYSICAL_PROCESSOR, + os: HOMEBREW_SYSTEM, + } + + # Fields can have high cardinality. + fields = { command: } + + report_influx(:test_bot_test, tags, fields) + end + def influx_message_displayed? config_true?(:influxanalyticsmessage) end diff --git a/docs/Analytics.md b/docs/Analytics.md index 49d5729bd34294..d233072fb50c28 100644 --- a/docs/Analytics.md +++ b/docs/Analytics.md @@ -8,6 +8,7 @@ Homebrew is provided free of charge and run entirely by volunteers in their spar - If a formula is widely used and is failing often it will enable us to prioritise fixing that formula over others. - Collecting the OS version allows us to decide which versions of macOS to prioritise for support and identify build failures that occur only on single versions. +- If a command is not widely used, it can be deprecated. ## How Long? @@ -15,7 +16,7 @@ Homebrew's anonymous analytics has a 365 day retention period in InfluxDB. ## What? -Homebrew's analytics record some shared information for every event: +Homebrew's analytics record some shared information for every formula or cask event: - Whether the data is being sent from CI, e.g. `true` if the CI environment variable is set. - Whether you are using the default install prefix (e.g. `/opt/homebrew`) or a custom one (e.g. `/home/mike/.brew`). If your prefix is custom, it will be sent as `custom-prefix` to preserve anonymity. @@ -33,6 +34,8 @@ Homebrew's analytics records the following different events: - The `install_on_request` event category and the Homebrew formula from a non-private GitHub tap you have requested to install (e.g. when explicitly named with a `brew install`) plus options. This allows us to differentiate the formulae that users intend to install from those pulled in as dependencies. - The `cask_install` event category and the Homebrew cask from a non-private GitHub tap you install as the action. This allows us to identify which casks where work should be prioritised, as well as how to handle possible deprecation or removal of any. - The `build_error` event category and the Homebrew formula plus options that failed to install as the action, e.g. `wget --HEAD`. This allows us to identify formulae that may need fixing. The details or logs of the build error are not sent. +- The `command_run` event category and the command and flags you run as the action. This allows us to identify which commands and flags are most commonly used and which are not used at all. +- The `test_bot_test` event category is only used in Homebrew's CI environment and is used to record the results of running tests on pull requests. You can also view all the information that is sent by Homebrew's analytics by setting `HOMEBREW_ANALYTICS_DEBUG=1` in your environment. Please note this will also stop any analytics from being sent.