diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d7c7e1dae039f..5932afcf1acb0 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -414,6 +414,8 @@ jobs: runs-on: macos-13 - name: test default formula (macOS 14 arm64) runs-on: macos-14 + env: + HOMEBREW_TEST_BOT_ANALYTICS: 1 steps: - name: Set up Homebrew id: set-up-homebrew diff --git a/Library/Homebrew/brew.rb b/Library/Homebrew/brew.rb index fd877a3610c76..b30c87bae1894 100644 --- a/Library/Homebrew/brew.rb +++ b/Library/Homebrew/brew.rb @@ -84,9 +84,12 @@ end if internal_cmd || Commands.external_ruby_v2_cmd_path(cmd) - cmd_class = Homebrew::AbstractCommand.command(T.must(cmd)) + cmd = T.must(cmd) + cmd_class = Homebrew::AbstractCommand.command(cmd) if cmd_class - cmd_class.new.run + command_instance = cmd_class.new + Utils::Analytics.report_command_run(command_instance) + command_instance.run else Homebrew.public_send Commands.method_name(cmd) end diff --git a/Library/Homebrew/test/utils/analytics_spec.rb b/Library/Homebrew/test/utils/analytics_spec.rb index 5caaf2fa704f3..4428b1423acab 100644 --- a/Library/Homebrew/test/utils/analytics_spec.rb +++ b/Library/Homebrew/test/utils/analytics_spec.rb @@ -144,6 +144,45 @@ end end + describe "::report_command_run" do + let(:command) { "audit" } + let(:options) { "--tap=" } + let(:command_instance) do + require "dev-cmd/audit" + Homebrew::DevCmd::Audit.new(["--tap=homebrew/core"]) + end + + it "passes to the influxdb method" do + ENV.delete("HOMEBREW_NO_ANALYTICS_THIS_RUN") + ENV.delete("HOMEBREW_NO_ANALYTICS") + ENV["HOMEBREW_ANALYTICS_DEBUG"] = "true" + expect(described_class).to receive(:report_influx).with( + :command_run, + hash_including(command:), + hash_including(options:), + ).once + described_class.report_command_run(command_instance) + end + end + + describe "::report_test_bot_test" do + let(:command) { "install wget" } + let(:passed) { true } + + it "passes to the influxdb method" do + ENV.delete("HOMEBREW_NO_ANALYTICS_THIS_RUN") + ENV.delete("HOMEBREW_NO_ANALYTICS") + ENV["HOMEBREW_ANALYTICS_DEBUG"] = "true" + ENV["HOMEBREW_TEST_BOT_ANALYTICS"] = "true" + expect(described_class).to receive(:report_influx).with( + :test_bot_test, + hash_including(passed:), + hash_including(command:), + ).once + described_class.report_test_bot_test(command, passed) + end + end + specify "::table_output" do results = { ack: 10, wget: 100 } expect { described_class.table_output("install", "30", results) } diff --git a/Library/Homebrew/utils/analytics.rb b/Library/Homebrew/utils/analytics.rb index 51bdebaa455f9..fe1770c0a9df5 100644 --- a/Library/Homebrew/utils/analytics.rb +++ b/Library/Homebrew/utils/analytics.rb @@ -99,10 +99,62 @@ def report_build_error(exception) return unless tap return unless tap.should_report_analytics? - options = exception.options.to_a.map(&:to_s).join(" ") + options = exception.options.to_a.compact.map(&:to_s).sort.uniq.join(" ") report_package_event(:build_error, package_name: formula.name, tap_name: tap.name, options:) end + sig { params(command_instance: Homebrew::AbstractCommand).void } + def report_command_run(command_instance) + return if not_this_run? || disabled? + + command = command_instance.class.command_name + + options_array = command_instance.args.options_only.to_a.compact + + # Strip out any flag values to reduce cardinality and preserve privacy. + options_array.map! { |option| option.sub(/=.*/, "=") } + options = options_array.sort.uniq.join(" ") + + # Tags must have low cardinality. + tags = { + command:, + ci: ENV["CI"].present?, + devcmdrun: config_true?(:devcmdrun), + developer: Homebrew::EnvConfig.developer?, + } + + # Fields can have high cardinality. + fields = { options: } + + report_influx(:command_run, tags, fields) + end + + sig { params(step_command_short: String, passed: T::Boolean).void } + def report_test_bot_test(step_command_short, passed) + return if not_this_run? || disabled? + return if ENV["HOMEBREW_TEST_BOT_ANALYTICS"].blank? + + # ensure passed is a boolean + passed = passed ? true : false + + # Tags must have low cardinality. + tags = { + passed:, + arch: HOMEBREW_PHYSICAL_PROCESSOR, + os: HOMEBREW_SYSTEM, + } + + # Strip out any flag values to reduce cardinality and preserve privacy. + command = step_command_short.split + .map { |arg| arg.sub(/=.*/, "=") } + .join(" ") + + # Fields can have high cardinality. + fields = { command: } + + report_influx(:test_bot_test, tags, fields) + end + def influx_message_displayed? config_true?(:influxanalyticsmessage) end diff --git a/docs/Analytics.md b/docs/Analytics.md index 49d5729bd3429..d233072fb50c2 100644 --- a/docs/Analytics.md +++ b/docs/Analytics.md @@ -8,6 +8,7 @@ Homebrew is provided free of charge and run entirely by volunteers in their spar - If a formula is widely used and is failing often it will enable us to prioritise fixing that formula over others. - Collecting the OS version allows us to decide which versions of macOS to prioritise for support and identify build failures that occur only on single versions. +- If a command is not widely used, it can be deprecated. ## How Long? @@ -15,7 +16,7 @@ Homebrew's anonymous analytics has a 365 day retention period in InfluxDB. ## What? -Homebrew's analytics record some shared information for every event: +Homebrew's analytics record some shared information for every formula or cask event: - Whether the data is being sent from CI, e.g. `true` if the CI environment variable is set. - Whether you are using the default install prefix (e.g. `/opt/homebrew`) or a custom one (e.g. `/home/mike/.brew`). If your prefix is custom, it will be sent as `custom-prefix` to preserve anonymity. @@ -33,6 +34,8 @@ Homebrew's analytics records the following different events: - The `install_on_request` event category and the Homebrew formula from a non-private GitHub tap you have requested to install (e.g. when explicitly named with a `brew install`) plus options. This allows us to differentiate the formulae that users intend to install from those pulled in as dependencies. - The `cask_install` event category and the Homebrew cask from a non-private GitHub tap you install as the action. This allows us to identify which casks where work should be prioritised, as well as how to handle possible deprecation or removal of any. - The `build_error` event category and the Homebrew formula plus options that failed to install as the action, e.g. `wget --HEAD`. This allows us to identify formulae that may need fixing. The details or logs of the build error are not sent. +- The `command_run` event category and the command and flags you run as the action. This allows us to identify which commands and flags are most commonly used and which are not used at all. +- The `test_bot_test` event category is only used in Homebrew's CI environment and is used to record the results of running tests on pull requests. You can also view all the information that is sent by Homebrew's analytics by setting `HOMEBREW_ANALYTICS_DEBUG=1` in your environment. Please note this will also stop any analytics from being sent.