Export and import custom events via CSV (#4096)

* Export and import custom events via CSV

* Add prop support of url for cloaked links and path for 404s in imported queries

* Handle custom events with empty URL and path properties gracefully

* Make events with properties logic DRY and fix missed cloaked link

* Add test for path property breakdown

* Update raw CH data fixture and extend CSV importer tests

* Fix broken query condition after rebase

* Update CHANGELOG.md

CHANGELOG.md

@@ -45,6 +45,7 @@ All notable changes to this project will be documented in this file.
 - Import custom events from Google Analytics 4
 - Ability to filter Search Console keywords by page, country and device plausible/analytics#4077
 - Add `DATA_DIR` env var for exports/imports plausible/analytics#4100
+- Add custom events support to CSV export and import
 
 ### Removed
 - Removed the nested custom event property breakdown UI when filtering by a goal in Goal Conversions

lib/plausible/exports.ex

@@ -4,6 +4,7 @@ defmodule Plausible.Exports do
  """
 
  use Plausible
+ use Plausible.Stats.Fragments
  import Ecto.Query
 
  @doc "Schedules CSV export job to S3 storage"
@@ -234,6 +235,8 @@ defmodule Plausible.Exports do
  filename.("imported_pages") => export_pages_q(site_id, timezone, date_range),
  filename.("imported_entry_pages") => export_entry_pages_q(site_id, timezone, date_range),
  filename.("imported_exit_pages") => export_exit_pages_q(site_id, timezone, date_range),
+ filename.("imported_custom_events") =>
+ export_custom_events_q(site_id, timezone, date_range),
  filename.("imported_locations") => export_locations_q(site_id, timezone, date_range),
  filename.("imported_devices") => export_devices_q(site_id, timezone, date_range),
  filename.("imported_browsers") => export_browsers_q(site_id, timezone, date_range),
@@ -465,6 +468,43 @@ defmodule Plausible.Exports do
  ]
  end
 
+ defp export_custom_events_q(site_id, timezone, date_range) do
+ from e in sampled("events_v2"),
+ where: ^export_filter(site_id, date_range),
+ where: e.name != "pageview",
+ group_by: [
+ selected_as(:date),
+ e.name,
+ selected_as(:link_url),
+ selected_as(:path)
+ ],
+ order_by: selected_as(:date),
+ select: [
+ date(e.timestamp, ^timezone),
+ e.name,
+ selected_as(
+ fragment(
+ "if(? in ?, ?, '')",
+ e.name,
+ ^Plausible.Imported.goals_with_url(),
+ get_by_key(e, :meta, "url")
+ ),
+ :link_url
+ ),
+ selected_as(
+ fragment(
+ "if(? in ?, ?, '')",
+ e.name,
+ ^Plausible.Imported.goals_with_path(),
+ get_by_key(e, :meta, "path")
+ ),
+ :path
+ ),
+ visitors(e),
+ selected_as(fragment("toUInt64(round(count()*any(_sample_factor)))"), :events)
+ ]
+ end
+
  defp export_locations_q(site_id, timezone, date_range) do
  from s in sampled("sessions_v2"),
  where: ^export_filter(site_id, date_range),

lib/plausible/imported.ex

@@ -38,6 +38,11 @@ defmodule Plausible.Imported do
  # Maximum number of complete imports to account for when querying stats
  @max_complete_imports 5
 
+ # Goals which can be filtered by url property
+ @goals_with_url ["Outbound Link: Click", "Cloaked Link: Click", "File Download"]
+ # Goals which can be filtered by path property
+ @goals_with_path ["404"]
+
  @spec schemas() :: [module()]
  def schemas, do: @tables
 
@@ -49,6 +54,16 @@ defmodule Plausible.Imported do
  @max_complete_imports
  end
 
+ @spec goals_with_url() :: [String.t()]
+ def goals_with_url() do
+ @goals_with_url
+ end
+
+ @spec goals_with_path() :: [String.t()]
+ def goals_with_path() do
+ @goals_with_path
+ end
+
  @spec load_import_data(Site.t()) :: Site.t()
  def load_import_data(%{import_data_loaded: true} = site), do: site
 

lib/plausible/imported/csv_importer.ex

@@ -154,6 +154,8 @@ defmodule Plausible.Imported.CSVImporter do
  "date Date, entry_page String, visitors UInt64, entrances UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
  "imported_exit_pages" =>
  "date Date, exit_page String, visitors UInt64, visit_duration UInt64, exits UInt64, bounces UInt32, pageviews UInt64",
+ "imported_custom_events" =>
+ "date Date, name String, link_url String, path String, visitors UInt64, events UInt64",
  "imported_locations" =>
  "date Date, country String, region String, city UInt64, visitors UInt64, visits UInt64, visit_duration UInt64, bounces UInt32, pageviews UInt64",
  "imported_operating_systems" =>

lib/plausible/imported/custom_event.ex

@@ -9,6 +9,7 @@ defmodule Plausible.Imported.CustomEvent do
  field :date, :date
  field :name, :string
  field :link_url, :string
+ field :path, :string
  field :visitors, Ch, type: "UInt64"
  field :events, Ch, type: "UInt64"
  end

lib/plausible/stats/imported.ex

@@ -7,6 +7,7 @@ defmodule Plausible.Stats.Imported do
 
  @no_ref "Direct / None"
  @not_set "(not set)"
+ @none "(none)"
 
  @property_to_table_mappings %{
  "visit:source" => "imported_sources",
@@ -28,12 +29,14 @@ defmodule Plausible.Stats.Imported do
  "visit:os_version" => "imported_operating_systems",
  "event:page" => "imported_pages",
  "event:name" => "imported_custom_events",
- "event:props:url" => "imported_custom_events"
+ "event:props:url" => "imported_custom_events",
+ "event:props:path" => "imported_custom_events"
  }
 
  @imported_properties Map.keys(@property_to_table_mappings)
 
- @goals_with_url ["Outbound Link: Click", "Cloaked Link: Click", "File Download"]
+ @goals_with_url Plausible.Imported.goals_with_url()
+ @goals_with_path Plausible.Imported.goals_with_path()
 
  @doc """
  Returns a boolean indicating whether the combination of filters and
@@ -52,6 +55,7 @@ defmodule Plausible.Stats.Imported do
  {0, "event:props:" <> _} -> false
  {0, _} -> true
  {1, "event:props:url"} -> has_special_goal_filter?(query, @goals_with_url)
+ {1, "event:props:path"} -> has_special_goal_filter?(query, @goals_with_path)
  {_, _} -> false
  end
  end
@@ -126,7 +130,7 @@ defmodule Plausible.Stats.Imported do
 
  join_on =
  case dim do
- :url ->
+ _ when dim in [:url, :path] ->
  dynamic([s, i], s.breakdown_prop_value == i.breakdown_prop_value)
 
  :os_version ->
@@ -224,6 +228,14 @@ defmodule Plausible.Stats.Imported do
  end
  end
 
+ defp maybe_apply_filter(q, query, "event:props:path", _) do
+ if name = find_special_goal_filter(query, @goals_with_path) do
+ where(q, [i], i.name == ^name)
+ else
+ q
+ end
+ end
+
  defp maybe_apply_filter(q, query, property, dim) do
  case Query.get_filter(query, property) do
  [:member, _, list] -> where(q, [i], field(i, ^dim) in ^list)
@@ -469,7 +481,17 @@ defmodule Plausible.Stats.Imported do
  defp group_imported_by(q, :url) do
  q
  |> group_by([i], i.link_url)
- |> select_merge([i], %{breakdown_prop_value: i.link_url})
+ |> select_merge([i], %{
+ breakdown_prop_value: fragment("if(not empty(?), ?, ?)", i.link_url, i.link_url, @none)
+ })
+ end
+
+ defp group_imported_by(q, :path) do
+ q
+ |> group_by([i], i.path)
+ |> select_merge([i], %{
+ breakdown_prop_value: fragment("if(not empty(?), ?, ?)", i.path, i.path, @none)
+ })
  end
 
  defp select_joined_dimension(q, :city) do
@@ -493,7 +515,7 @@ defmodule Plausible.Stats.Imported do
  })
  end
 
- defp select_joined_dimension(q, :url) do
+ defp select_joined_dimension(q, dim) when dim in [:url, :path] do
  select_merge(q, [s, i], %{
  breakdown_prop_value:
  fragment(

test/plausible/exports_test.exs

@@ -12,6 +12,7 @@ defmodule Plausible.ExportsTest do
 
  assert Map.keys(queries) == [
  "imported_browsers.csv",
+ "imported_custom_events.csv",
  "imported_devices.csv",
  "imported_entry_pages.csv",
  "imported_exit_pages.csv",
@@ -31,6 +32,7 @@ defmodule Plausible.ExportsTest do
 
  assert Map.keys(queries) == [
  "imported_browsers_20230101_20240312.csv",
+ "imported_custom_events_20230101_20240312.csv",
  "imported_devices_20230101_20240312.csv",
  "imported_entry_pages_20230101_20240312.csv",
  "imported_exit_pages_20230101_20240312.csv",
@@ -50,6 +52,7 @@ defmodule Plausible.ExportsTest do
 
  assert Map.keys(queries) == [
  "imported_browsers.ch",
+ "imported_custom_events.ch",
  "imported_devices.ch",
  "imported_entry_pages.ch",
  "imported_exit_pages.ch",