Refactor internal Query schema and introduce WhereBuilder

extra/lib/plausible/stats/goal/revenue.ex

@@ -4,6 +4,8 @@ defmodule Plausible.Stats.Goal.Revenue do
  """
  import Ecto.Query
 
+ alias Plausible.Stats.Query
+
  @revenue_metrics [:average_revenue, :total_revenue]
 
  def revenue_metrics() do
@@ -37,10 +39,10 @@ defmodule Plausible.Stats.Goal.Revenue do
  """
  def get_revenue_tracking_currency(site, query, metrics) do
  goal_filters =
- case query.filters do
- %{"event:goal" => {:is, {_, goal_name}}} -> [goal_name]
- %{"event:goal" => {:member, list}} -> Enum.map(list, fn {_, goal_name} -> goal_name end)
- _any -> []
+ case Query.get_filter(query, "event:goal") do
+ [:is, "event:goal", {_, goal_name}] -> [goal_name]
+ [:member, "event:goal", list] -> Enum.map(list, fn {_, goal_name} -> goal_name end)
+ _ -> []
  end
 
  requested_revenue_metrics? = Enum.any?(metrics, &(&1 in @revenue_metrics))

lib/plausible/google/api.ex

@@ -74,16 +74,16 @@ defmodule Plausible.Google.API do
  end
  end
 
- def fetch_stats(site, %{filters: %{} = filters, date_range: date_range}, limit) do
+ def fetch_stats(site, query, limit) do
  with {:ok, site} <- ensure_search_console_property(site),
  {:ok, access_token} <- maybe_refresh_token(site.google_auth),
  {:ok, search_console_filters} <-
- SearchConsole.Filters.transform(site.google_auth.property, filters),
+ SearchConsole.Filters.transform(site.google_auth.property, query.filters),
  {:ok, stats} <-
  HTTP.list_stats(
  access_token,
  site.google_auth.property,
- date_range,
+ query.date_range,
  limit,
  search_console_filters
  ) do

lib/plausible/google/search_console/filters.ex

@@ -3,12 +3,11 @@ defmodule Plausible.Google.SearchConsole.Filters do
  import Plausible.Stats.Base, only: [page_regex: 1]
 
  def transform(property, plausible_filters) do
- plausible_filters = Map.drop(plausible_filters, ["visit:source"])
-
  search_console_filters =
  Enum.reduce_while(plausible_filters, [], fn plausible_filter, search_console_filters ->
  case transform_filter(property, plausible_filter) do
  :unsupported -> {:halt, :unsupported_filters}
+ :ignore -> {:cont, search_console_filters}
  search_console_filter -> {:cont, [search_console_filter | search_console_filters]}
  end
  end)
@@ -20,53 +19,55 @@ defmodule Plausible.Google.SearchConsole.Filters do
  end
  end
 
- defp transform_filter(property, {"event:page", filter}) do
- transform_filter(property, {"visit:entry_page", filter})
+ defp transform_filter(property, [op, "event:page" | rest]) do
+ transform_filter(property, [op, "visit:entry_page" | rest])
  end
 
- defp transform_filter(property, {"visit:entry_page", {:is, page}}) when is_binary(page) do
+ defp transform_filter(property, [:is, "visit:entry_page", page]) when is_binary(page) do
  %{dimension: "page", expression: property_url(property, page)}
  end
 
- defp transform_filter(property, {"visit:entry_page", {:member, pages}}) when is_list(pages) do
+ defp transform_filter(property, [:member, "visit:entry_page", pages]) when is_list(pages) do
  expression =
  Enum.map_join(pages, "|", fn page -> property_url(property, Regex.escape(page)) end)
 
  %{dimension: "page", operator: "includingRegex", expression: expression}
  end
 
- defp transform_filter(property, {"visit:entry_page", {:matches, page}}) when is_binary(page) do
+ defp transform_filter(property, [:matches, "visit:entry_page", page]) when is_binary(page) do
  page = page_regex(property_url(property, page))
  %{dimension: "page", operator: "includingRegex", expression: page}
  end
 
- defp transform_filter(property, {"visit:entry_page", {:matches_member, pages}})
+ defp transform_filter(property, [:matches_member, "visit:entry_page", pages])
  when is_list(pages) do
  expression =
  Enum.map_join(pages, "|", fn page -> page_regex(property_url(property, page)) end)
 
  %{dimension: "page", operator: "includingRegex", expression: expression}
  end
 
- defp transform_filter(_property, {"visit:screen", {:is, device}}) when is_binary(device) do
+ defp transform_filter(_property, [:is, "visit:screen", device]) when is_binary(device) do
  %{dimension: "device", expression: search_console_device(device)}
  end
 
- defp transform_filter(_property, {"visit:screen", {:member, devices}}) when is_list(devices) do
+ defp transform_filter(_property, [:member, "visit:screen", devices]) when is_list(devices) do
  expression = devices |> Enum.join("|")
  %{dimension: "device", operator: "includingRegex", expression: expression}
  end
 
- defp transform_filter(_property, {"visit:country", {:is, country}}) when is_binary(country) do
+ defp transform_filter(_property, [:is, "visit:country", country]) when is_binary(country) do
  %{dimension: "country", expression: search_console_country(country)}
  end
 
- defp transform_filter(_property, {"visit:country", {:member, countries}})
+ defp transform_filter(_property, [:member, "visit:country", countries])
  when is_list(countries) do
  expression = Enum.map_join(countries, "|", &search_console_country/1)
  %{dimension: "country", operator: "includingRegex", expression: expression}
  end
 
+ defp transform_filter(_, [_, "visit:source" | _rest]), do: :ignore
+
  defp transform_filter(_, _filter), do: :unsupported
 
  defp property_url("sc-domain:" <> domain, page), do: "https://" <> domain <> page

lib/plausible/stats/aggregate.ex

@@ -70,10 +70,7 @@ defmodule Plausible.Stats.Aggregate do
  defp neighbor_aggregate_time_on_page(site, query) do
  q =
  from(
- e in base_event_query(site, %Query{
- query
- | filters: Map.delete(query.filters, "event:page")
- }),
+ e in base_event_query(site, Query.remove_filters(query, ["event:page"])),
  select: {
  fragment("? as p", e.pathname),
  fragment("? as t", e.timestamp),
@@ -86,32 +83,32 @@ defmodule Plausible.Stats.Aggregate do
  where_param_idx = length(base_query_raw_params)
 
  {where_clause, where_arg} =
- case query.filters["event:page"] do
- {:is, page} ->
+ case Query.get_filter(query, "event:page") do
+ [:is, _, page] ->
  {"p = {$#{where_param_idx}:String}", page}
 
- {:is_not, page} ->
+ [:is_not, _, page] ->
  {"p != {$#{where_param_idx}:String}", page}
 
- {:member, page} ->
+ [:member, _, page] ->
  {"p IN {$#{where_param_idx}:Array(String)}", page}
 
- {:not_member, page} ->
+ [:not_member, _, page] ->
  {"p NOT IN {$#{where_param_idx}:Array(String)}", page}
 
- {:matches, expr} ->
+ [:matches, _, expr] ->
  regex = page_regex(expr)
  {"match(p, {$#{where_param_idx}:String})", regex}
 
- {:matches_member, exprs} ->
+ [:matches_member, _, exprs] ->
  page_regexes = Enum.map(exprs, &page_regex/1)
  {"multiMatchAny(p, {$#{where_param_idx}:Array(String)})", page_regexes}
 
- {:not_matches_member, exprs} ->
+ [:not_matches_member, _, exprs] ->
  page_regexes = Enum.map(exprs, &page_regex/1)
  {"not(multiMatchAny(p, {$#{where_param_idx}:Array(String)}))", page_regexes}
 
- {:does_not_match, expr} ->
+ [:does_not_match, _, expr] ->
  regex = page_regex(expr)
  {"not(match(p, {$#{where_param_idx}:String}))", regex}
  end
@@ -148,29 +145,29 @@ defmodule Plausible.Stats.Aggregate do
 
  defp window_aggregate_time_on_page(site, query) do
  windowed_pages_q =
- from e in base_event_query(site, %Query{
- query
- | filters: Map.delete(query.filters, "event:page")
- }),
- select: %{
- next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),
- next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),
- timestamp: e.timestamp,
- pathname: e.pathname,
- session_id: e.session_id
- },
- windows: [
- event_horizon: [
- partition_by: e.session_id,
- order_by: e.timestamp,
- frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")
- ]
- ]
+ from e in base_event_query(site, Query.remove_filters(query, ["event:page"])),
+ select: %{
+ next_timestamp: over(fragment("leadInFrame(?)", e.timestamp), :event_horizon),
+ next_pathname: over(fragment("leadInFrame(?)", e.pathname), :event_horizon),
+ timestamp: e.timestamp,
+ pathname: e.pathname,
+ session_id: e.session_id
+ },
+ windows: [
+ event_horizon: [
+ partition_by: e.session_id,
+ order_by: e.timestamp,
+ frame: fragment("ROWS BETWEEN CURRENT ROW AND 1 FOLLOWING")
+ ]
+ ]
+
+ event_page_filter = Query.get_filter(query, "event:page")
 
  timed_page_transitions_q =
  from e in Ecto.Query.subquery(windowed_pages_q),
  group_by: [e.pathname, e.next_pathname, e.session_id],
- where: ^Plausible.Stats.Base.dynamic_filter_condition(query, "event:page", :pathname),
+ where:
+ ^Plausible.Stats.Filters.WhereBuilder.build_condition(:pathname, event_page_filter),
  where: e.next_timestamp != 0,
  select: %{
  pathname: e.pathname,