Frontend: Use jsonurl, update filtering (#4117)

* Start refactoring, supporting jsonurl

* WIP: Listing of applied filters works

* WIP: Remove some unneeded hackery

* WIP: Handle country labels better

* WIP: Handle multiple filters better

* WIP: Serializing filters for API queries

* Split modal code into 3

* Merge code paths for prop and other filter modals

* Get suggestions working again

* Display prop filters properly

* Handle re-opening a props modal properly

* Better label handling

* Better linking to filter modals

* Remove unneeded component

* Standardize how we update query more

* Use updatedQuery to remove more usecases of URLSearchParams

* Dont export toFilterQuery

* Custom toString for PlausibleSearchParams

* Fix props suggestions/filtering

* Refactor isFilteringOnFixedValue

* Improved encoding - goals now work again

* fix a typo

* Handle more cases where query.filters[ is used

* Fix locations tab changing behavior

* Fix for `setQuery` not to double up ?

* Handle goal filters properly now

* Delete dead code

* Update special goals handling

* Update <ListReport /> linking

* Show labeled values in list of filters

* Updae Props component handling of storage keys

* re-add special case handling in devices view

* Fix modal-related typo

* Get updatedQuery callsites working

* Update location modals linking

* Update props details model linking logic

* Switch back tab from props when removing goal filter

* Remove query.filters usage from within <Referrers /> component

* Private escapeFilterValue

* Fix sources/index.js

* Legacy redirect logic

* Update comment

* Disabled options in props modal

* Update escaping and is_not operator

* Restore `false` search property handling meaning unset

* changelog

* Fix filtering after clicking on a map

* FilterOperatorSelector

* replaceFilterByPrefix

* Improve naming for filter modals/groups

CHANGELOG.md

@@ -63,6 +63,7 @@ All notable changes to this project will be documented in this file.
 - GA/SC sections moved to new settings: Integrations
 - Replace `CLICKHOUSE_MAX_BUFFER_SIZE` with `CLICKHOUSE_MAX_BUFFER_SIZE_BYTES`
 - Validate metric isn't queried multiple times
+- Filters in dashboard are represented by jsonurl
 
 ### Fixed
 - Creating many sites no longer leads to cookie overflow

assets/js/dashboard.js

@@ -6,6 +6,7 @@ import Router from './dashboard/router'
 import ErrorBoundary from './dashboard/error-boundary'
 import * as api from './dashboard/api'
 import * as timer from './dashboard/util/realtime-update-timer'
+import { filtersBackwardsCompatibilityRedirect } from './dashboard/query';
 
 timer.start()
 
@@ -40,6 +41,8 @@ if (container) {
  api.setSharedLinkAuth(sharedLinkAuth)
  }
 
+ filtersBackwardsCompatibilityRedirect()
+
  const app = (
  <ErrorBoundary>
  <Router site={site} loggedIn={loggedIn} currentUserRole={currentUserRole} />

assets/js/dashboard/api.js

@@ -1,4 +1,5 @@
 import { formatISO } from './util/date'
+import { serializeApiFilters } from './util/filters'
 
 let abortController = new AbortController()
 let SHARED_LINK_AUTH = null
@@ -30,19 +31,13 @@ export function cancelAll() {
  abortController = new AbortController()
 }
 
-function serializeFilters(filters) {
- const cleaned = {}
- Object.entries(filters).forEach(([key, val]) => val ? cleaned[key] = val : null);
- return JSON.stringify(cleaned)
-}
-
 export function serializeQuery(query, extraQuery = []) {
  const queryObj = {}
  if (query.period) { queryObj.period = query.period }
  if (query.date) { queryObj.date = formatISO(query.date) }
  if (query.from) { queryObj.from = formatISO(query.from) }
  if (query.to) { queryObj.to = formatISO(query.to) }
- if (query.filters) { queryObj.filters = serializeFilters(query.filters) }
+ if (query.filters) { queryObj.filters = serializeApiFilters(query.filters) }
  if (query.experimental_session_count) { queryObj.experimental_session_count = query.experimental_session_count }
  if (query.with_imported) { queryObj.with_imported = query.with_imported }
  if (SHARED_LINK_AUTH) { queryObj.auth = SHARED_LINK_AUTH }

assets/js/dashboard/components/filter-type-selector.js → assets/js/dashboard/components/filter-operator-selector.js

@@ -6,7 +6,7 @@ import { ChevronDownIcon } from '@heroicons/react/20/solid'
 import { isFreeChoiceFilter, supportsIsNot } from "../util/filters";
 import classNames from "classnames";
 
-export default function FilterTypeSelector(props) {
+export default function FilterOperatorSelector(props) {
  const filterName = props.forFilter
 
  function renderTypeItem(type, shouldDisplay) {

assets/js/dashboard/filters.js

@@ -4,81 +4,70 @@ import { AdjustmentsVerticalIcon, MagnifyingGlassIcon, XMarkIcon, PencilSquareIc
 import classNames from 'classnames'
 import { Menu, Transition } from '@headlessui/react'
 
-import { appliedFilters, navigateToQuery } from './query'
+import { navigateToQuery } from './query'
 import {
- FILTER_GROUPS,
+ FILTER_GROUP_TO_MODAL_TYPE,
+ cleanLabels,
+ FILTER_MODAL_TO_FILTER_GROUP,
  formatFilterGroup,
- filterGroupForFilter,
- parseQueryFilter,
- parseQueryPropsFilter,
- formattedFilters
-} from "./util/filters";
-
-function removeFilter(filterType, key, history, query) {
- const newOpts = {}
- if (filterType === 'props') {
- if (Object.keys(query.filters.props).length == 1) {
- newOpts.props = false
- } else {
- newOpts.props = JSON.stringify({
- ...query.filters.props,
- [key]: undefined,
- })
- }
- } else {
- newOpts[key] = false
- }
- if (key === 'country') { newOpts.country_labels = false }
- if (key === 'region') { newOpts.region_labels = false }
- if (key === 'city') { newOpts.city_labels = false }
+ formattedFilters,
+ EVENT_PROPS_PREFIX,
+ getPropertyKeyFromFilterKey,
+ getLabel
+} from "./util/filters"
+
+function removeFilter(filterIndex, history, query) {
+ const newFilters = query.filters.filter((_filter, index) => filterIndex != index)
+ const newLabels = cleanLabels(newFilters, query.labels)
 
  navigateToQuery(
  history,
  query,
- newOpts
+ { filters: newFilters, labels: newLabels }
  )
 }
 
 function clearAllFilters(history, query) {
- const newOpts = Object.keys(query.filters).reduce((acc, red) => ({ ...acc, [red]: false }), {});
  navigateToQuery(
  history,
  query,
- newOpts
+ { filters: false, labels: false }
  );
 }
 
-function filterText(filterType, key, query) {
- const formattedFilter = formattedFilters[key]
+function filterText(query, [operation, filterKey, clauses]) {
+ const formattedFilter = formattedFilters[filterKey]
 
- if (filterType === "props") {
- const { propKey, clauses, type } = parseQueryPropsFilter(query).find((filter) => filter.propKey.value === key)
- return <>Property <b>{propKey.label}</b> {type} {clauses.map(({label}) => <b key={label}>{label}</b>).reduce((prev, curr) => [prev, ' or ', curr])} </>
- } else if (formattedFilter) {
- const {type, clauses} = parseQueryFilter(query, key)
- return <>{formattedFilter} {type} {clauses.map(({label}) => <b key={label}>{label}</b>).reduce((prev, curr) => [prev, ' or ', curr])} </>
+ if (formattedFilter) {
+ return <>{formattedFilter} {operation} {clauses.map((value) => <b key={value}>{getLabel(query.labels, filterKey, value)}</b>).reduce((prev, curr) => [prev, ' or ', curr])} </>
+ } else if (filterKey.startsWith(EVENT_PROPS_PREFIX)) {
+ const propKey = getPropertyKeyFromFilterKey(filterKey)
+ return <>Property <b>{propKey}</b> {operation} {clauses.map((label) => <b key={label}>{label}</b>).reduce((prev, curr) => [prev, ' or ', curr])} </>
  }
 
- throw new Error(`Unknown filter: ${key}`)
+ throw new Error(`Unknown filter: ${filterKey}`)
 }
 
-function renderDropdownFilter(site, history, { key, value, filterType }, query) {
+function renderDropdownFilter(filterIndex, filter, site, history, query) {
+ const [_operation, filterKey, _clauses] = filter
+
+ const type = filterKey.startsWith(EVENT_PROPS_PREFIX) ? 'props' : filterKey
  return (
- <Menu.Item key={`${filterType}::${key}`}>
- <div className="px-3 md:px-4 sm:py-2 py-3 text-sm leading-tight flex items-center justify-between" key={key + value}>
+ <Menu.Item key={filterIndex}>
+ <div className="px-3 md:px-4 sm:py-2 py-3 text-sm leading-tight flex items-center justify-between" key={filterIndex}>
  <Link
- title={`Edit filter: ${formattedFilters[filterType]}`}
- to={{ pathname: `/${encodeURIComponent(site.domain)}/filter/${filterGroupForFilter(filterType)}`, search: window.location.search }}
+ title={`Edit filter: ${formattedFilters[type]}`}
+ to={{ pathname: `/${encodeURIComponent(site.domain)}/filter/${FILTER_GROUP_TO_MODAL_TYPE[type]}`, search: window.location.search }}
  className="group flex w-full justify-between items-center"
  style={{ width: 'calc(100% - 1.5rem)' }}
  >
- <span className="inline-block w-full truncate">{filterText(filterType, key, query)}</span>
+ <span className="inline-block w-full truncate">{filterText(query, filter)}</span>
  <PencilSquareIcon className="w-4 h-4 ml-1 cursor-pointer group-hover:text-indigo-700 dark:group-hover:text-indigo-500" />
  </Link>
  <b
- title={`Remove filter: ${formattedFilters[filterType]}`}
+ title={`Remove filter: ${formattedFilters[type]}`}
  className="ml-2 cursor-pointer hover:text-indigo-700 dark:hover:text-indigo-500"
- onClick={() => removeFilter(filterType, key, history, query)}
+ onClick={() => removeFilter(filterIndex, history, query)}
  >
  <XMarkIcon className="w-4 h-4" />
  </b>
@@ -109,18 +98,18 @@ function DropdownContent({ history, site, query, wrapped }) {
  const [addingFilter, setAddingFilter] = useState(false);
 
  if (wrapped === 0 || addingFilter) {
- let filterGroups = {...FILTER_GROUPS}
- if (!site.propsAvailable) delete filterGroups.props
+ let filterModals = {...FILTER_MODAL_TO_FILTER_GROUP}
+ if (!site.propsAvailable) delete filterModals.props
 
- return Object.keys(filterGroups).map((option) => filterDropdownOption(site, option))
+ return Object.keys(filterModals).map((option) => filterDropdownOption(site, option))
  }
 
  return (
  <>
  <div className="border-b border-gray-200 dark:border-gray-500 px-4 sm:py-2 py-3 text-sm leading-tight hover:text-indigo-700 dark:hover:text-indigo-500 hover:cursor-pointer" onClick={() => setAddingFilter(true)}>
  + Add filter
  </div>
- {appliedFilters(query).map((filter) => renderDropdownFilter(site, history, filter, query))}
+ {query.filters.map((filter, index) => renderDropdownFilter(index, filter, site, history, query))}
  <Menu.Item key="clear">
  <div className="border-t border-gray-200 dark:border-gray-500 px-4 sm:py-2 py-3 text-sm leading-tight hover:text-indigo-700 dark:hover:text-indigo-500 hover:cursor-pointer" onClick={() => clearAllFilters(history, query)}>
  Clear All Filters
@@ -193,15 +182,15 @@ class Filters extends React.Component {
  const { wrapped, viewport } = this.state;
 
  // Always wrap on mobile
- if (appliedFilters(this.props.query).length > 0 && viewport <= 768) {
+ if (this.props.query.filters.length > 0 && viewport <= 768) {
  this.setState({ wrapped: 2 })
  return;
  }
 
  this.setState({ wrapped: 0 });
 
  // Don't rewrap if we're already properly wrapped, there are no DOM children, or there is only filter
- if (wrapped !== 1 || !items || appliedFilters(this.props.query).length === 1) {
+ if (wrapped !== 1 || !items || this.props.query.filters.length === 1) {
  return;
  };
 
@@ -217,20 +206,26 @@ class Filters extends React.Component {
  });
  };
 
- renderListFilter(history, { key, value, filterType }, query) {
+ renderListFilter(filterIndex, filter, history, query) {
+ const text = filterText(query, filter)
+ const [_operation, filterKey, _clauses] = filter
+ const type = filterKey.startsWith(EVENT_PROPS_PREFIX) ? 'props' : filterKey
  return (
- <span key={`${filterType}::${key}`} title={value} className="flex bg-white dark:bg-gray-800 text-gray-700 dark:text-gray-300 shadow text-sm rounded mr-2 items-center">
+ <span key={filterIndex} className="flex bg-white dark:bg-gray-800 text-gray-700 dark:text-gray-300 shadow text-sm rounded mr-2 items-center">
  <Link
- title={`Edit filter: ${formattedFilters[filterType]}`}
+ title={`Edit filter: ${formattedFilters[type]}`}
  className="flex w-full h-full items-center py-2 pl-3"
- to={{ pathname: `/${encodeURIComponent(this.props.site.domain)}/filter/${filterGroupForFilter(filterType)}`, search: window.location.search }}
+ to={{
+ pathname: `/${encodeURIComponent(this.props.site.domain)}/filter/${FILTER_GROUP_TO_MODAL_TYPE[type]}`,
+ search: window.location.search
+ }}
  >
- <span className="inline-block max-w-2xs md:max-w-xs truncate">{filterText(filterType, key, query)}</span>
+ <span className="inline-block max-w-2xs md:max-w-xs truncate">{text}</span>
  </Link>
  <span
- title={`Remove filter: ${formattedFilters[filterType]}`}
+ title={`Remove filter: ${formattedFilters[type]}`}
  className="flex h-full w-full px-2 cursor-pointer hover:text-indigo-700 dark:hover:text-indigo-500 items-center"
- onClick={() => removeFilter(filterType, key, history, query)}
+ onClick={() => removeFilter(filterIndex, history, query)}
  >
  <XMarkIcon className="w-4 h-4" />
  </span>
@@ -240,7 +235,7 @@ class Filters extends React.Component {
 
  renderDropdownButton() {
  if (this.state.wrapped === 2) {
- const filterCount = appliedFilters(this.props.query).length
+ const filterCount = this.props.query.filters.length
  return (
  <>
  <AdjustmentsVerticalIcon className="-ml-1 mr-1 h-4 w-4" aria-hidden="true" />
@@ -309,7 +304,7 @@ class Filters extends React.Component {
  if (this.state.wrapped !== 2) {
  return (
  <div id="filters" className="flex flex-wrap">
- {(appliedFilters(query).map((filter) => this.renderListFilter(history, filter, query)))}
+ {query.filters.map((filter, index) => this.renderListFilter(index, filter, history, query))}
  </div>
  );
  }