feat: Split testing

api/pyproject.toml

@@ -123,6 +123,12 @@ optional = true
 [tool.poetry.group.ldap.dependencies]
 flagsmith-ldap = { git = "https://github.com/flagsmith/flagsmith-ldap", tag = "v0.1.0" }
 
+[tool.poetry.group.split-testing]
+optional = true
+
+[tool.poetry.group.split-testing.dependencies]
+flagsmith-split-testing = { git = "https://github.com/flagsmith/flagsmith-split-testing", tag = "v0.1.1" }
+
 [tool.poetry.group.dev.dependencies]
 django-test-migrations = "~1.2.0"
 responses = "~0.22.0"

frontend/common/services/useConversionEvent.ts

@@ -0,0 +1,50 @@
+import { Res } from 'common/types/responses'
+import { Req } from 'common/types/requests'
+import { service } from 'common/service'
+import Utils from 'common/utils/utils'
+
+export const conversionEventService = service
+ .enhanceEndpoints({ addTagTypes: ['ConversionEvent'] })
+ .injectEndpoints({
+ endpoints: (builder) => ({
+ getConversionEvents: builder.query<
+ Res['conversionEvents'],
+ Req['getConversionEvents']
+ >({
+ providesTags: [{ id: 'LIST', type: 'ConversionEvent' }],
+ query: (query) => {
+ return {
+ url: `conversion-event-types/?${Utils.toParam(query)}`,
+ }
+ },
+ }),
+ // END OF ENDPOINTS
+ }),
+ })
+
+export async function getConversionEvents(
+ store: any,
+ data: Req['getConversionEvents'],
+ options?: Parameters<
+ typeof conversionEventService.endpoints.getConversionEvents.initiate
+ >[1],
+) {
+ return store.dispatch(
+ conversionEventService.endpoints.getConversionEvents.initiate(
+ data,
+ options,
+ ),
+ )
+}
+// END OF FUNCTION_EXPORTS
+
+export const {
+ useGetConversionEventsQuery,
+ // END OF EXPORTS
+} = conversionEventService
+
+/* Usage examples:
+const { data, isLoading } = useGetConversionEventsQuery({ id: 2 }, {}) //get hook
+const [createConversionEvents, { isLoading, data, isSuccess }] = useCreateConversionEventsMutation() //create hook
+conversionEventService.endpoints.getConversionEvents.select({id: 2})(store.getState()) //access data from any function
+*/

frontend/common/services/useSplitTest.ts

@@ -0,0 +1,114 @@
+import {
+ PagedResponse,
+ PConfidence,
+ Res,
+ ServersideSplitTestResult,
+ SplitTestResult,
+} from 'common/types/responses'
+import { Req } from 'common/types/requests'
+import { service } from 'common/service'
+import Utils from 'common/utils/utils'
+import { groupBy, sortBy } from 'lodash'
+
+export const splitTestService = service
+ .enhanceEndpoints({ addTagTypes: ['SplitTest'] })
+ .injectEndpoints({
+ endpoints: (builder) => ({
+ getSplitTest: builder.query<Res['splitTest'], Req['getSplitTest']>({
+ providesTags: (res, _, q) => [
+ { id: q?.conversion_event_type_id, type: 'SplitTest' },
+ ],
+ query: (query: Req['getSplitTest']) => ({
+ url: `split-testing/?${Utils.toParam(query)}`,
+ }),
+ transformResponse: (res: PagedResponse<ServersideSplitTestResult>) => {
+ const groupedFeatures = groupBy(
+ res.results,
+ (item) => item.feature.id,
+ )
+
+ const results: SplitTestResult[] = Object.keys(groupedFeatures).map(
+ (group) => {
+ const features = groupedFeatures[group]
+ let minP = Number.MAX_SAFE_INTEGER
+ let maxP = Number.MIN_SAFE_INTEGER
+ let maxConversionCount = Number.MIN_SAFE_INTEGER
+ let maxConversionPercentage = Number.MIN_SAFE_INTEGER
+ let minConversion = Number.MAX_SAFE_INTEGER
+ let maxConversionPValue = 0
+ const results = sortBy(
+ features.map((v) => {
+ if (v.pvalue < minP) {
+ minP = v.pvalue
+ }
+ if (v.pvalue > maxP) {
+ maxP = v.pvalue
+ }
+ const conversion = v.conversion_count
+ ? Math.round(
+ (v.conversion_count / v.evaluation_count) * 100,
+ )
+ : 0
+ if (conversion > maxConversionPercentage) {
+ maxConversionCount = v.conversion_count
+ maxConversionPercentage = conversion
+ maxConversionPValue = v.pvalue
+ }
+ if (conversion < minConversion) {
+ minConversion = conversion
+ }
+
+ return {
+ confidence: Utils.convertToPConfidence(v.pvalue),
+ conversion_count: v.conversion_count,
+ conversion_percentage: conversion,
+ evaluation_count: v.evaluation_count,
+ pvalue: v.pvalue,
+ value_data: v.value_data,
+ } as SplitTestResult['results'][number]
+ }),
+ 'conversion_count',
+ )
+ return {
+ conversion_variance: maxConversionPercentage - minConversion,
+ feature: features[0].feature,
+ max_conversion_count: maxConversionCount,
+ max_conversion_percentage: maxConversionPercentage,
+ max_conversion_pvalue: maxConversionPValue,
+ results: sortBy(results, (v) => -v.conversion_count),
+ }
+ },
+ )
+ return {
+ ...res,
+ results,
+ }
+ },
+ }),
+ // END OF ENDPOINTS
+ }),
+ })
+
+export async function getSplitTest(
+ store: any,
+ data: Req['getSplitTest'],
+ options?: Parameters<
+ typeof splitTestService.endpoints.getSplitTest.initiate
+ >[1],
+) {
+ return store.dispatch(
+ splitTestService.endpoints.getSplitTest.initiate(data, options),
+ )
+}
+// END OF FUNCTION_EXPORTS
+
+export const {
+ useGetSplitTestQuery,
+ // END OF EXPORTS
+} = splitTestService
+
+/* Usage examples:
+const { data, isLoading } = useGetSplitTestQuery({ id: 2 }, {}) //get hook
+const [createSplitTest, { isLoading, data, isSuccess }] = useCreateSplitTestMutation() //create hook
+splitTestService.endpoints.getSplitTest.select({id: 2})(store.getState()) //access data from any function
+*/

frontend/common/types/requests.ts

@@ -248,5 +248,9 @@ export type Req = {
  }
  getAuditLogItem: { id: string }
  getProject: { id: string }
+ getConversionEvents: PagedRequest<{ q?: string; environment_id: string }>
+ getSplitTest: PagedRequest<{
+ conversion_event_type_id: string
+ }>
  // END OF TYPES
 }

frontend/common/types/responses.ts

@@ -6,8 +6,8 @@ export type EdgePagedResponse<T> = PagedResponse<T> & {
 }
 export type PagedResponse<T> = {
  count?: number
- next?: string
- previous?: string
+ next?: string | null
+ previous?: string | null
  results: T[]
 }
 export type FlagsmithValue = string | number | boolean | null
@@ -382,6 +382,59 @@ export type RolePermissionUser = {
  id: number
  role_name: string
 }
+
+export type ServersideSplitTestResult = {
+ conversion_count: number
+ evaluation_count: number
+ feature: {
+ created_date: string
+ default_enabled: boolean
+ description: any
+ id: number
+ initial_value: string
+ name: string
+ type: string
+ }
+ pvalue: number
+ value_data: FeatureStateValue
+}
+
+export type PConfidence =
+ | 'VERY_LOW'
+ | 'LOW'
+ | 'REASONABLE'
+ | 'HIGH'
+ | 'VERY_HIGH'
+export type SplitTestResult = {
+ results: {
+ conversion_count: number
+ evaluation_count: number
+ conversion_percentage: number
+ pvalue: number
+ confidence: PConfidence
+ value_data: FeatureStateValue
+ }[]
+ feature: {
+ created_date: string
+ default_enabled: boolean
+ description: any
+ id: number
+ initial_value: string
+ name: string
+ type: string
+ }
+ max_conversion_percentage: number
+ max_conversion_count: number
+ conversion_variance: number
+ max_conversion_pvalue: number
+}
+
+export type ConversionEvent = {
+ id: number
+ name: string
+ updated_at: string
+ created_at: string
+}
 export type FeatureVersion = {
  created_at: string
  updated_at: string
@@ -486,5 +539,7 @@ export type Res = {
  flagsmithProjectImport: { id: string }
  featureImports: PagedResponse<FeatureImport>
  serversideEnvironmentKeys: APIKey[]
+ conversionEvents: PagedResponse<ConversionEvent>
+ splitTest: PagedResponse<SplitTestResult>
  // END OF TYPES
 }

frontend/common/utils/utils.tsx

@@ -8,6 +8,7 @@ import {
  MultivariateFeatureStateValue,
  MultivariateOption,
  Organisation,
+ PConfidence,
  Project as ProjectType,
  ProjectFlag,
  SegmentCondition,
@@ -78,6 +79,13 @@ const Utils = Object.assign({}, require('./base/_utils'), {
  return typeof value === 'number'
  },
 
+ convertToPConfidence(value: number) {
+ if (value > 0.05) return 'LOW' as PConfidence
+ if (value >= 0.01) return 'REASONABLE' as PConfidence
+ if (value > 0.002) return 'HIGH' as PConfidence
+ return 'VERY_HIGH' as PConfidence
+ },
+
  displayLimitAlert(type: string, percentage: number | undefined) {
  const envOrProject =
  type === 'segment overrides' ? 'environment' : 'project'
@@ -327,6 +335,7 @@ const Utils = Object.assign({}, require('./base/_utils'), {
  }
  return valid
  },
+
  getPlansPermission: (permission: string) => {
  const isOrgPermission = permission !== '2FA'
  const plans = isOrgPermission
@@ -344,10 +353,10 @@ const Utils = Object.assign({}, require('./base/_utils'), {
  )
  return !!found
  },
-
  getProjectColour(index: number) {
  return Constants.projectColors[index % (Constants.projectColors.length - 1)]
  },
+
  getSDKEndpoint(_project: ProjectType) {
  const project = _project || ProjectStore.model
 

frontend/web/components/Aside.js

@@ -14,7 +14,7 @@ import Icon from './Icon'
 import ProjectSelect from './ProjectSelect'
 import AsideProjectButton from './AsideProjectButton'
 import Constants from 'common/constants'
-import { star, warning, pricetag } from 'ionicons/icons'
+import { star, warning, pricetag, flask } from 'ionicons/icons'
 import { IonIcon } from '@ionic/react'
 
 const Aside = class extends Component {
@@ -524,6 +524,22 @@ const Aside = class extends Component {
  </NavLink>,
  )}
 
+ {Utils.getFlagsmithHasFeature(
+ 'split_testing',
+ ) && (
+ <NavLink
+ id='env-settings-link'
+ className='aside__environment-list-item mt-1'
+ to={`/project/${project.id}/environment/${environment.api_key}/split-tests`}
+ >
+ <IonIcon
+ size={24}
+ icon={flask}
+ className='me-2'
+ />
+ Split Tests
+ </NavLink>
+ )}
  {environmentAdmin && (
  <NavLink
  id='env-settings-link'

frontend/web/components/Confidence.tsx

@@ -0,0 +1,25 @@
+import { FC } from 'react'
+import cn from 'classnames'
+import Utils from 'common/utils/utils'
+import Format from 'common/utils/format'
+
+type ConfidenceType = {
+ pValue: number
+}
+
+const Confidence: FC<ConfidenceType> = ({ pValue }) => {
+ const confidence = Utils.convertToPConfidence(pValue)
+ const confidenceDisplay = Format.enumeration.get(confidence)
+
+ const confidenceClass = cn({
+ 'text-danger': confidence === 'VERY_LOW' || confidence === 'LOW',
+ 'text-muted': !['VERY_LOW', 'LOW', 'HIGH', 'VERY_HIGH'].includes(
+ confidence,
+ ),
+ 'text-success': confidence === 'HIGH' || confidence === 'VERY_HIGH',
+ })
+
+ return <div className={confidenceClass}>{confidenceDisplay}</div>
+}
+
+export default Confidence