feat: split testing UI

.github/workflows/platform-pull-request.yml

@@ -117,7 +117,7 @@ jobs:
       scan: false
 
   docker-build-private-cloud:
-    if: github.event.pull_request.draft == false && needs.permissions-check.outputs.can-write == 'true'
+    if: needs.permissions-check.outputs.can-write == 'true'
     needs: [permissions-check, docker-prepare-report-comment]
     name: Build Private Cloud Image
     uses: ./.github/workflows/.reusable-docker-build.yml

Dockerfile

@@ -111,7 +111,7 @@ ARG RBAC_REVISION
 RUN --mount=type=secret,id=github_private_cloud_token \
   echo "https://$(cat /run/secrets/github_private_cloud_token):@github.com" > ${HOME}/.git-credentials && \
   git config --global credential.helper store && \
-  make install-packages opts='--without dev --with saml,auth-controller,ldap,workflows,licensing' && \
+  make install-packages opts='--without dev --with saml,auth-controller,ldap,workflows,licensing,split-testing' && \
   make install-private-modules
 
 # * api-runtime

api/pyproject.toml

@@ -170,7 +170,7 @@ pygithub = "2.1.1"
 hubspot-api-client = "^8.2.1"
 djangorestframework-dataclasses = "^1.3.1"
 pyotp = "^2.9.0"
-flagsmith-task-processor = { git = "https://github.com/Flagsmith/flagsmith-task-processor", tag = "v1.2.0" }
+flagsmith-task-processor = { git = "https://github.com/Flagsmith/flagsmith-task-processor", tag = "v1.1.1" }
 flagsmith-common = { git = "https://github.com/Flagsmith/flagsmith-common", tag = "v1.4.2" }
 tzdata = "^2024.1"
 djangorestframework-simplejwt = "^5.3.1"
@@ -206,6 +206,12 @@ optional = true
 [tool.poetry.group.licensing.dependencies]
 licensing = { git = "https://github.com/flagsmith/licensing", tag = "v0.1.0" }
 
+[tool.poetry.group.split-testing]
+optional = true
+
+[tool.poetry.group.split-testing.dependencies]
+flagsmith-split-testing = { git = "https://github.com/flagsmith/flagsmith-split-testing", tag = "v0.1.3-demo1" }
+
 [tool.poetry.group.dev.dependencies]
 django-test-migrations = "~1.2.0"
 responses = "~0.22.0"

docker-compose.yml

@@ -21,7 +21,7 @@ services:
       start_period: 20s
 
   flagsmith:
-    image: flagsmith.docker.scarf.sh/flagsmith/flagsmith:latest
+    image: ghcr.io/flagsmith/flagsmith-private-cloud:pr-5093
     environment:
       # All environments variables are available here:
       # API: https://docs.flagsmith.com/deployment/locally-api#environment-variables
@@ -70,7 +70,7 @@ services:
   # The flagsmith_processor service is only needed if TASK_RUN_METHOD set to TASK_PROCESSOR
   # in the application environment
   flagsmith_processor:
-    image: flagsmith.docker.scarf.sh/flagsmith/flagsmith:latest
+    image: ghcr.io/flagsmith/flagsmith-private-cloud:pr-5093
     environment:
       DATABASE_URL: postgresql://postgres:password@postgres:5432/flagsmith
       USE_POSTGRES_FOR_ANALYTICS: 'true'

frontend/common/services/useConversionEvent.ts

@@ -0,0 +1,50 @@
+import { Res } from 'common/types/responses'
+import { Req } from 'common/types/requests'
+import { service } from 'common/service'
+import Utils from 'common/utils/utils'
+
+export const conversionEventService = service
+  .enhanceEndpoints({ addTagTypes: ['ConversionEvent'] })
+  .injectEndpoints({
+    endpoints: (builder) => ({
+      getConversionEvents: builder.query<
+        Res['conversionEvents'],
+        Req['getConversionEvents']
+      >({
+        providesTags: [{ id: 'LIST', type: 'ConversionEvent' }],
+        query: (query) => {
+          return {
+            url: `conversion-event-types/?${Utils.toParam(query)}`,
+          }
+        },
+      }),
+      // END OF ENDPOINTS
+    }),
+  })
+
+export async function getConversionEvents(
+  store: any,
+  data: Req['getConversionEvents'],
+  options?: Parameters<
+    typeof conversionEventService.endpoints.getConversionEvents.initiate
+  >[1],
+) {
+  return store.dispatch(
+    conversionEventService.endpoints.getConversionEvents.initiate(
+      data,
+      options,
+    ),
+  )
+}
+// END OF FUNCTION_EXPORTS
+
+export const {
+  useGetConversionEventsQuery,
+  // END OF EXPORTS
+} = conversionEventService
+
+/* Usage examples:
+const { data, isLoading } = useGetConversionEventsQuery({ id: 2 }, {}) //get hook
+const [createConversionEvents, { isLoading, data, isSuccess }] = useCreateConversionEventsMutation() //create hook
+conversionEventService.endpoints.getConversionEvents.select({id: 2})(store.getState()) //access data from any function
+*/

frontend/common/services/useSplitTest.ts

@@ -0,0 +1,114 @@
+import {
+  PagedResponse,
+  PConfidence,
+  Res,
+  ServersideSplitTestResult,
+  SplitTestResult,
+} from 'common/types/responses'
+import { Req } from 'common/types/requests'
+import { service } from 'common/service'
+import Utils from 'common/utils/utils'
+import { groupBy, sortBy } from 'lodash'
+
+export const splitTestService = service
+  .enhanceEndpoints({ addTagTypes: ['SplitTest'] })
+  .injectEndpoints({
+    endpoints: (builder) => ({
+      getSplitTest: builder.query<Res['splitTest'], Req['getSplitTest']>({
+        providesTags: (res, _, q) => [
+          { id: q?.conversion_event_type_id, type: 'SplitTest' },
+        ],
+        query: (query: Req['getSplitTest']) => ({
+          url: `split-testing/?${Utils.toParam(query)}`,
+        }),
+        transformResponse: (res: PagedResponse<ServersideSplitTestResult>) => {
+          const groupedFeatures = groupBy(
+            res.results,
+            (item) => item.feature.id,
+          )
+
+          const results: SplitTestResult[] = Object.keys(groupedFeatures).map(
+            (group) => {
+              const features = groupedFeatures[group]
+              let minP = Number.MAX_SAFE_INTEGER
+              let maxP = Number.MIN_SAFE_INTEGER
+              let maxConversionCount = Number.MIN_SAFE_INTEGER
+              let maxConversionPercentage = Number.MIN_SAFE_INTEGER
+              let minConversion = Number.MAX_SAFE_INTEGER
+              let maxConversionPValue = 0
+              const results = sortBy(
+                features.map((v) => {
+                  if (v.pvalue < minP) {
+                    minP = v.pvalue
+                  }
+                  if (v.pvalue > maxP) {
+                    maxP = v.pvalue
+                  }
+                  const conversion = v.conversion_count
+                    ? Math.round(
+                        (v.conversion_count / v.evaluation_count) * 100,
+                      )
+                    : 0
+                  if (conversion > maxConversionPercentage) {
+                    maxConversionCount = v.conversion_count
+                    maxConversionPercentage = conversion
+                    maxConversionPValue = v.pvalue
+                  }
+                  if (conversion < minConversion) {
+                    minConversion = conversion
+                  }
+
+                  return {
+                    confidence: Utils.convertToPConfidence(v.pvalue),
+                    conversion_count: v.conversion_count,
+                    conversion_percentage: conversion,
+                    evaluation_count: v.evaluation_count,
+                    pvalue: v.pvalue,
+                    value_data: v.value_data,
+                  } as SplitTestResult['results'][number]
+                }),
+                'conversion_count',
+              )
+              return {
+                conversion_variance: maxConversionPercentage - minConversion,
+                feature: features[0].feature,
+                max_conversion_count: maxConversionCount,
+                max_conversion_percentage: maxConversionPercentage,
+                max_conversion_pvalue: maxConversionPValue,
+                results: sortBy(results, (v) => -v.conversion_count),
+              }
+            },
+          )
+          return {
+            ...res,
+            results,
+          }
+        },
+      }),
+      // END OF ENDPOINTS
+    }),
+  })
+
+export async function getSplitTest(
+  store: any,
+  data: Req['getSplitTest'],
+  options?: Parameters<
+    typeof splitTestService.endpoints.getSplitTest.initiate
+  >[1],
+) {
+  return store.dispatch(
+    splitTestService.endpoints.getSplitTest.initiate(data, options),
+  )
+}
+// END OF FUNCTION_EXPORTS
+
+export const {
+  useGetSplitTestQuery,
+  // END OF EXPORTS
+} = splitTestService
+
+/* Usage examples:
+const { data, isLoading } = useGetSplitTestQuery({ id: 2 }, {}) //get hook
+const [createSplitTest, { isLoading, data, isSuccess }] = useCreateSplitTestMutation() //create hook
+splitTestService.endpoints.getSplitTest.select({id: 2})(store.getState()) //access data from any function
+*/

frontend/common/types/requests.ts

@@ -540,5 +540,9 @@ export type Req = {
     environmentId: string
     data: Identity
   }
+  getConversionEvents: PagedRequest<{ q?: string; environment_id: string }>
+  getSplitTest: PagedRequest<{
+    conversion_event_type_id: string
+  }>
   // END OF TYPES
 }

frontend/common/types/responses.ts

@@ -638,6 +638,21 @@ export type SAMLAttributeMapping = {
   django_attribute_name: AttributeName
   idp_attribute_name: string
 }
+export type ServersideSplitTestResult = {
+  conversion_count: number
+  evaluation_count: number
+  feature: {
+    created_date: string
+    default_enabled: boolean
+    description: any
+    id: number
+    initial_value: string
+    name: string
+    type: string
+  }
+  pvalue: number
+  value_data: FeatureStateValue
+}
 
 export type HealthEventType = 'HEALTHY' | 'UNHEALTHY'
 
@@ -658,6 +673,42 @@ export type HealthProvider = {
   webhook_url: number
 }
 
+export type PConfidence =
+  | 'VERY_LOW'
+  | 'LOW'
+  | 'REASONABLE'
+  | 'HIGH'
+  | 'VERY_HIGH'
+export type SplitTestResult = {
+  results: {
+    conversion_count: number
+    evaluation_count: number
+    conversion_percentage: number
+    pvalue: number
+    confidence: PConfidence
+    value_data: FeatureStateValue
+  }[]
+  feature: {
+    created_date: string
+    default_enabled: boolean
+    description: any
+    id: number
+    initial_value: string
+    name: string
+    type: string
+  }
+  max_conversion_percentage: number
+  max_conversion_count: number
+  conversion_variance: number
+  max_conversion_pvalue: number
+}
+
+export type ConversionEvent = {
+  id: number
+  name: string
+  updated_at: string
+  created_at: string
+}
 export type Res = {
   segments: PagedResponse<Segment>
   segment: Segment
@@ -784,5 +835,7 @@ export type Res = {
     metadata_xml: string
   }
   samlAttributeMapping: PagedResponse<SAMLAttributeMapping>
+  conversionEvents: PagedResponse<ConversionEvent>
+  splitTest: PagedResponse<SplitTestResult>
   // END OF TYPES
 }

frontend/common/utils/utils.tsx

@@ -13,6 +13,7 @@ import {
   ProjectFlag,
   SegmentCondition,
   Tag,
+  PConfidence
 } from 'common/types/responses'
 import flagsmith from 'flagsmith'
 import { ReactNode } from 'react'
@@ -57,7 +58,12 @@ const Utils = Object.assign({}, require('./base/_utils'), {
     img.src = src
     document.body.appendChild(img)
   },
-
+  convertToPConfidence(value: number) {
+    if (value > 0.05) return 'LOW' as PConfidence
+    if (value >= 0.01) return 'REASONABLE' as PConfidence
+    if (value > 0.002) return 'HIGH' as PConfidence
+    return 'VERY_HIGH' as PConfidence
+  },
   calculateControl(
     multivariateOptions: MultivariateOption[],
     variations?: MultivariateFeatureStateValue[],

frontend/web/components/Confidence.tsx

@@ -0,0 +1,25 @@
+import { FC } from 'react'
+import cn from 'classnames'
+import Utils from 'common/utils/utils'
+import Format from 'common/utils/format'
+
+type ConfidenceType = {
+  pValue: number
+}
+
+const Confidence: FC<ConfidenceType> = ({ pValue }) => {
+  const confidence = Utils.convertToPConfidence(pValue)
+  const confidenceDisplay = Format.enumeration.get(confidence)
+
+  const confidenceClass = cn({
+    'text-danger': confidence === 'VERY_LOW' || confidence === 'LOW',
+    'text-muted': !['VERY_LOW', 'LOW', 'HIGH', 'VERY_HIGH'].includes(
+      confidence,
+    ),
+    'text-success': confidence === 'HIGH' || confidence === 'VERY_HIGH',
+  })
+
+  return <div className={confidenceClass}>{confidenceDisplay}</div>
+}
+
+export default Confidence