feat(llm-observability): Generations view

frontend/src/layout/navigation-3000/navigationLogic.tsx

@@ -502,7 +502,7 @@ export const navigation3000Logic = kea<navigation3000LogicType>([
                                   identifier: Scene.LLMObservability,
                                   label: 'LLM observability',
                                   icon: <IconAI />,
-                                  to: urls.llmObservability(),
+                                  to: urls.llmObservability('dashboard'),
                                   tag: 'beta' as const,
                               }
                             : null,

frontend/src/lib/taxonomy.tsx

@@ -1423,6 +1423,11 @@ export const CORE_FILTER_DEFINITIONS_BY_GROUP = {
                 'The trace ID of the request made to the LLM API. Used to group together multiple generations into a single trace',
             examples: ['c9222e05-8708-41b8-98ea-d4a21849e761'],
         },
+        $ai_request_url: {
+            label: 'AI Request URL (LLM)',
+            description: 'The full URL of the request made to the LLM API',
+            examples: ['https://api.openai.com/v1/chat/completions'],
+        },
     },
     numerical_event_properties: {}, // Same as event properties, see assignment below
     person_properties: {}, // Currently person properties are the same as event properties, see assignment below

frontend/src/queries/nodes/DataTable/renderColumnMeta.tsx

@@ -3,7 +3,7 @@ import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types'
 import { SortingIndicator } from 'lib/lemon-ui/LemonTable/sorting'
 
 import { getQueryFeatures, QueryFeature } from '~/queries/nodes/DataTable/queryFeatures'
-import { extractExpressionComment } from '~/queries/nodes/DataTable/utils'
+import { extractExpressionComment, removeExpressionComment } from '~/queries/nodes/DataTable/utils'
 import { DataTableNode, DataVisualizationNode, EventsQuery } from '~/queries/schema'
 import { QueryContext } from '~/queries/types'
 import { isDataTableNode, isHogQLQuery, trimQuotes } from '~/queries/utils'
@@ -52,7 +52,7 @@ export function renderColumnMeta<T extends DataVisualizationNode | DataTableNode
     } else if (key.startsWith('properties.')) {
         title = (
             <PropertyKeyInfo
-                value={trimQuotes(key.substring(11))}
+                value={trimQuotes(removeExpressionComment(key.substring(11)))}
                 type={TaxonomicFilterGroupType.EventProperties}
                 disableIcon
             />

frontend/src/scenes/llm-observability/LLMObservabilityScene.tsx

@@ -1,13 +1,18 @@
-import { LemonBanner, Link } from '@posthog/lemon-ui'
+import { LemonBanner, LemonTabs, Link } from '@posthog/lemon-ui'
 import clsx from 'clsx'
 import { BindLogic, useActions, useValues } from 'kea'
 import { QueryCard } from 'lib/components/Cards/InsightCard/QueryCard'
 import { DateFilter } from 'lib/components/DateFilter/DateFilter'
+import { PropertyFilters } from 'lib/components/PropertyFilters/PropertyFilters'
+import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types'
 import { TestAccountFilterSwitch } from 'lib/components/TestAccountFiltersSwitch'
 import { SceneExport } from 'scenes/sceneTypes'
+import { urls } from 'scenes/urls'
 
 import { dataNodeCollectionLogic } from '~/queries/nodes/DataNode/dataNodeCollectionLogic'
+import { DataTable } from '~/queries/nodes/DataTable/DataTable'
 import { InsightVizNode, NodeKind } from '~/queries/schema/schema-general'
+import { isEventsQuery } from '~/queries/utils'
 
 import { LLM_OBSERVABILITY_DATA_COLLECTION_NODE_ID, llmObservabilityLogic } from './llmObservabilityLogic'
 
@@ -19,12 +24,22 @@ const Filters = (): JSX.Element => {
     const {
         dateFilter: { dateTo, dateFrom },
         shouldFilterTestAccounts,
+        generationsQuery,
+        propertyFilters,
     } = useValues(llmObservabilityLogic)
-    const { setDates, setShouldFilterTestAccounts } = useActions(llmObservabilityLogic)
+    const { setDates, setShouldFilterTestAccounts, setPropertyFilters } = useActions(llmObservabilityLogic)
 
     return (
-        <div className="mb-4 flex justify-between items-center">
-            <DateFilter dateFrom={dateFrom} dateTo={dateTo} onChange={setDates} />
+        <div className="flex justify-between items-center gap-4 py-4 -mt-4 mb-4 border-b">
+            <div className="flex items-center gap-4">
+                <DateFilter dateFrom={dateFrom} dateTo={dateTo} onChange={setDates} />
+                <PropertyFilters
+                    propertyFilters={propertyFilters}
+                    taxonomicGroupTypes={generationsQuery.showPropertyFilter as TaxonomicFilterGroupType[]}
+                    onChange={setPropertyFilters}
+                    pageKey="llm-observability"
+                />
+            </div>
             <TestAccountFilterSwitch checked={shouldFilterTestAccounts} onChange={setShouldFilterTestAccounts} />
         </div>
     )
@@ -73,13 +88,62 @@ const IngestionStatusCheck = (): JSX.Element | null => {
     )
 }
 
+function LLMObservabilityDashboard(): JSX.Element {
+    return (
+        <>
+            <Filters />
+            <Tiles />
+        </>
+    )
+}
+
+function LLMObservabilityGenerations(): JSX.Element {
+    const { setDates, setShouldFilterTestAccounts, setPropertyFilters } = useActions(llmObservabilityLogic)
+    const { generationsQuery } = useValues(llmObservabilityLogic)
+
+    return (
+        <DataTable
+            query={generationsQuery}
+            setQuery={(query) => {
+                if (!isEventsQuery(query.source)) {
+                    throw new Error('Invalid query')
+                }
+                setDates(query.source.after || null, query.source.before || null)
+                setShouldFilterTestAccounts(query.source.filterTestAccounts || false)
+                setPropertyFilters(query.source.properties || [])
+            }}
+            context={{
+                emptyStateHeading: 'There were no generations in this period',
+                emptyStateDetail: 'Try changing the date range or filters.',
+            }}
+            uniqueKey="llm-observability-generations"
+        />
+    )
+}
+
 export function LLMObservabilityScene(): JSX.Element {
+    const { activeTab } = useValues(llmObservabilityLogic)
+
     return (
         <BindLogic logic={dataNodeCollectionLogic} props={{ key: LLM_OBSERVABILITY_DATA_COLLECTION_NODE_ID }}>
             <IngestionStatusCheck />
-
-            <Filters />
-            <Tiles />
+            <LemonTabs
+                activeKey={activeTab}
+                tabs={[
+                    {
+                        key: 'dashboard',
+                        label: 'Dashboard',
+                        content: <LLMObservabilityDashboard />,
+                        link: urls.llmObservability('dashboard'),
+                    },
+                    {
+                        key: 'generations',
+                        label: 'Generations',
+                        content: <LLMObservabilityGenerations />,
+                        link: urls.llmObservability('generations'),
+                    },
+                ]}
+            />
         </BindLogic>
     )
 }

frontend/src/scenes/llm-observability/llmObservabilityLogic.tsx

@@ -1,11 +1,16 @@
 import { actions, afterMount, kea, path, reducers, selectors } from 'kea'
 import { loaders } from 'kea-loaders'
+import { urlToAction } from 'kea-router'
 import api from 'lib/api'
+import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types'
 import { STALE_EVENT_SECONDS } from 'lib/constants'
 import { dayjs } from 'lib/dayjs'
+import { LLMObservabilityTab, urls } from 'scenes/urls'
 
-import { NodeKind, TrendsQuery } from '~/queries/schema/schema-general'
+import { groupsModel } from '~/models/groupsModel'
+import { DataTableNode, NodeKind, TrendsQuery } from '~/queries/schema/schema-general'
 import {
+    AnyPropertyFilter,
     BaseMathType,
     ChartDisplayType,
     EventDefinition,
@@ -39,11 +44,19 @@ export const llmObservabilityLogic = kea<llmObservabilityLogicType>([
     path(['scenes', 'llm-observability', 'llmObservabilityLogic']),
 
     actions({
+        setActiveTab: (activeTab: LLMObservabilityTab) => ({ activeTab }),
         setDates: (dateFrom: string | null, dateTo: string | null) => ({ dateFrom, dateTo }),
         setShouldFilterTestAccounts: (shouldFilterTestAccounts: boolean) => ({ shouldFilterTestAccounts }),
+        setPropertyFilters: (propertyFilters: AnyPropertyFilter[]) => ({ propertyFilters }),
     }),
 
     reducers({
+        activeTab: [
+            'dashboard' as LLMObservabilityTab,
+            {
+                setActiveTab: (_, { activeTab }) => activeTab,
+            },
+        ],
         dateFilter: [
             {
                 dateFrom: INITIAL_DATE_FROM,
@@ -59,12 +72,38 @@ export const llmObservabilityLogic = kea<llmObservabilityLogicType>([
                 setShouldFilterTestAccounts: (_, { shouldFilterTestAccounts }) => shouldFilterTestAccounts,
             },
         ],
+        propertyFilters: [
+            [] as AnyPropertyFilter[],
+            {
+                setPropertyFilters: (_, { propertyFilters }) => propertyFilters,
+            },
+        ],
+    }),
+
+    loaders({
+        hasSentAiGenerationEvent: {
+            __default: undefined as boolean | undefined,
+            loadAIEventDefinition: async (): Promise<boolean> => {
+                const aiGenerationDefinition = await api.eventDefinitions.list({
+                    event_type: EventDefinitionType.Event,
+                    search: '$ai_generation',
+                })
+
+                // no need to worry about pagination here, event names beginning with $ are reserved, and we're not
+                // going to add enough reserved event names that match this search term to cause problems
+                const definition = aiGenerationDefinition.results.find((r) => r.name === '$ai_generation')
+                if (definition && !isDefinitionStale(definition)) {
+                    return true
+                }
+                return false
+            },
+        },
     }),
 
     selectors({
         tiles: [
-            (s) => [s.dateFilter, s.shouldFilterTestAccounts],
-            (dateFilter, shouldFilterTestAccounts): QueryTile[] => [
+            (s) => [s.dateFilter, s.shouldFilterTestAccounts, s.propertyFilters],
+            (dateFilter, shouldFilterTestAccounts, propertyFilters): QueryTile[] => [
                 {
                     title: 'Generative AI users',
                     description: 'To count users, set `distinct_id` in LLM tracking.',
@@ -78,6 +117,7 @@ export const llmObservabilityLogic = kea<llmObservabilityLogicType>([
                             },
                         ],
                         dateRange: { date_from: dateFilter.dateFrom, date_to: dateFilter.dateTo },
+                        properties: propertyFilters,
                         filterTestAccounts: shouldFilterTestAccounts,
                     },
                 },
@@ -94,6 +134,7 @@ export const llmObservabilityLogic = kea<llmObservabilityLogicType>([
                             },
                         ],
                         dateRange: { date_from: dateFilter.dateFrom, date_to: dateFilter.dateTo },
+                        properties: propertyFilters,
                         filterTestAccounts: shouldFilterTestAccounts,
                     },
                 },
@@ -115,6 +156,7 @@ export const llmObservabilityLogic = kea<llmObservabilityLogicType>([
                             display: ChartDisplayType.BoldNumber,
                         },
                         dateRange: { date_from: dateFilter.dateFrom, date_to: dateFilter.dateTo },
+                        properties: propertyFilters,
                         filterTestAccounts: shouldFilterTestAccounts,
                     },
                 },
@@ -142,6 +184,7 @@ export const llmObservabilityLogic = kea<llmObservabilityLogicType>([
                             decimalPlaces: 2,
                         },
                         dateRange: { date_from: dateFilter.dateFrom, date_to: dateFilter.dateTo },
+                        properties: propertyFilters,
                         filterTestAccounts: shouldFilterTestAccounts,
                     },
                 },
@@ -168,6 +211,7 @@ export const llmObservabilityLogic = kea<llmObservabilityLogicType>([
                             showValuesOnSeries: true,
                         },
                         dateRange: { date_from: dateFilter.dateFrom, date_to: dateFilter.dateTo },
+                        properties: propertyFilters,
                         filterTestAccounts: shouldFilterTestAccounts,
                     },
                 },
@@ -182,6 +226,7 @@ export const llmObservabilityLogic = kea<llmObservabilityLogicType>([
                             },
                         ],
                         dateRange: { date_from: dateFilter.dateFrom, date_to: dateFilter.dateTo },
+                        properties: propertyFilters,
                         filterTestAccounts: shouldFilterTestAccounts,
                     },
                 },
@@ -206,6 +251,7 @@ export const llmObservabilityLogic = kea<llmObservabilityLogicType>([
                             yAxisScaleType: 'log10',
                         },
                         dateRange: { date_from: dateFilter.dateFrom, date_to: dateFilter.dateTo },
+                        properties: propertyFilters,
                         filterTestAccounts: shouldFilterTestAccounts,
                     },
                 },
@@ -226,31 +272,69 @@ export const llmObservabilityLogic = kea<llmObservabilityLogicType>([
                             display: ChartDisplayType.ActionsBarValue,
                         },
                         dateRange: { date_from: dateFilter.dateFrom, date_to: dateFilter.dateTo },
+                        properties: propertyFilters,
                         filterTestAccounts: shouldFilterTestAccounts,
                     },
                 },
             ],
         ],
+        generationsQuery: [
+            (s) => [
+                s.dateFilter,
+                s.shouldFilterTestAccounts,
+                s.propertyFilters,
+                groupsModel.selectors.groupsTaxonomicTypes,
+            ],
+            (dateFilter, shouldFilterTestAccounts, propertyFilters, groupsTaxonomicTypes): DataTableNode => ({
+                kind: NodeKind.DataTableNode,
+                source: {
+                    kind: NodeKind.EventsQuery,
+                    select: [
+                        '*',
+                        'uuid',
+                        'person',
+                        'properties.$ai_trace_id -- Trace ID',
+                        "f'${round(toFloat(properties.$ai_total_cost_usd), 4)}' -- Total cost",
+                        "f'{properties.$ai_input_tokens} → {properties.$ai_output_tokens} (∑ {properties.$ai_input_tokens + properties.$ai_output_tokens})' -- Token usage",
+                        "f'{properties.$ai_latency} s' -- Latency",
+                        'timestamp',
+                    ],
+                    orderBy: ['timestamp DESC'],
+                    after: dateFilter.dateFrom || undefined,
+                    before: dateFilter.dateTo || undefined,
+                    filterTestAccounts: shouldFilterTestAccounts,
+                    event: '$ai_generation',
+                    properties: propertyFilters,
+                },
+                showDateRange: true,
+                showReload: true,
+                showSearch: true,
+                showTestAccountFilters: true,
+                showPropertyFilter: [
+                    TaxonomicFilterGroupType.EventProperties,
+                    TaxonomicFilterGroupType.PersonProperties,
+                    ...groupsTaxonomicTypes,
+                    TaxonomicFilterGroupType.Cohorts,
+                    TaxonomicFilterGroupType.HogQLExpression,
+                ],
+                showExport: true,
+            }),
+        ],
     }),
-    loaders({
-        hasSentAiGenerationEvent: {
-            __default: undefined as boolean | undefined,
-            loadAIEventDefinition: async (): Promise<boolean> => {
-                const aiGenerationDefinition = await api.eventDefinitions.list({
-                    event_type: EventDefinitionType.Event,
-                    search: '$ai_generation',
-                })
 
-                // no need to worry about pagination here, event names beginning with $ are reserved, and we're not
-                // going to add enough reserved event names that match this search term to cause problems
-                const definition = aiGenerationDefinition.results.find((r) => r.name === '$ai_generation')
-                if (definition && !isDefinitionStale(definition)) {
-                    return true
-                }
-                return false
-            },
+    urlToAction(({ actions, values }) => ({
+        [urls.llmObservability('dashboard')]: () => {
+            if (values.activeTab !== 'dashboard') {
+                actions.setActiveTab('dashboard')
+            }
         },
-    }),
+        [urls.llmObservability('generations')]: () => {
+            if (values.activeTab !== 'generations') {
+                actions.setActiveTab('generations')
+            }
+        },
+    })),
+
     afterMount(({ actions }) => {
         actions.loadAIEventDefinition()
     }),

frontend/src/scenes/scenes.ts

@@ -659,5 +659,6 @@ export const routes: Record<string, Scene> = {
     [urls.messagingBroadcasts()]: Scene.MessagingBroadcasts,
     [urls.messagingBroadcast(':id')]: Scene.MessagingBroadcasts,
     [urls.messagingBroadcastNew()]: Scene.MessagingBroadcasts,
-    [urls.llmObservability()]: Scene.LLMObservability,
+    [urls.llmObservability('dashboard')]: Scene.LLMObservability,
+    [urls.llmObservability('generations')]: Scene.LLMObservability,
 }

frontend/src/scenes/urls.ts

@@ -35,6 +35,8 @@ import { SurveysTabs } from './surveys/surveysLogic'
  * Sync the paths with AutoProjectMiddleware!
  */
 
+export type LLMObservabilityTab = 'dashboard' | 'generations'
+
 export const urls = {
     absolute: (path = ''): string => window.location.origin + path,
     default: (): string => '/',
@@ -259,5 +261,6 @@ export const urls = {
     insightAlert: (insightShortId: InsightShortId, alertId: AlertType['id']): string =>
         `/insights/${insightShortId}/alerts?alert_id=${alertId}`,
     sessionAttributionExplorer: (): string => '/web/session-attribution-explorer',
-    llmObservability: (): string => '/llm-observability',
+    llmObservability: (tab?: LLMObservabilityTab): string =>
+        `/llm-observability${tab !== 'dashboard' ? '/' + tab : ''}`,
 }