diff --git a/frontend/src/layout/navigation-3000/navigationLogic.tsx b/frontend/src/layout/navigation-3000/navigationLogic.tsx index 0dcacab864b0b..bec5045532f0d 100644 --- a/frontend/src/layout/navigation-3000/navigationLogic.tsx +++ b/frontend/src/layout/navigation-3000/navigationLogic.tsx @@ -502,7 +502,7 @@ export const navigation3000Logic = kea([ identifier: Scene.LLMObservability, label: 'LLM observability', icon: , - to: urls.llmObservability(), + to: urls.llmObservability('dashboard'), tag: 'beta' as const, } : null, diff --git a/frontend/src/lib/taxonomy.tsx b/frontend/src/lib/taxonomy.tsx index a89cfe7f192cc..ce8d14eac7ecb 100644 --- a/frontend/src/lib/taxonomy.tsx +++ b/frontend/src/lib/taxonomy.tsx @@ -1423,6 +1423,11 @@ export const CORE_FILTER_DEFINITIONS_BY_GROUP = { 'The trace ID of the request made to the LLM API. Used to group together multiple generations into a single trace', examples: ['c9222e05-8708-41b8-98ea-d4a21849e761'], }, + $ai_request_url: { + label: 'AI Request URL (LLM)', + description: 'The full URL of the request made to the LLM API', + examples: ['https://api.openai.com/v1/chat/completions'], + }, }, numerical_event_properties: {}, // Same as event properties, see assignment below person_properties: {}, // Currently person properties are the same as event properties, see assignment below diff --git a/frontend/src/queries/nodes/DataTable/renderColumnMeta.tsx b/frontend/src/queries/nodes/DataTable/renderColumnMeta.tsx index 1905862986584..412941e8c10a4 100644 --- a/frontend/src/queries/nodes/DataTable/renderColumnMeta.tsx +++ b/frontend/src/queries/nodes/DataTable/renderColumnMeta.tsx @@ -3,7 +3,7 @@ import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types' import { SortingIndicator } from 'lib/lemon-ui/LemonTable/sorting' import { getQueryFeatures, QueryFeature } from '~/queries/nodes/DataTable/queryFeatures' -import { extractExpressionComment } from '~/queries/nodes/DataTable/utils' +import { extractExpressionComment, removeExpressionComment } from '~/queries/nodes/DataTable/utils' import { DataTableNode, DataVisualizationNode, EventsQuery } from '~/queries/schema' import { QueryContext } from '~/queries/types' import { isDataTableNode, isHogQLQuery, trimQuotes } from '~/queries/utils' @@ -52,7 +52,7 @@ export function renderColumnMeta diff --git a/frontend/src/scenes/llm-observability/LLMObservabilityScene.tsx b/frontend/src/scenes/llm-observability/LLMObservabilityScene.tsx index 10dbb71839839..15e42a85175c9 100644 --- a/frontend/src/scenes/llm-observability/LLMObservabilityScene.tsx +++ b/frontend/src/scenes/llm-observability/LLMObservabilityScene.tsx @@ -1,13 +1,18 @@ -import { LemonBanner, Link } from '@posthog/lemon-ui' +import { LemonBanner, LemonTabs, Link } from '@posthog/lemon-ui' import clsx from 'clsx' import { BindLogic, useActions, useValues } from 'kea' import { QueryCard } from 'lib/components/Cards/InsightCard/QueryCard' import { DateFilter } from 'lib/components/DateFilter/DateFilter' +import { PropertyFilters } from 'lib/components/PropertyFilters/PropertyFilters' +import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types' import { TestAccountFilterSwitch } from 'lib/components/TestAccountFiltersSwitch' import { SceneExport } from 'scenes/sceneTypes' +import { urls } from 'scenes/urls' import { dataNodeCollectionLogic } from '~/queries/nodes/DataNode/dataNodeCollectionLogic' +import { DataTable } from '~/queries/nodes/DataTable/DataTable' import { InsightVizNode, NodeKind } from '~/queries/schema/schema-general' +import { isEventsQuery } from '~/queries/utils' import { LLM_OBSERVABILITY_DATA_COLLECTION_NODE_ID, llmObservabilityLogic } from './llmObservabilityLogic' @@ -19,12 +24,22 @@ const Filters = (): JSX.Element => { const { dateFilter: { dateTo, dateFrom }, shouldFilterTestAccounts, + generationsQuery, + propertyFilters, } = useValues(llmObservabilityLogic) - const { setDates, setShouldFilterTestAccounts } = useActions(llmObservabilityLogic) + const { setDates, setShouldFilterTestAccounts, setPropertyFilters } = useActions(llmObservabilityLogic) return ( -
- +
+
+ + +
) @@ -73,13 +88,62 @@ const IngestionStatusCheck = (): JSX.Element | null => { ) } +function LLMObservabilityDashboard(): JSX.Element { + return ( + <> + + + + ) +} + +function LLMObservabilityGenerations(): JSX.Element { + const { setDates, setShouldFilterTestAccounts, setPropertyFilters } = useActions(llmObservabilityLogic) + const { generationsQuery } = useValues(llmObservabilityLogic) + + return ( + { + if (!isEventsQuery(query.source)) { + throw new Error('Invalid query') + } + setDates(query.source.after || null, query.source.before || null) + setShouldFilterTestAccounts(query.source.filterTestAccounts || false) + setPropertyFilters(query.source.properties || []) + }} + context={{ + emptyStateHeading: 'There were no generations in this period', + emptyStateDetail: 'Try changing the date range or filters.', + }} + uniqueKey="llm-observability-generations" + /> + ) +} + export function LLMObservabilityScene(): JSX.Element { + const { activeTab } = useValues(llmObservabilityLogic) + return ( - - - + , + link: urls.llmObservability('dashboard'), + }, + { + key: 'generations', + label: 'Generations', + content: , + link: urls.llmObservability('generations'), + }, + ]} + /> ) } diff --git a/frontend/src/scenes/llm-observability/llmObservabilityLogic.tsx b/frontend/src/scenes/llm-observability/llmObservabilityLogic.tsx index 34e877f529cd0..6e1bc4cdc46ff 100644 --- a/frontend/src/scenes/llm-observability/llmObservabilityLogic.tsx +++ b/frontend/src/scenes/llm-observability/llmObservabilityLogic.tsx @@ -1,11 +1,16 @@ import { actions, afterMount, kea, path, reducers, selectors } from 'kea' import { loaders } from 'kea-loaders' +import { urlToAction } from 'kea-router' import api from 'lib/api' +import { TaxonomicFilterGroupType } from 'lib/components/TaxonomicFilter/types' import { STALE_EVENT_SECONDS } from 'lib/constants' import { dayjs } from 'lib/dayjs' +import { LLMObservabilityTab, urls } from 'scenes/urls' -import { NodeKind, TrendsQuery } from '~/queries/schema/schema-general' +import { groupsModel } from '~/models/groupsModel' +import { DataTableNode, NodeKind, TrendsQuery } from '~/queries/schema/schema-general' import { + AnyPropertyFilter, BaseMathType, ChartDisplayType, EventDefinition, @@ -39,11 +44,19 @@ export const llmObservabilityLogic = kea([ path(['scenes', 'llm-observability', 'llmObservabilityLogic']), actions({ + setActiveTab: (activeTab: LLMObservabilityTab) => ({ activeTab }), setDates: (dateFrom: string | null, dateTo: string | null) => ({ dateFrom, dateTo }), setShouldFilterTestAccounts: (shouldFilterTestAccounts: boolean) => ({ shouldFilterTestAccounts }), + setPropertyFilters: (propertyFilters: AnyPropertyFilter[]) => ({ propertyFilters }), }), reducers({ + activeTab: [ + 'dashboard' as LLMObservabilityTab, + { + setActiveTab: (_, { activeTab }) => activeTab, + }, + ], dateFilter: [ { dateFrom: INITIAL_DATE_FROM, @@ -59,12 +72,38 @@ export const llmObservabilityLogic = kea([ setShouldFilterTestAccounts: (_, { shouldFilterTestAccounts }) => shouldFilterTestAccounts, }, ], + propertyFilters: [ + [] as AnyPropertyFilter[], + { + setPropertyFilters: (_, { propertyFilters }) => propertyFilters, + }, + ], + }), + + loaders({ + hasSentAiGenerationEvent: { + __default: undefined as boolean | undefined, + loadAIEventDefinition: async (): Promise => { + const aiGenerationDefinition = await api.eventDefinitions.list({ + event_type: EventDefinitionType.Event, + search: '$ai_generation', + }) + + // no need to worry about pagination here, event names beginning with $ are reserved, and we're not + // going to add enough reserved event names that match this search term to cause problems + const definition = aiGenerationDefinition.results.find((r) => r.name === '$ai_generation') + if (definition && !isDefinitionStale(definition)) { + return true + } + return false + }, + }, }), selectors({ tiles: [ - (s) => [s.dateFilter, s.shouldFilterTestAccounts], - (dateFilter, shouldFilterTestAccounts): QueryTile[] => [ + (s) => [s.dateFilter, s.shouldFilterTestAccounts, s.propertyFilters], + (dateFilter, shouldFilterTestAccounts, propertyFilters): QueryTile[] => [ { title: 'Generative AI users', description: 'To count users, set `distinct_id` in LLM tracking.', @@ -78,6 +117,7 @@ export const llmObservabilityLogic = kea([ }, ], dateRange: { date_from: dateFilter.dateFrom, date_to: dateFilter.dateTo }, + properties: propertyFilters, filterTestAccounts: shouldFilterTestAccounts, }, }, @@ -94,6 +134,7 @@ export const llmObservabilityLogic = kea([ }, ], dateRange: { date_from: dateFilter.dateFrom, date_to: dateFilter.dateTo }, + properties: propertyFilters, filterTestAccounts: shouldFilterTestAccounts, }, }, @@ -115,6 +156,7 @@ export const llmObservabilityLogic = kea([ display: ChartDisplayType.BoldNumber, }, dateRange: { date_from: dateFilter.dateFrom, date_to: dateFilter.dateTo }, + properties: propertyFilters, filterTestAccounts: shouldFilterTestAccounts, }, }, @@ -142,6 +184,7 @@ export const llmObservabilityLogic = kea([ decimalPlaces: 2, }, dateRange: { date_from: dateFilter.dateFrom, date_to: dateFilter.dateTo }, + properties: propertyFilters, filterTestAccounts: shouldFilterTestAccounts, }, }, @@ -168,6 +211,7 @@ export const llmObservabilityLogic = kea([ showValuesOnSeries: true, }, dateRange: { date_from: dateFilter.dateFrom, date_to: dateFilter.dateTo }, + properties: propertyFilters, filterTestAccounts: shouldFilterTestAccounts, }, }, @@ -182,6 +226,7 @@ export const llmObservabilityLogic = kea([ }, ], dateRange: { date_from: dateFilter.dateFrom, date_to: dateFilter.dateTo }, + properties: propertyFilters, filterTestAccounts: shouldFilterTestAccounts, }, }, @@ -206,6 +251,7 @@ export const llmObservabilityLogic = kea([ yAxisScaleType: 'log10', }, dateRange: { date_from: dateFilter.dateFrom, date_to: dateFilter.dateTo }, + properties: propertyFilters, filterTestAccounts: shouldFilterTestAccounts, }, }, @@ -226,31 +272,69 @@ export const llmObservabilityLogic = kea([ display: ChartDisplayType.ActionsBarValue, }, dateRange: { date_from: dateFilter.dateFrom, date_to: dateFilter.dateTo }, + properties: propertyFilters, filterTestAccounts: shouldFilterTestAccounts, }, }, ], ], + generationsQuery: [ + (s) => [ + s.dateFilter, + s.shouldFilterTestAccounts, + s.propertyFilters, + groupsModel.selectors.groupsTaxonomicTypes, + ], + (dateFilter, shouldFilterTestAccounts, propertyFilters, groupsTaxonomicTypes): DataTableNode => ({ + kind: NodeKind.DataTableNode, + source: { + kind: NodeKind.EventsQuery, + select: [ + '*', + 'uuid', + 'person', + 'properties.$ai_trace_id -- Trace ID', + "f'${round(toFloat(properties.$ai_total_cost_usd), 4)}' -- Total cost", + "f'{properties.$ai_input_tokens} → {properties.$ai_output_tokens} (∑ {properties.$ai_input_tokens + properties.$ai_output_tokens})' -- Token usage", + "f'{properties.$ai_latency} s' -- Latency", + 'timestamp', + ], + orderBy: ['timestamp DESC'], + after: dateFilter.dateFrom || undefined, + before: dateFilter.dateTo || undefined, + filterTestAccounts: shouldFilterTestAccounts, + event: '$ai_generation', + properties: propertyFilters, + }, + showDateRange: true, + showReload: true, + showSearch: true, + showTestAccountFilters: true, + showPropertyFilter: [ + TaxonomicFilterGroupType.EventProperties, + TaxonomicFilterGroupType.PersonProperties, + ...groupsTaxonomicTypes, + TaxonomicFilterGroupType.Cohorts, + TaxonomicFilterGroupType.HogQLExpression, + ], + showExport: true, + }), + ], }), - loaders({ - hasSentAiGenerationEvent: { - __default: undefined as boolean | undefined, - loadAIEventDefinition: async (): Promise => { - const aiGenerationDefinition = await api.eventDefinitions.list({ - event_type: EventDefinitionType.Event, - search: '$ai_generation', - }) - // no need to worry about pagination here, event names beginning with $ are reserved, and we're not - // going to add enough reserved event names that match this search term to cause problems - const definition = aiGenerationDefinition.results.find((r) => r.name === '$ai_generation') - if (definition && !isDefinitionStale(definition)) { - return true - } - return false - }, + urlToAction(({ actions, values }) => ({ + [urls.llmObservability('dashboard')]: () => { + if (values.activeTab !== 'dashboard') { + actions.setActiveTab('dashboard') + } }, - }), + [urls.llmObservability('generations')]: () => { + if (values.activeTab !== 'generations') { + actions.setActiveTab('generations') + } + }, + })), + afterMount(({ actions }) => { actions.loadAIEventDefinition() }), diff --git a/frontend/src/scenes/scenes.ts b/frontend/src/scenes/scenes.ts index 47dec0854d1ef..59f7281d0111a 100644 --- a/frontend/src/scenes/scenes.ts +++ b/frontend/src/scenes/scenes.ts @@ -659,5 +659,6 @@ export const routes: Record = { [urls.messagingBroadcasts()]: Scene.MessagingBroadcasts, [urls.messagingBroadcast(':id')]: Scene.MessagingBroadcasts, [urls.messagingBroadcastNew()]: Scene.MessagingBroadcasts, - [urls.llmObservability()]: Scene.LLMObservability, + [urls.llmObservability('dashboard')]: Scene.LLMObservability, + [urls.llmObservability('generations')]: Scene.LLMObservability, } diff --git a/frontend/src/scenes/urls.ts b/frontend/src/scenes/urls.ts index 344ab31102071..c25de47ddf9f8 100644 --- a/frontend/src/scenes/urls.ts +++ b/frontend/src/scenes/urls.ts @@ -35,6 +35,8 @@ import { SurveysTabs } from './surveys/surveysLogic' * Sync the paths with AutoProjectMiddleware! */ +export type LLMObservabilityTab = 'dashboard' | 'generations' + export const urls = { absolute: (path = ''): string => window.location.origin + path, default: (): string => '/', @@ -259,5 +261,6 @@ export const urls = { insightAlert: (insightShortId: InsightShortId, alertId: AlertType['id']): string => `/insights/${insightShortId}/alerts?alert_id=${alertId}`, sessionAttributionExplorer: (): string => '/web/session-attribution-explorer', - llmObservability: (): string => '/llm-observability', + llmObservability: (tab?: LLMObservabilityTab): string => + `/llm-observability${tab !== 'dashboard' ? '/' + tab : ''}`, }