Merge branch 'master' into feat/flags-to-prod

frontend/src/lib/components/PropertyKeyInfo.tsx

@@ -38,11 +38,8 @@ export const PropertyKeyInfo = React.forwardRef<HTMLSpanElement, PropertyKeyInfo
     const valueDisplayText = (coreDefinition ? coreDefinition.label : value)?.trim() ?? ''
     const valueDisplayElement = valueDisplayText === '' ? <i>(empty string)</i> : valueDisplayText
 
-    const recognizedSource: 'posthog' | 'langfuse' | null = coreDefinition
-        ? 'posthog'
-        : value.startsWith('langfuse ')
-        ? 'langfuse'
-        : null
+    const recognizedSource: 'posthog' | 'langfuse' | null =
+        coreDefinition || value.startsWith('$') ? 'posthog' : value.startsWith('langfuse ') ? 'langfuse' : null
 
     const innerContent = (
         <span
@@ -68,7 +65,9 @@ export const PropertyKeyInfo = React.forwardRef<HTMLSpanElement, PropertyKeyInfo
             overlay={
                 <div className="PropertyKeyInfo__overlay">
                     <div className="PropertyKeyInfo__header">
-                        {!!coreDefinition && <span className="PropertyKeyInfo__logo" />}
+                        {!!coreDefinition && (
+                            <span className={`PropertyKeyInfo__logo PropertyKeyInfo__logo--${recognizedSource}`} />
+                        )}
                         {coreDefinition.label}
                     </div>
                     {coreDefinition.description || coreDefinition.examples ? (

frontend/src/lib/taxonomy.tsx

@@ -177,7 +177,12 @@ export const CORE_FILTER_DEFINITIONS_BY_GROUP = {
         },
         $ai_generation: {
             label: 'AI Generation',
-            description: 'A call to a generative AI model, e.g. an LLM',
+            description: 'A call to a generative AI model (LLM)',
+        },
+        $ai_trace: {
+            label: 'AI Trace',
+            description:
+                'A generative AI trace. Usually a trace tracks a single user interaction and contains one or more AI generation calls',
         },
         $ai_metric: {
             label: 'AI Metric',
@@ -1378,9 +1383,9 @@ export const CORE_FILTER_DEFINITIONS_BY_GROUP = {
             description: 'The number of tokens in the input prmopt that was sent to the LLM API',
             examples: [23],
         },
-        $ai_output: {
+        $ai_output_choices: {
             label: 'AI Output (LLM)',
-            description: 'The output JSON that was received from the LLM API',
+            description: 'The output message choices JSON that was received from the LLM API',
             examples: [
                 '{"choices": [{"text": "Quantum computing is a type of computing that harnesses the power of quantum mechanics to perform operations on data."}]}',
             ],
@@ -1420,6 +1425,29 @@ export const CORE_FILTER_DEFINITIONS_BY_GROUP = {
             description: 'The parameters used to configure the model in the LLM API, in JSON',
             examples: ['{"temperature": 0.5, "max_tokens": 50}'],
         },
+        $ai_stream: {
+            label: 'AI Stream (LLM)',
+            description: 'Whether the response from the LLM API was streamed',
+            examples: ['true', 'false'],
+        },
+        $ai_temperature: {
+            label: 'AI Temperature (LLM)',
+            description: 'The temperature parameter used in the request to the LLM API',
+            examples: [0.7, 1.0],
+        },
+        $ai_input_state: {
+            label: 'AI Input State (LLM)',
+            description: 'Input state of the LLM agent',
+        },
+        $ai_output_state: {
+            label: 'AI Output State (LLM)',
+            description: 'Output state of the LLM agent',
+        },
+        $ai_trace_name: {
+            label: 'AI Trace Name (LLM)',
+            description: 'The name given to this trace of LLM API calls',
+            examples: ['LangGraph'],
+        },
         $ai_provider: {
             label: 'AI Provider (LLM)',
             description: 'The provider of the AI model used to generate the output from the LLM API',

frontend/src/queries/nodes/DataVisualization/Components/Charts/LineGraph.tsx

@@ -188,10 +188,28 @@ export const LineGraph = (): JSX.Element => {
                     },
                     scaleID: hasLeftYAxis ? 'yLeft' : 'yRight',
                     value: cur.value,
+                    enter: (ctx) => {
+                        if (ctx.chart.options.plugins?.annotation?.annotations) {
+                            const annotations = ctx.chart.options.plugins.annotation.annotations as Record<string, any>
+                            if (annotations[`line${curIndex}`]) {
+                                annotations[`line${curIndex}`].label.content = `${cur.label}: ${cur.value}`
+                                ctx.chart.update()
+                            }
+                        }
+                    },
+                    leave: (ctx) => {
+                        if (ctx.chart.options.plugins?.annotation?.annotations) {
+                            const annotations = ctx.chart.options.plugins.annotation.annotations as Record<string, any>
+                            if (annotations[`line${curIndex}`]) {
+                                annotations[`line${curIndex}`].label.content = cur.label
+                                ctx.chart.update()
+                            }
+                        }
+                    },
                 }
 
                 acc.annotations[`line${curIndex}`] = {
-                    type: 'line',
+                    type: 'line' as const,
                     ...line,
                 }
 

frontend/src/queries/nodes/DataVisualization/Components/Variables/variablesLogic.ts

@@ -60,6 +60,7 @@ export const variablesLogic = kea<variablesLogicType>([
         }),
         setEditorQuery: (query: string) => ({ query }),
         updateSourceQuery: true,
+        resetVariables: true,
     })),
     propsChanged(({ props, actions }, oldProps) => {
         if (oldProps.queryInput !== props.queryInput) {
@@ -103,6 +104,9 @@ export const variablesLogic = kea<variablesLogicType>([
 
                     return stateCopy
                 },
+                resetVariables: () => {
+                    return []
+                },
             },
         ],
         editorQuery: [
@@ -191,6 +195,11 @@ export const variablesLogic = kea<variablesLogicType>([
         editorQuery: (query: string) => {
             const queryVariableMatches = getVariablesFromQuery(query)
 
+            if (!queryVariableMatches.length) {
+                actions.resetVariables()
+                return
+            }
+
             queryVariableMatches?.forEach((match) => {
                 if (match === null) {
                     return
@@ -212,10 +221,16 @@ export const variablesLogic = kea<variablesLogicType>([
                 return
             }
 
+            const queryVariableMatches = getVariablesFromQuery(query.source.query)
+
             const variables = Object.values(query.source.variables ?? {})
 
             if (variables.length) {
-                actions.addVariables(variables)
+                variables.forEach((variable) => {
+                    if (queryVariableMatches.includes(variable.code_name)) {
+                        actions.addVariable(variable)
+                    }
+                })
             }
         },
     })),

frontend/src/queries/nodes/DataVisualization/dataVisualizationLogic.ts

@@ -780,7 +780,10 @@ export const dataVisualizationLogic = kea<dataVisualizationLogicType>([
         dataVisualizationProps: [() => [(_, props) => props], (props): DataVisualizationLogicProps => props],
         isTableVisualization: [
             (state) => [state.visualizationType],
-            (visualizationType): boolean => visualizationType === ChartDisplayType.ActionsTable,
+            (visualizationType): boolean =>
+                // BoldNumber relies on yAxis formatting so it's considered a table visualization
+                visualizationType === ChartDisplayType.ActionsTable ||
+                visualizationType === ChartDisplayType.BoldNumber,
         ],
         showTableSettings: [
             (state) => [state.visualizationType],

frontend/src/queries/nodes/HogQLX/render.tsx

@@ -59,7 +59,7 @@ export function renderHogQLX(value: any): JSX.Element {
             const { href, children, source, target } = rest
             return (
                 <ErrorBoundary>
-                    <Link to={href} target={target ?? '_self'}>
+                    <Link to={href} target={target ?? undefined}>
                         {children ?? source ? renderHogQLX(children ?? source) : href}
                     </Link>
                 </ErrorBoundary>

frontend/src/scenes/data-warehouse/settings/source/Schemas.tsx

@@ -75,6 +75,7 @@ export const SchemaTable = ({ schemas, isLoading }: SchemaTableProps): JSX.Eleme
                             return (
                                 <LemonSelect
                                     className="my-1"
+                                    disabled={!schema.should_sync}
                                     value={schema.sync_frequency || '6hour'}
                                     onChange={(value) =>
                                         updateSchema({ ...schema, sync_frequency: value as DataWarehouseSyncInterval })

package.json

@@ -162,7 +162,7 @@
         "pmtiles": "^2.11.0",
         "postcss": "^8.4.31",
         "postcss-preset-env": "^9.3.0",
-        "posthog-js": "1.210.0",
+        "posthog-js": "1.210.2",
         "posthog-js-lite": "3.0.0",
         "prettier": "^2.8.8",
         "prop-types": "^15.7.2",

posthog/hogql/database/database.py

@@ -586,7 +586,7 @@ def serialize_database(
     warehouse_schemas = (
         list(
             ExternalDataSchema.objects.exclude(deleted=True)
-            .filter(table_id__in=[table.id for table in warehouse_tables])
+            .filter(team_id=context.team_id, table_id__in=[table.id for table in warehouse_tables])
             .all()
         )
         if len(warehouse_tables) > 0

posthog/temporal/data_imports/external_data_job.py

@@ -287,6 +287,7 @@ async def run(self, inputs: ExternalDataWorkflowInputs):
                 run_id=job_id,
                 schema_id=inputs.external_data_schema_id,
                 source_id=inputs.external_data_source_id,
+                reset_pipeline=inputs.reset_pipeline,
             )
 
             timeout_params = (

posthog/temporal/data_imports/workflow_activities/import_data_sync.py

@@ -2,7 +2,7 @@
 import uuid
 from datetime import datetime
 from dateutil import parser
-from typing import Any
+from typing import Any, Optional
 
 from django.conf import settings
 from django.db import close_old_connections
@@ -35,6 +35,7 @@ class ImportDataActivityInputs:
     schema_id: uuid.UUID
     source_id: uuid.UUID
     run_id: str
+    reset_pipeline: Optional[bool] = None
 
 
 def process_incremental_last_value(value: Any | None, field_type: IncrementalFieldType | None) -> Any | None:
@@ -92,7 +93,14 @@ def import_data_activity_sync(inputs: ImportDataActivityInputs):
 
         schema: ExternalDataSchema | None = model.schema
         assert schema is not None
-        reset_pipeline = schema.sync_type_config.get("reset_pipeline", False) is True
+
+        if inputs.reset_pipeline is not None:
+            reset_pipeline = inputs.reset_pipeline
+        else:
+            reset_pipeline = schema.sync_type_config.get("reset_pipeline", False) is True
+
+        logger.debug(f"schema.sync_type_config = {schema.sync_type_config}")
+        logger.debug(f"reset_pipeline = {reset_pipeline}")
 
         schema = (
             ExternalDataSchema.objects.prefetch_related("source")
@@ -101,24 +109,26 @@ def import_data_activity_sync(inputs: ImportDataActivityInputs):
         )
 
         endpoints = [schema.name]
+        processed_incremental_last_value = None
 
-        if settings.TEMPORAL_TASK_QUEUE == DATA_WAREHOUSE_TASK_QUEUE_V2:
-            # Get the V2 last value, if it's not set yet (e.g. the first run), then fallback to the V1 value
-            processed_incremental_last_value = process_incremental_last_value(
-                schema.sync_type_config.get("incremental_field_last_value_v2"),
-                schema.sync_type_config.get("incremental_field_type"),
-            )
+        if reset_pipeline is not True:
+            if settings.TEMPORAL_TASK_QUEUE == DATA_WAREHOUSE_TASK_QUEUE_V2:
+                # Get the V2 last value, if it's not set yet (e.g. the first run), then fallback to the V1 value
+                processed_incremental_last_value = process_incremental_last_value(
+                    schema.sync_type_config.get("incremental_field_last_value_v2"),
+                    schema.sync_type_config.get("incremental_field_type"),
+                )
 
-            if processed_incremental_last_value is None:
+                if processed_incremental_last_value is None:
+                    processed_incremental_last_value = process_incremental_last_value(
+                        schema.sync_type_config.get("incremental_field_last_value"),
+                        schema.sync_type_config.get("incremental_field_type"),
+                    )
+            else:
                 processed_incremental_last_value = process_incremental_last_value(
                     schema.sync_type_config.get("incremental_field_last_value"),
                     schema.sync_type_config.get("incremental_field_type"),
                 )
-        else:
-            processed_incremental_last_value = process_incremental_last_value(
-                schema.sync_type_config.get("incremental_field_last_value"),
-                schema.sync_type_config.get("incremental_field_type"),
-            )
 
         if schema.is_incremental:
             logger.debug(f"Incremental last value being used is: {processed_incremental_last_value}")

posthog/temporal/utils.py

@@ -1,4 +1,5 @@
 import dataclasses
+from typing import Optional
 import uuid
 
 
@@ -9,3 +10,4 @@ class ExternalDataWorkflowInputs:
     external_data_source_id: uuid.UUID
     external_data_schema_id: uuid.UUID | None = None
     billable: bool = True
+    reset_pipeline: Optional[bool] = None

products/llm_observability/frontend/ConversationDisplay/ConversationMessagesDisplay.tsx

@@ -67,11 +67,11 @@ export function LLMMessageDisplay({ message, isOutput }: { message: CompatMessag
                 'rounded border text-default',
                 isOutput
                     ? 'bg-[var(--bg-fill-success-tertiary)]'
-                    : role === 'system'
-                    ? 'bg-[var(--bg-fill-tertiary)]'
                     : role === 'user'
-                    ? 'bg-[var(--bg-fill-primary)]'
-                    : 'bg-[var(--bg-fill-info-tertiary)]'
+                    ? 'bg-[var(--bg-fill-tertiary)]'
+                    : role === 'assistant'
+                    ? 'bg-[var(--bg-fill-info-tertiary)]'
+                    : null // e.g. system
             )}
         >
             <div className="flex items-center gap-1 w-full px-2 h-6 text-xs font-medium">

products/llm_observability/frontend/LLMObservabilityTracesScene.tsx

@@ -62,12 +62,14 @@ export function LLMObservabilityTraces(): JSX.Element {
 const IDColumn: QueryContextColumnComponent = ({ record }) => {
     const row = record as LLMTrace
     return (
-        <Link
-            className="ph-no-capture font-semibold"
-            to={urls.llmObservabilityTrace(row.id, { timestamp: removeMilliseconds(row.createdAt) })}
-        >
-            {row.id}
-        </Link>
+        <strong>
+            <Link
+                className="ph-no-capture"
+                to={urls.llmObservabilityTrace(row.id, { timestamp: removeMilliseconds(row.createdAt) })}
+            >
+                {row.id.slice(0, 4)}...{row.id.slice(-4)}
+            </Link>
+        </strong>
     )
 }