From 5337e03bf01423021baa899a220a4e0b1cf23ac0 Mon Sep 17 00:00:00 2001 From: Lars Grammel Date: Wed, 19 Apr 2023 23:51:38 +0800 Subject: [PATCH] Add cost calculation. --- README.md | 33 ++++++++++--- examples/wikipedia/README.md | 1 + examples/wikipedia/src/runWikipediaAgent.ts | 21 +++++++- packages/agent/README.md | 37 ++++++++++---- .../src/agent/calculateRunCostInMillicent.ts | 16 ++++++ packages/agent/src/agent/index.ts | 1 + .../src/agent/observer/combineObservers.ts | 47 ++++++++++++++++++ packages/agent/src/agent/observer/index.ts | 1 + .../openai/api/generateChatCompletion.ts | 2 +- ...ompletion.ts => generateTextCompletion.ts} | 14 +++--- .../agent/src/provider/openai/api/index.ts | 2 +- .../openai/calculateCallCostInMillicent.ts | 28 ----------- .../calculateChatCompletionCostInMillicent.ts | 25 ++++++++++ .../calculateOpenAiCallCostInMillicent.ts | 49 +++++++++++++++++++ .../calculateTextCompletionCostInMillicent.ts | 23 +++++++++ packages/agent/src/provider/openai/index.ts | 4 +- .../{completionModel.ts => textModel.ts} | 18 +++---- 17 files changed, 255 insertions(+), 67 deletions(-) create mode 100644 packages/agent/src/agent/calculateRunCostInMillicent.ts create mode 100644 packages/agent/src/agent/observer/combineObservers.ts rename packages/agent/src/provider/openai/api/{generateCompletion.ts => generateTextCompletion.ts} (79%) delete mode 100644 packages/agent/src/provider/openai/calculateCallCostInMillicent.ts create mode 100644 packages/agent/src/provider/openai/calculateChatCompletionCostInMillicent.ts create mode 100644 packages/agent/src/provider/openai/calculateOpenAiCallCostInMillicent.ts create mode 100644 packages/agent/src/provider/openai/calculateTextCompletionCostInMillicent.ts rename packages/agent/src/provider/openai/{completionModel.ts => textModel.ts} (53%) diff --git a/README.md b/README.md index 5e66445..a98a663 100644 --- a/README.md +++ b/README.md @@ -4,8 +4,7 @@ JS Agent is a composable and extensible framework for creating GPT agents with JavaScript and TypeScript. -While creating an agent prototype is easy, increasing its reliability and robustness is very hard -and requires considerable experimentation. JS Agent provides building blocks and tooling to help you develop rock-solid agents faster. +While creating an agent prototype is easy, increasing its reliability and robustness is difficult and requires considerable experimentation. JS Agent provides robust building blocks and tooling to help you develop rock-solid agents faster. **⚠️ JS Agent is currently in its initial experimental phase. Prior to reaching version 0.1, there may breaking changes in each release.** @@ -20,9 +19,10 @@ See examples below for details on how to implement and run an agent. ## Features - Agent definition and execution - - Observable agent runs (to support console output, UIs, server runs, webapps, etc.) - - Recording of LLM calls for each agent run - - Controller to limit the number of steps + - Observe agent runs (to support console output, UIs, server runs, webapps, etc.) + - Record all LLM calls of an agent run + - Calculate the cost of LLM calls and agent runs + - Stop agent runs when certain criteria are met, e.g. to limit the number of steps - Supported LLM models - OpenAI text completion models (`text-davinci-003` etc.) - OpenAI chat completion models (`gpt-4`, `gpt-3.5-turbo`) @@ -145,9 +145,26 @@ export async function runWikipediaAgent({ }), }), controller: $.agent.controller.maxSteps(20), - observer: $.agent.showRunInConsole({ - name: "Wikipedia Agent", - }), + observer: $.agent.observer.combineObservers( + $.agent.observer.showRunInConsole({ name: "Wikipedia Agent" }), + { + async onRunFinished({ run }) { + const runCostInMillicent = await $.agent.calculateRunCostInMillicent({ + run, + }); + + console.log( + `Run cost: $${(runCostInMillicent / 1000 / 100).toFixed(2)}` + ); + + console.log( + `LLM calls: ${ + run.recordedCalls.filter((call) => call.success).length + }` + ); + }, + } + ), }); } ``` diff --git a/examples/wikipedia/README.md b/examples/wikipedia/README.md index 1bd62b6..4538772 100644 --- a/examples/wikipedia/README.md +++ b/examples/wikipedia/README.md @@ -8,6 +8,7 @@ Answers questions using Wikipedia articles. It searches using a Programmable Sea - Custom tool configuration (`readWikipediaArticleAction`, `searchWikipediaAction`) - `GenerateNextStepLoop` loop with tools and custom prompt - `maxSteps` `RunController` to limit the maximum number of steps +- Cost calculation and extracting information from LLM calls after the run ## Usage diff --git a/examples/wikipedia/src/runWikipediaAgent.ts b/examples/wikipedia/src/runWikipediaAgent.ts index 94f83d7..7797e36 100644 --- a/examples/wikipedia/src/runWikipediaAgent.ts +++ b/examples/wikipedia/src/runWikipediaAgent.ts @@ -80,6 +80,25 @@ export async function runWikipediaAgent({ }), }), controller: $.agent.controller.maxSteps(20), - observer: $.agent.observer.showRunInConsole({ name: "Wikipedia Agent" }), + observer: $.agent.observer.combineObservers( + $.agent.observer.showRunInConsole({ name: "Wikipedia Agent" }), + { + async onRunFinished({ run }) { + const runCostInMillicent = await $.agent.calculateRunCostInMillicent({ + run, + }); + + console.log( + `Run cost: $${(runCostInMillicent / 1000 / 100).toFixed(2)}` + ); + + console.log( + `LLM calls: ${ + run.recordedCalls.filter((call) => call.success).length + }` + ); + }, + } + ), }); } diff --git a/packages/agent/README.md b/packages/agent/README.md index a9cc72f..47814bd 100644 --- a/packages/agent/README.md +++ b/packages/agent/README.md @@ -1,9 +1,8 @@ -# JS Agent: Build AI Agents with JS & TS +# JS Agent: Build GPT Agents with JS & TS -JS Agent is a composable and extensible framework for creating AI agents with JavaScript and TypeScript. +JS Agent is a composable and extensible framework for creating GPT agents with JavaScript and TypeScript. -While creating an AI agent prototype is easy, increasing its reliability and robustness is very hard -and requires considerable experimentation. JS Agent provides building blocks and tooling to help you develop rock-solid agents faster. +While creating an agent prototype is easy, increasing its reliability and robustness is difficult and requires considerable experimentation. JS Agent provides robust building blocks and tooling to help you develop rock-solid agents faster. **⚠️ JS Agent is currently in its initial experimental phase. Prior to reaching version 0.1, there may breaking changes in each release.** @@ -18,9 +17,10 @@ See examples below for details on how to implement and run an agent. ## Features - Agent definition and execution - - Observable agent runs (to support console output, UIs, server runs, webapps, etc.) - - Recording of LLM calls for each agent run - - Controller to limit the number of steps + - Observe agent runs (to support console output, UIs, server runs, webapps, etc.) + - Record all LLM calls of an agent run + - Calculate the cost of LLM calls and agent runs + - Stop agent runs when certain criteria are met, e.g. to limit the number of steps - Supported LLM models - OpenAI text completion models (`text-davinci-003` etc.) - OpenAI chat completion models (`gpt-4`, `gpt-3.5-turbo`) @@ -143,9 +143,26 @@ export async function runWikipediaAgent({ }), }), controller: $.agent.controller.maxSteps(20), - observer: $.agent.showRunInConsole({ - name: "Wikipedia Agent", - }), + observer: $.agent.observer.combineObservers( + $.agent.observer.showRunInConsole({ name: "Wikipedia Agent" }), + { + async onRunFinished({ run }) { + const runCostInMillicent = await $.agent.calculateRunCostInMillicent({ + run, + }); + + console.log( + `Run cost: $${(runCostInMillicent / 1000 / 100).toFixed(2)}` + ); + + console.log( + `LLM calls: ${ + run.recordedCalls.filter((call) => call.success).length + }` + ); + }, + } + ), }); } ``` diff --git a/packages/agent/src/agent/calculateRunCostInMillicent.ts b/packages/agent/src/agent/calculateRunCostInMillicent.ts new file mode 100644 index 0000000..5c799d7 --- /dev/null +++ b/packages/agent/src/agent/calculateRunCostInMillicent.ts @@ -0,0 +1,16 @@ +import { calculateOpenAiCallCostInMillicent } from "../provider/openai/calculateOpenAiCallCostInMillicent"; +import { Run } from "./Run"; + +export const calculateRunCostInMillicent = async ({ run }: { run: Run }) => { + const callCostsInMillicent = run.recordedCalls.map((call) => { + if (call.success && call.metadata.model.vendor === "openai") { + return calculateOpenAiCallCostInMillicent(call); + } + return undefined; + }); + + return callCostsInMillicent.reduce( + (sum: number, cost) => sum + (cost ?? 0), + 0 + ); +}; diff --git a/packages/agent/src/agent/index.ts b/packages/agent/src/agent/index.ts index 0eb6f3b..4215fbe 100644 --- a/packages/agent/src/agent/index.ts +++ b/packages/agent/src/agent/index.ts @@ -1,6 +1,7 @@ export * from "./GenerateCall"; export * from "./Run"; export * from "./RunContext"; +export * from "./calculateRunCostInMillicent"; export * as controller from "./controller/index"; export * as observer from "./observer/index"; export * from "./runAgent"; diff --git a/packages/agent/src/agent/observer/combineObservers.ts b/packages/agent/src/agent/observer/combineObservers.ts new file mode 100644 index 0000000..6408f06 --- /dev/null +++ b/packages/agent/src/agent/observer/combineObservers.ts @@ -0,0 +1,47 @@ +import { RunObserver } from "./RunObserver"; + +export const combineObservers = (...observers: RunObserver[]): RunObserver => ({ + onRunStarted: ({ run }) => { + observers.forEach((observer) => observer.onRunStarted?.({ run })); + }, + + onRunFinished: ({ run, result }) => { + observers.forEach((observer) => observer.onRunFinished?.({ run, result })); + }, + + onStepGenerationStarted: ({ run }) => { + observers.forEach((observer) => + observer.onStepGenerationStarted?.({ run }) + ); + }, + + onStepGenerationFinished: ({ run, generatedText, step }) => { + observers.forEach((observer) => + observer.onStepGenerationFinished?.({ run, generatedText, step }) + ); + }, + + onLoopIterationStarted: ({ run, loop }) => { + observers.forEach((observer) => + observer.onLoopIterationStarted?.({ run, loop }) + ); + }, + + onLoopIterationFinished: ({ run, loop }) => { + observers.forEach((observer) => + observer.onLoopIterationFinished?.({ run, loop }) + ); + }, + + onStepExecutionStarted: ({ run, step }) => { + observers.forEach((observer) => + observer.onStepExecutionStarted?.({ run, step }) + ); + }, + + onStepExecutionFinished: ({ run, step, result }) => { + observers.forEach((observer) => + observer.onStepExecutionFinished?.({ run, step, result }) + ); + }, +}); diff --git a/packages/agent/src/agent/observer/index.ts b/packages/agent/src/agent/observer/index.ts index 0f16e76..37313a2 100644 --- a/packages/agent/src/agent/observer/index.ts +++ b/packages/agent/src/agent/observer/index.ts @@ -1,2 +1,3 @@ export * from "./RunObserver"; +export * from "./combineObservers"; export * from "./showRunInConsole"; diff --git a/packages/agent/src/provider/openai/api/generateChatCompletion.ts b/packages/agent/src/provider/openai/api/generateChatCompletion.ts index 4a7e676..7dbc267 100644 --- a/packages/agent/src/provider/openai/api/generateChatCompletion.ts +++ b/packages/agent/src/provider/openai/api/generateChatCompletion.ts @@ -2,7 +2,7 @@ import axios from "axios"; import zod from "zod"; import { OpenAIChatMessage } from "../OpenAIChatMessage"; -const OpenAIChatCompletionSchema = zod.object({ +export const OpenAIChatCompletionSchema = zod.object({ id: zod.string(), object: zod.literal("chat.completion"), created: zod.number(), diff --git a/packages/agent/src/provider/openai/api/generateCompletion.ts b/packages/agent/src/provider/openai/api/generateTextCompletion.ts similarity index 79% rename from packages/agent/src/provider/openai/api/generateCompletion.ts rename to packages/agent/src/provider/openai/api/generateTextCompletion.ts index 26b9be2..7613a32 100644 --- a/packages/agent/src/provider/openai/api/generateCompletion.ts +++ b/packages/agent/src/provider/openai/api/generateTextCompletion.ts @@ -1,7 +1,7 @@ import axios from "axios"; import zod from "zod"; -const OpenAICompletionSchema = zod.object({ +export const OpenAITextCompletionSchema = zod.object({ id: zod.string(), object: zod.literal("text_completion"), created: zod.number(), @@ -21,9 +21,9 @@ const OpenAICompletionSchema = zod.object({ }), }); -export type OpenAICompletion = zod.infer; +export type OpenAITextCompletion = zod.infer; -export type OpenAICompletionModel = +export type OpenAITextCompletionModel = | "text-davinci-003" | "text-davinci-002" | "code-davinci-002" @@ -36,7 +36,7 @@ export type OpenAICompletionModel = | "babbage" | "ada"; -export async function generateCompletion({ +export async function generateTextCompletion({ apiKey, prompt, model, @@ -48,13 +48,13 @@ export async function generateCompletion({ }: { apiKey: string; prompt: string; - model: OpenAICompletionModel; + model: OpenAITextCompletionModel; n?: number; temperature?: number; maxTokens?: number; presencePenalty?: number; frequencyPenalty?: number; -}): Promise { +}): Promise { const response = await axios.post( "https://api.openai.com/v1/completions", JSON.stringify({ @@ -74,5 +74,5 @@ export async function generateCompletion({ } ); - return OpenAICompletionSchema.parse(response.data); + return OpenAITextCompletionSchema.parse(response.data); } diff --git a/packages/agent/src/provider/openai/api/index.ts b/packages/agent/src/provider/openai/api/index.ts index b2e042f..6bdb14c 100644 --- a/packages/agent/src/provider/openai/api/index.ts +++ b/packages/agent/src/provider/openai/api/index.ts @@ -1,2 +1,2 @@ export * from "./generateChatCompletion.js"; -export * from "./generateCompletion.js"; +export * from "./generateTextCompletion.js"; diff --git a/packages/agent/src/provider/openai/calculateCallCostInMillicent.ts b/packages/agent/src/provider/openai/calculateCallCostInMillicent.ts deleted file mode 100644 index 47e13c2..0000000 --- a/packages/agent/src/provider/openai/calculateCallCostInMillicent.ts +++ /dev/null @@ -1,28 +0,0 @@ -import { OpenAIChatCompletionModel } from "./api/generateChatCompletion"; - -export function calculateCallCostInMillicent(parameters: { - model: OpenAIChatCompletionModel; - usage: { - prompt_tokens: number; - completion_tokens: number; - }; -}) { - const model = parameters.model; - - switch (model) { - case "gpt-4": { - const promptTokenCount = parameters.usage.prompt_tokens; - const completionTokenCount = parameters.usage.completion_tokens; - return promptTokenCount * 3 + completionTokenCount * 6; // see https://openai.com/pricing - } - case "gpt-3.5-turbo": { - const promptTokenCount = parameters.usage.prompt_tokens; - const completionTokenCount = parameters.usage.completion_tokens; - return promptTokenCount * 0.2 + completionTokenCount * 0.2; // see https://openai.com/pricing - } - default: { - const _exhaustiveCheck: never = model; - throw new Error(`Unknown model: ${model}`); - } - } -} diff --git a/packages/agent/src/provider/openai/calculateChatCompletionCostInMillicent.ts b/packages/agent/src/provider/openai/calculateChatCompletionCostInMillicent.ts new file mode 100644 index 0000000..e61ad80 --- /dev/null +++ b/packages/agent/src/provider/openai/calculateChatCompletionCostInMillicent.ts @@ -0,0 +1,25 @@ +import { + OpenAIChatCompletion, + OpenAIChatCompletionModel, +} from "./api/generateChatCompletion"; + +// see https://openai.com/pricing +const promptTokenCostInMillicent = { + "gpt-4": 3, + "gpt-3.5-turbo": 0.2, +}; + +const completionTokenCostInMillicent = { + "gpt-4": 6, + "gpt-3.5-turbo": 0.2, +}; + +export const calculateChatCompletionCostInMillicent = ({ + model, + completion, +}: { + model: OpenAIChatCompletionModel; + completion: OpenAIChatCompletion; +}) => + completion.usage.prompt_tokens * promptTokenCostInMillicent[model] + + completion.usage.completion_tokens * completionTokenCostInMillicent[model]; diff --git a/packages/agent/src/provider/openai/calculateOpenAiCallCostInMillicent.ts b/packages/agent/src/provider/openai/calculateOpenAiCallCostInMillicent.ts new file mode 100644 index 0000000..94f69b9 --- /dev/null +++ b/packages/agent/src/provider/openai/calculateOpenAiCallCostInMillicent.ts @@ -0,0 +1,49 @@ +import { GenerateCall } from "../../agent"; +import { OpenAIChatCompletionSchema } from "./api/generateChatCompletion"; +import { OpenAITextCompletionSchema } from "./api/generateTextCompletion"; +import { calculateChatCompletionCostInMillicent } from "./calculateChatCompletionCostInMillicent"; +import { calculateTextCompletionCostInMillicent } from "./calculateTextCompletionCostInMillicent"; + +export function calculateOpenAiCallCostInMillicent( + call: GenerateCall & { + success: true; + } +) { + if (call.metadata.model.vendor !== "openai") { + throw new Error(`Incorrect vendor: ${call.metadata.model.vendor}`); + } + + const model = call.metadata.model.name; + + switch (model) { + case "gpt-3.5-turbo": + case "gpt-4": { + return calculateChatCompletionCostInMillicent({ + model, + completion: OpenAIChatCompletionSchema.parse(call.rawOutput), + }); + } + + case "text-davinci-003": + case "text-davinci-003": + case "text-davinci-002": + case "code-davinci-002": + case "code-davinci-002": + case "text-curie-001": + case "text-babbage-001": + case "text-ada-001": + case "davinci": + case "curie": + case "babbage": + case "ada": { + return calculateTextCompletionCostInMillicent({ + model, + completion: OpenAITextCompletionSchema.parse(call.rawOutput), + }); + } + + default: { + throw new Error(`Unknown model: ${model}`); + } + } +} diff --git a/packages/agent/src/provider/openai/calculateTextCompletionCostInMillicent.ts b/packages/agent/src/provider/openai/calculateTextCompletionCostInMillicent.ts new file mode 100644 index 0000000..4c3f872 --- /dev/null +++ b/packages/agent/src/provider/openai/calculateTextCompletionCostInMillicent.ts @@ -0,0 +1,23 @@ +import { OpenAITextCompletion, OpenAITextCompletionModel } from "./api"; + +// see https://openai.com/pricing +const tokenCostInMillicent = { + davinci: 2, + "text-davinci-003": 2, + "text-davinci-002": 2, + curie: 0.2, + "text-curie-001": 0.2, + babbage: 0.05, + "text-babbage-001": 0.05, + ada: 0.04, + "text-ada-001": 0.04, + "code-davinci-002": 0, +}; + +export const calculateTextCompletionCostInMillicent = ({ + model, + completion, +}: { + model: OpenAITextCompletionModel; + completion: OpenAITextCompletion; +}) => tokenCostInMillicent[model] * completion.usage.total_tokens; diff --git a/packages/agent/src/provider/openai/index.ts b/packages/agent/src/provider/openai/index.ts index effe712..f1b1477 100644 --- a/packages/agent/src/provider/openai/index.ts +++ b/packages/agent/src/provider/openai/index.ts @@ -1,5 +1,5 @@ export * from "./OpenAIChatMessage.js"; export * as api from "./api"; -export * from "./calculateCallCostInMillicent.js"; +export * from "./calculateOpenAiCallCostInMillicent.js"; export * from "./chatModel.js"; -export * from "./completionModel.js"; +export * from "./textModel.js"; diff --git a/packages/agent/src/provider/openai/completionModel.ts b/packages/agent/src/provider/openai/textModel.ts similarity index 53% rename from packages/agent/src/provider/openai/completionModel.ts rename to packages/agent/src/provider/openai/textModel.ts index 5861f6f..db4e0dc 100644 --- a/packages/agent/src/provider/openai/completionModel.ts +++ b/packages/agent/src/provider/openai/textModel.ts @@ -1,9 +1,9 @@ import { GeneratorModel } from "../../text/generate/generate"; import { - OpenAICompletion, - OpenAICompletionModel, - generateCompletion, -} from "./api/generateCompletion"; + OpenAITextCompletion, + OpenAITextCompletionModel, + generateTextCompletion, +} from "./api/generateTextCompletion"; export const completionModel = ({ apiKey, @@ -12,21 +12,21 @@ export const completionModel = ({ maxTokens, }: { apiKey: string; - model: OpenAICompletionModel; + model: OpenAITextCompletionModel; temperature?: number; maxTokens?: number; -}): GeneratorModel => ({ +}): GeneratorModel => ({ vendor: "openai", name: model, - generate: async (input: string): Promise => - generateCompletion({ + generate: async (input: string): Promise => + generateTextCompletion({ apiKey, prompt: input, model, temperature, maxTokens, }), - extractOutput: async (rawOutput: OpenAICompletion): Promise => { + extractOutput: async (rawOutput: OpenAITextCompletion): Promise => { return rawOutput.choices[0].text; }, });