forked from lobehub/lobe-chat
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
✨ feat: spport qwen-vl and tool call for qwen (lobehub#3114)
* feat: spport qwen-vl and tool call for qwen * fix: make transformResponseToStream a util for testability * test: append unit test for non-streaming response * test: update unit-test against LobeQwenAI models
- Loading branch information
Showing
7 changed files
with
763 additions
and
87 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,28 +1,128 @@ | ||
| import OpenAI from 'openai'; | ||
|
|
||
| import { ModelProvider } from '../types'; | ||
| import { LobeOpenAICompatibleFactory } from '../utils/openaiCompatibleFactory'; | ||
|
|
||
| export const LobeQwenAI = LobeOpenAICompatibleFactory({ | ||
| baseURL: 'https://dashscope.aliyuncs.com/compatible-mode/v1', | ||
| chatCompletion: { | ||
| handlePayload: (payload) => { | ||
| const top_p = payload.top_p; | ||
| return { | ||
| ...payload, | ||
| stream: payload.stream ?? true, | ||
| top_p: top_p && top_p >= 1 ? 0.9999 : top_p, | ||
| } as OpenAI.ChatCompletionCreateParamsStreaming; | ||
| }, | ||
| }, | ||
| constructorOptions: { | ||
| defaultHeaders: { | ||
| 'Content-Type': 'application/json', | ||
| }, | ||
| }, | ||
| debug: { | ||
| chatCompletion: () => process.env.DEBUG_QWEN_CHAT_COMPLETION === '1', | ||
| }, | ||
|
|
||
| provider: ModelProvider.Qwen, | ||
| }); | ||
| import { omit } from 'lodash-es'; | ||
| import OpenAI, { ClientOptions } from 'openai'; | ||
|
|
||
| import Qwen from '@/config/modelProviders/qwen'; | ||
|
|
||
| import { LobeOpenAICompatibleRuntime, LobeRuntimeAI } from '../BaseAI'; | ||
| import { AgentRuntimeErrorType } from '../error'; | ||
| import { ChatCompetitionOptions, ChatStreamPayload, ModelProvider } from '../types'; | ||
| import { AgentRuntimeError } from '../utils/createError'; | ||
| import { debugStream } from '../utils/debugStream'; | ||
| import { handleOpenAIError } from '../utils/handleOpenAIError'; | ||
| import { transformResponseToStream } from '../utils/openaiCompatibleFactory'; | ||
| import { StreamingResponse } from '../utils/response'; | ||
| import { QwenAIStream } from '../utils/streams'; | ||
|
|
||
| const DEFAULT_BASE_URL = 'https://dashscope.aliyuncs.com/compatible-mode/v1'; | ||
|
|
||
| /** | ||
| * Use DashScope OpenAI compatible mode for now. | ||
| * DashScope OpenAI [compatible mode](https://help.aliyun.com/zh/dashscope/developer-reference/tongyi-qianwen-vl-plus-api) currently supports base64 image input for vision models e.g. qwen-vl-plus. | ||
| * You can use images input either: | ||
| * 1. Use qwen-vl-* out of box with base64 image_url input; | ||
| * or | ||
| * 2. Set S3-* enviroment variables properly to store all uploaded files. | ||
| */ | ||
| export class LobeQwenAI extends LobeOpenAICompatibleRuntime implements LobeRuntimeAI { | ||
| client: OpenAI; | ||
| baseURL: string; | ||
|
|
||
| constructor({ | ||
| apiKey, | ||
| baseURL = DEFAULT_BASE_URL, | ||
| ...res | ||
| }: ClientOptions & Record<string, any> = {}) { | ||
| super(); | ||
| if (!apiKey) throw AgentRuntimeError.createError(AgentRuntimeErrorType.InvalidProviderAPIKey); | ||
| this.client = new OpenAI({ apiKey, baseURL, ...res }); | ||
| this.baseURL = this.client.baseURL; | ||
| } | ||
|
|
||
| async models() { | ||
| return Qwen.chatModels; | ||
| } | ||
|
|
||
| async chat(payload: ChatStreamPayload, options?: ChatCompetitionOptions) { | ||
| try { | ||
| const params = this.buildCompletionParamsByModel(payload); | ||
|
|
||
| const response = await this.client.chat.completions.create( | ||
| params as OpenAI.ChatCompletionCreateParamsStreaming & { result_format: string }, | ||
| { | ||
| headers: { Accept: '*/*' }, | ||
| signal: options?.signal, | ||
| }, | ||
| ); | ||
|
|
||
| if (params.stream) { | ||
| const [prod, debug] = response.tee(); | ||
|
|
||
| if (process.env.DEBUG_QWEN_CHAT_COMPLETION === '1') { | ||
| debugStream(debug.toReadableStream()).catch(console.error); | ||
| } | ||
|
|
||
| return StreamingResponse(QwenAIStream(prod, options?.callback), { | ||
| headers: options?.headers, | ||
| }); | ||
| } | ||
|
|
||
| const stream = transformResponseToStream(response as unknown as OpenAI.ChatCompletion); | ||
|
|
||
| return StreamingResponse(QwenAIStream(stream, options?.callback), { | ||
| headers: options?.headers, | ||
| }); | ||
| } catch (error) { | ||
| if ('status' in (error as any)) { | ||
| switch ((error as Response).status) { | ||
| case 401: { | ||
| throw AgentRuntimeError.chat({ | ||
| endpoint: this.baseURL, | ||
| error: error as any, | ||
| errorType: AgentRuntimeErrorType.InvalidProviderAPIKey, | ||
| provider: ModelProvider.Qwen, | ||
| }); | ||
| } | ||
|
|
||
| default: { | ||
| break; | ||
| } | ||
| } | ||
| } | ||
| const { errorResult, RuntimeError } = handleOpenAIError(error); | ||
| const errorType = RuntimeError || AgentRuntimeErrorType.ProviderBizError; | ||
|
|
||
| throw AgentRuntimeError.chat({ | ||
| endpoint: this.baseURL, | ||
| error: errorResult, | ||
| errorType, | ||
| provider: ModelProvider.Qwen, | ||
| }); | ||
| } | ||
| } | ||
|
|
||
| private buildCompletionParamsByModel(payload: ChatStreamPayload) { | ||
| const { model, top_p, stream, messages, tools } = payload; | ||
| const isVisionModel = model.startsWith('qwen-vl'); | ||
|
|
||
| const params = { | ||
| ...payload, | ||
| messages, | ||
| result_format: 'message', | ||
| stream: !!tools?.length ? false : stream ?? true, | ||
| top_p: top_p && top_p >= 1 ? 0.999 : top_p, | ||
| }; | ||
|
|
||
| /* Qwen-vl models temporarily do not support parameters below. */ | ||
| /* Notice: `top_p` imposes significant impact on the result，the default 1 or 0.999 is not a proper choice. */ | ||
| return isVisionModel | ||
| ? omit( | ||
| params, | ||
| 'presence_penalty', | ||
| 'frequency_penalty', | ||
| 'temperature', | ||
| 'result_format', | ||
| 'top_p', | ||
| ) | ||
| : params; | ||
| } | ||
| } |
Oops, something went wrong.