diff --git a/README.ENG.md b/README.ENG.md index 1ee46a16..2a17f6a8 100644 --- a/README.ENG.md +++ b/README.ENG.md @@ -93,9 +93,8 @@ mweb = lazyllm.WebModule(ppl, port=23456).start().wait() ```python import lazyllm -from lazyllm import pipeline, parallel, Identity, warp, package -import time -import re, json +from lazyllm import pipeline, warp, bind +from lazyllm.components.formatter import JsonFormatter toc_prompt=""" You are now an intelligent assistant. Your task is to understand the user's input and convert the outline into a list of nested dictionaries. Each dictionary contains a `title` and a `describe`, where the `title` should clearly indicate the level using Markdown format, and the `describe` is a description and writing guide for that section. @@ -134,19 +133,18 @@ This is the expanded content for writing. Receive as follows: """ + +writer_prompt = {"system": completion_prompt, "user": '{"title": {title}, "describe": {describe}}'} ``` ```python -t1 = lazyllm.OnlineChatModule(source="openai", stream=False, prompter=ChatPrompter(instruction=toc_prompt)) -t2 = lazyllm.OnlineChatModule(source="openai", stream=False, prompter=ChatPrompter(instruction=completion_prompt)) - -spliter = lambda s: tuple(eval(re.search(r'\[\s*\{.*\}\s*\]', s['message']['content'], re.DOTALL).group())) -writter = pipeline(lambda d: json.dumps(d, ensure_ascii=False), t2, lambda d : d['message']['content']) -collector = lambda dict_tuple, repl_tuple: "\n".join([v for d in [{**d, "describe": repl_tuple[i]} for i, d in enumerate(dict_tuple)] for v in d.values()]) -m = pipeline(t1, spliter, parallel(Identity, warp(writter)), collector) +with pipeline() as ppl: + ppl.outline_writer = lazyllm.OnlineChatModule(source="openai", stream=False).formatter(JsonFormatter()).prompt(toc_prompt) + ppl.story_generater = warp(lazyllm.OnlineChatModule(source="openai", stream=False).prompt(writer_prompt)) + ppl.synthesizer = (lambda *storys, outlines: "\n".join([f"{o['title']}\n{s}" for s, o in zip(storys, outlines)])) | bind(outlines=ppl.outline_writer) -print(m({'query': 'Please help me write an article about the application of artificial intelligence in the medical field.'})) +print(ppl({'query': 'Please help me write an article about the application of artificial intelligence in the medical field.'})) ``` ## What can LazyLLM do diff --git a/README.md b/README.md index 522101d8..3d88235c 100644 --- a/README.md +++ b/README.md @@ -90,9 +90,8 @@ mweb = lazyllm.WebModule(ppl, port=23456).start().wait() ```python import lazyllm -from lazyllm import pipeline, parallel, Identity, warp, package -import time -import re, json +from lazyllm import pipeline, warp, bind +from lazyllm.components.formatter import JsonFormatter toc_prompt=""" 你现在是一个智能助手。你的任务是理解用户的输入,将大纲以列表嵌套字典的列表。每个字典包含一个 `title` 和 `describe`,其中 `title` 中需要用Markdown格式标清层级,`describe` `describe` 是对该段的描述和写作指导。 @@ -129,19 +128,18 @@ completion_prompt=""" 接收如下: """ + +writer_prompt = {"system": completion_prompt, "user": '{"title": {title}, "describe": {describe}}'} ``` ```python -t1 = lazyllm.OnlineChatModule(source="openai", stream=False, prompter=ChatPrompter(instruction=toc_prompt)) -t2 = lazyllm.OnlineChatModule(source="openai", stream=False, prompter=ChatPrompter(instruction=completion_prompt)) - -spliter = lambda s: tuple(eval(re.search(r'\[\s*\{.*\}\s*\]', s['message']['content'], re.DOTALL).group())) -writter = pipeline(lambda d: json.dumps(d, ensure_ascii=False), t2, lambda d : d['message']['content']) -collector = lambda dict_tuple, repl_tuple: "\n".join([v for d in [{**d, "describe": repl_tuple[i]} for i, d in enumerate(dict_tuple)] for v in d.values()]) -m = pipeline(t1, spliter, parallel(Identity, warp(writter)), collector) +with pipeline() as ppl: + ppl.outline_writer = lazyllm.OnlineChatModule(source="openai", stream=False).formatter(JsonFormatter()).prompt(toc_prompt) + ppl.story_generater = warp(lazyllm.OnlineChatModule(source="openai", stream=False).prompt(writer_prompt)) + ppl.synthesizer = (lambda *storys, outlines: "\n".join([f"{o['title']}\n{s}" for s, o in zip(storys, outlines)])) | bind(outlines=ppl.outline_writer) -print(m({'query':'请帮我写一篇关于人工智能在医疗领域应用的文章。'})) +print(ppl({'query':'请帮我写一篇关于人工智能在医疗领域应用的文章。'})) ``` ## 四、功能点 diff --git a/docs/source/api/components.rst b/docs/source/api/components.rst index 82505a3f..d2cff17e 100644 --- a/docs/source/api/components.rst +++ b/docs/source/api/components.rst @@ -60,3 +60,18 @@ ModelDownloader .. autoclass:: lazyllm.components.ModelDownloader :members: :exclude-members: + +Formatter +========== + +.. autoclass:: lazyllm.components.formatter.LazyLLMFormatterBase + :members: + :exclude-members: + +.. autoclass:: lazyllm.components.JsonFormatter + :members: + :exclude-members: + +.. autoclass:: lazyllm.components.EmptyFormatter + :members: + :exclude-members: diff --git a/docs/source/best_practice/prompt.rst b/docs/source/best_practice/prompt.rst index 05356a9d..9f7058a1 100644 --- a/docs/source/best_practice/prompt.rst +++ b/docs/source/best_practice/prompt.rst @@ -53,9 +53,10 @@ LazyLLM Prompter的设计思路 - PrompterTemplate中可选的字段有: - system: 系统提示,一般会读取模型的归属信息并进行设置,如不设置默认为 ``You are an AI-Agent developed by LazyLLM.`` 。 - - instruction: 任务指令,由 ``InstructionTemplate`` 拼接用户的输入得到。这个是应用开发者需要着重了解的字段。 + - instruction: 任务指令,由 ``InstructionTemplate`` 拼接用户的输入得到。这个是应用开发者需要着重了解的字段。如果instruction是字符串,则默认是系统指令,如果是字典,且其键值只能是 ``system`` 和 ``user`` 。``system`` 指定的是系统级指令, ``user`` 指定的是用户级指令。 - history: 历史对话,由用户的输入得到,格式为 ``[[a, b], [c, d]]`` 或 ``[{"role": "user", "content": ""}, {"role": "assistant", "content": ""}]`` - tools: 可以使用的工具,在构造 ``prompter`` 时传入或者由用户使用时传入,当构造 ``prompter`` 时定义了工具之后,将禁止用户使用时再次传入。格式为 ``[{"type": "function", "function": {"name": "", "description": "", "parameters": {}, "required": []}]`` + - user: 用户级指令,可选指令,由用户通过instruction指定。 - sos: ``start of system`` , 标志着系统提示的开始,该符号由模型填入,开发者和用户均无需考虑 - eos: ``end of system`` , 标志着系统提示的结束,该符号由模型填入,开发者和用户均无需考虑 - soh: ``start of human`` , 标志着用户输入的开始,常用于多轮对话中作为分隔符。该符号由模型填入,开发者和用户均无需考虑 @@ -63,8 +64,8 @@ LazyLLM Prompter的设计思路 - soa: ``start of assistant`` , 标志着模型输出的开始,常用于多轮对话中作为分隔符。该符号由模型填入,开发者和用户均无需考虑 - eoa: ``end of assistant`` , 标志着模型输出的结束,常用于多轮对话中作为分隔符。该符号由模型填入,开发者和用户均无需考虑 - ``TrainableModule`` 所使用的内置的Prompt的拼接规则如下: - - AlpacaPrompter: ``{system}\n{instruction}\n{tools}### Response:\n`` - - ChatPrompter: ``{sos}{system}{instruction}{tools}{eos}\n\n{history}\n{soh}\n{input}\n{eoh}{soa}\n`` + - AlpacaPrompter: ``{system}\n{instruction}\n{tools}\n{user}### Response:\n`` + - ChatPrompter: ``{sos}{system}{instruction}{tools}{eos}\n\n{history}\n{soh}\n{user}{input}\n{eoh}{soa}\n`` - ``OnlineChatModule`` 的输出格式为: ``dict(messages=[{"role": "system", "content": ""}, {"role": "user", "content": ""}, ...], tools=[])`` .. note:: @@ -74,7 +75,7 @@ LazyLLM Prompter的设计思路 **InstructionTemplate**: 每个Prompter内置的,用于结合用户输入的 ``instruction`` ,产生最终的 ``instruction`` 的模板。 ``InstructionTemplate`` 中的用到的2个字段是: -- ``instruction`` : 由开发者在构造 ``Prompter`` 时传入,可带若干个待填充的槽位,用于填充用户的输入。 +- ``instruction`` : 由开发者在构造 ``Prompter`` 时传入,可带若干个待填充的槽位,用于填充用户的输入。或者指定系统级指令和用户级指令,当指定用户级指令时,需要使用字典类型,且键值为 ``user`` 和 ``system`` 。 - ``extro_keys`` : 需要用户调用大模型时额外提供的信息,有开发者在构造 ``Prompter`` 时传入,会自动转换成 ``instruction`` 中的槽位。 .. note:: @@ -105,11 +106,11 @@ Prompt生成过程解析 "Below is an instruction that describes a task, paired with extra messages such as input that provides " "further context if possible. Write a response that appropriately completes the request.\\n\\n ### " "Instruction:\\n 你是一个由LazyLLM开发的知识问答助手,你的任务是根据提供的上下文信息来回答用户的问题。上下文信息是背景," - "用户的问题是输入, 现在请你做出回答。### Response:\\n}" + "用户的问题是问题, 现在请你做出回答。### Response:\\n}" 4. ``AlpacaPrompter`` 读取 ``system`` 和 ``tools`` 字段,其中 ``system`` 字段由 ``Module`` 设置,而 ``tools`` 字段则会在后面的 :ref:`bestpractice.prompt.tools` 一节中介绍。 5. 如果 ``prompter`` 的结果用于线上模型( ``OnlineChatModule`` ),则不会再进一步拼接 ``PromptTemplate`` ,而是会直接得到一个dict,即 ``{'messages': [{'role': 'system', 'content': 'You are an AI-Agent developed by LazyLLM.\nBelow is an instruction that describes a task, paired with extra messages such as input that provides further context if possible. Write a response that appropriately completes the request.\n\n ### Instruction:\n你是一个由LazyLLM开发的知识问答助手,你的任务是根据提供的上下文信息来回答用户的问题。上下文信息是背景,用户的问题是输入,现在请你做出回答。\n\n'}, {'role': 'user', 'content': ''}]}`` -6. 如果 ``prompter`` 的结果用于线下模型( ``TrainableModule`` ),则会通过 ``PromptTemplate`` 得到最终的结果: ``You are an AI-Agent developed by LazyLLM.\nBelow is an instruction that describes a task, paired with extra messages such as input that provides further context if possible. Write a response that appropriately completes the request.\n\n ### Instruction:\n你是一个由LazyLLM开发的知识问答助手,你的任务是根据提供的上下文信息来回答用户的问题。上下文信息是背景,用户的问题是输入,现在请你做出回答。\n\n\n### Response:\n`` +6. 如果 ``prompter`` 的结果用于线下模型( ``TrainableModule`` ),则会通过 ``PromptTemplate`` 得到最终的结果: ``You are an AI-Agent developed by LazyLLM.\nBelow is an instruction that describes a task, paired with extra messages such as input that provides further context if possible. Write a response that appropriately completes the request.\n\n ### Instruction:\n你是一个由LazyLLM开发的知识问答助手,你的任务是根据提供的上下文信息来回答用户的问题。上下文信息是背景,用户的问题是问题,现在请你做出回答。\n\n\n### Response:\n`` 定义和使用Prompter ------------------------- @@ -153,6 +154,7 @@ Query为string,而非dict - 当使用 ``ChatPrompter`` 时,不同于 ``AlpacaPrompter`` ,在 ``instruction`` 中定义槽位不是必须的。 - 如果不定义槽位,则输入会放到对话中作为用户的输入,在 ```` 和 ```` 之间。 - 如果像 ``AlpacaPrompter`` 一样定义了槽位,也可以任意取一个名字,此时输入会放到 ```` 字段中。 + - 如果 ``instruction`` 中指定了系统级指令和用户级指令,则在拼接完成后,系统级指令放在prompt_template中的{instruction}位置,用户级指令放在{user}位置。 .. _bestpractice.prompt.tools: @@ -286,4 +288,4 @@ Query为string,而非dict - ``TrainableModule`` 需要手动调用 ``start`` 以启动服务,想了解更多关于 ``TrainableModule`` 的用法,可以参考 :ref:`api.module` LazyLLM中内置的场景Prompt -------------------------- \ No newline at end of file +------------------------- diff --git a/lazyllm/__init__.py b/lazyllm/__init__.py index 080b6b97..4486e124 100644 --- a/lazyllm/__init__.py +++ b/lazyllm/__init__.py @@ -7,7 +7,8 @@ Loop as loop, Switch as switch, IFS as ifs, Warp as warp) from .components import (LazyLLMDataprocBase, LazyLLMFinetuneBase, LazyLLMDeployBase, LazyLLMValidateBase, register as component_register, Prompter, - AlpacaPrompter, ChatPrompter, FastapiApp) + AlpacaPrompter, ChatPrompter, FastapiApp, JsonFormatter) + from .module import (ModuleBase, UrlModule, TrainableModule, ActionModule, ServerModule, TrialModule, register as module_register, OnlineChatModule, OnlineEmbeddingModule) @@ -33,6 +34,7 @@ 'AlpacaPrompter', 'ChatPrompter', 'FastapiApp', + 'JsonFormatter', # flow 'LazyLLMFlowsBase', # pipeline, parallel diff --git a/lazyllm/common/common.py b/lazyllm/common/common.py index 637b64d8..a6fb5114 100644 --- a/lazyllm/common/common.py +++ b/lazyllm/common/common.py @@ -334,11 +334,11 @@ class LazyLlmRequest(struct): def split(self, flag=None): if flag is None: - assert len(self.kwargs) == 0 and isinstance(self.input, tuple), ( + assert len(self.kwargs) == 0 and isinstance(self.input, (tuple, list)), ( f'Only tuple input can be split automatically, your input is {self.input} <{type(self.input)}>') return [LazyLlmRequest(input=inp, global_parameters=self.global_parameters) for inp in self.input] elif isinstance(flag, int): - assert len(self.kwargs) == 0 and isinstance(self.input, tuple), ( + assert len(self.kwargs) == 0 and isinstance(self.input, (tuple, list)), ( f'Only tuple input can be split automatically, your input is {self.input} <{type(self.input)}>') assert flag == len(self.input), 'input size mismatch with split number' return [LazyLlmRequest(input=inp, global_parameters=self.global_parameters) for inp in self.input] @@ -346,7 +346,7 @@ def split(self, flag=None): if isinstance(self.input, dict): assert len(self.kwargs) == 0, 'Cannot provived input and kwargs at the same time for split' d = self.input - elif isinstance(self.input, tuple): + elif isinstance(self.input, (tuple, list)): return self.split(len(flag)) else: assert not self.input, 'Cannot provived input and kwargs at the same time for split' diff --git a/lazyllm/common/logger.py b/lazyllm/common/logger.py index 8a9cc15e..9f951f46 100644 --- a/lazyllm/common/logger.py +++ b/lazyllm/common/logger.py @@ -18,7 +18,7 @@ "log_format", str, "{process}: {time:YYYY-MM-DD HH:mm:ss} {extra[name]} " - "{level}: ({name}) {message}", + "{level}: ({name}:{line}) {message}", "LOG_FORMAT", ) lazyllm.config.add("log_dir", str, "~/.lazyllm", "LOG_DIR") diff --git a/lazyllm/components/__init__.py b/lazyllm/components/__init__.py index 7a93c8a0..183feafa 100644 --- a/lazyllm/components/__init__.py +++ b/lazyllm/components/__init__.py @@ -6,6 +6,7 @@ from .validate import LazyLLMValidateBase from .auto import AutoDeploy, AutoFinetune from .utils import ModelDownloader +from .formatter import FormatterBase, EmptyFormatter, JsonFormatter __all__ = [ 'register', @@ -19,5 +20,8 @@ 'FastapiApp', 'AutoDeploy', 'AutoFinetune', - 'ModelDownloader' + 'ModelDownloader', + 'FormatterBase', + 'EmptyFormatter', + 'JsonFormatter' ] diff --git a/lazyllm/components/formatter/__init__.py b/lazyllm/components/formatter/__init__.py new file mode 100644 index 00000000..f40c2d42 --- /dev/null +++ b/lazyllm/components/formatter/__init__.py @@ -0,0 +1,10 @@ +from .formatterBase import LazyLLMFormatterBase, LazyLLMFormatterBase as FormatterBase, EmptyFormatter +from .jsonFormatter import JsonFormatter + + +__all__ = [ + 'LazyLLMFormatterBase', + 'FormatterBase', + 'EmptyFormatter', + 'JsonFormatter' +] diff --git a/lazyllm/components/formatter/formatterBase.py b/lazyllm/components/formatter/formatterBase.py new file mode 100644 index 00000000..9889d1d7 --- /dev/null +++ b/lazyllm/components/formatter/formatterBase.py @@ -0,0 +1,50 @@ +from ...common import LazyLLMRegisterMetaClass + +def is_number(s: str): + try: + int(s) + return True + except ValueError: + if s == "None" or len(s) == 0: + return False + else: + raise ValueError("Invalid number: " + s + ". You can enter an integer, None or an empyt string.") + +class LazyLLMFormatterBase(metaclass=LazyLLMRegisterMetaClass): + def __init__(self, formatter: str = None): + self._formatter = formatter + if self._formatter: + self._parse_formatter() + else: + self._slices = None + + def _parse_formatter(self): + # Remove the surrounding brackets + slice_str = self._formatter.strip()[1:-1] + dimensions = slice_str.split(",") + slices = [] + + for dim in dimensions: + if ":" in dim: + parts = dim.split(":") + start = int(parts[0]) if is_number(parts[0]) else None + end = int(parts[1]) if len(parts) > 1 and is_number(parts[1]) else None + step = int(parts[2]) if len(parts) > 2 and is_number(parts[2]) else None + slices.append(slice(start, end, step)) + else: + slices.append(dim.strip()) + self._slices = slices + + def _load(self, msg: str): + raise NotImplementedError("This parse str function is not implemented.") + + def _parse_py_data_by_formatter(self, py_data): + raise NotImplementedError("This data parse function is not implemented.") + + def format(self, msg): + if isinstance(msg, str): msg = self._load(msg) + return self._parse_py_data_by_formatter(msg) + +class EmptyFormatter(LazyLLMFormatterBase): + def format(self, msg): + return msg diff --git a/lazyllm/components/formatter/jsonFormatter.py b/lazyllm/components/formatter/jsonFormatter.py new file mode 100644 index 00000000..cd79c7ba --- /dev/null +++ b/lazyllm/components/formatter/jsonFormatter.py @@ -0,0 +1,57 @@ +import json +from .formatterBase import LazyLLMFormatterBase as FormatterBase +import lazyllm + +class JsonFormatter(FormatterBase): + def _extract_json_from_string(self, mixed_str: str): + json_objects = [] + brace_level = 0 + current_json = "" + in_string = False + + for char in mixed_str: + if char == '"' and (len(current_json) == 0 or current_json[-1] != '\\'): + in_string = not in_string + + if not in_string: + if char == '{': + if brace_level == 0: + current_json = "" + brace_level += 1 + elif char == '}': + brace_level -= 1 + + if brace_level > 0 or (brace_level == 0 and char == '}'): + current_json += char + + if brace_level == 0 and current_json: + try: + json.loads(current_json) + json_objects.append(current_json) + current_json = "" + except json.JSONDecodeError: + continue + + return json_objects + + def _load(self, msg: str): + # Convert str to json format + assert msg.count("{") == msg.count("}"), f"{msg} is not a valid json string." + try: + json_strs = self._extract_json_from_string(msg) + if len(json_strs) == 0: + raise TypeError(f"{msg} is not a valid json string.") + res = [] + for json_str in json_strs: + res.append(json.loads(json_str)) + return res if len(res) > 1 else res[0] + except Exception as e: + lazyllm.LOG.info(f"Error: {e}") + return "" + + def _parse_py_data_by_formatter(self, data, *, slices=None): + if slices is None: slices = self._slices + if not slices: return data + if isinstance(slices[0], slice): return [self._parse_py_data_by_formatter(d, slices=slices[1:]) + for d in data[slices[0]]] + else: return self._parse_py_data_by_formatter(data[slices[0]], slices=slices[1:]) diff --git a/lazyllm/components/prompter/alpacaPrompter.py b/lazyllm/components/prompter/alpacaPrompter.py index 9bcbaf8f..6231750d 100644 --- a/lazyllm/components/prompter/alpacaPrompter.py +++ b/lazyllm/components/prompter/alpacaPrompter.py @@ -1,15 +1,21 @@ -from typing import List, Union, Optional +from typing import List, Union, Optional, Dict from .builtinPrompt import LazyLLMPrompterBase class AlpacaPrompter(LazyLLMPrompterBase): - def __init__(self, instruction: Union[None, str] = None, + def __init__(self, instruction: Union[None, str, Dict[str, str]] = None, extro_keys: Union[None, List[str]] = None, show: bool = False, tools: Optional[List] = None): super(__class__, self).__init__(show, tools=tools) + if isinstance(instruction, dict): + splice_struction = instruction.get("system", "") + \ + AlpacaPrompter.ISA + instruction.get("user", "") + AlpacaPrompter.ISE + instruction = splice_struction instruction_template = ("Below is an instruction that describes a task, paired with extra messages such as " "input that provides further context if possible. Write a response that " f"appropriately completes the request.\n\n ### Instruction:\n{instruction}" "\n\n" + LazyLLMPrompterBase._get_extro_key_template(extro_keys)) - self._init_prompt("{system}\n{instruction}\n{tools}### Response:\n", instruction_template, "### Response:") + self._init_prompt("{system}\n{instruction}\n{tools}\n{user}### Response:\n", + instruction_template, + "### Response:") def _check_values(self, instruction, input, history, tools): assert not history, f"Chat history is not supported in {__class__}." diff --git a/lazyllm/components/prompter/builtinPrompt.py b/lazyllm/components/prompter/builtinPrompt.py index b3a0d1be..455ba97a 100644 --- a/lazyllm/components/prompter/builtinPrompt.py +++ b/lazyllm/components/prompter/builtinPrompt.py @@ -1,10 +1,14 @@ from typing import Dict, Union, Any, List, Callable, Optional from ...common import LazyLLMRegisterMetaClass from lazyllm import LOG +from functools import reduce import json import re class LazyLLMPrompterBase(metaclass=LazyLLMRegisterMetaClass): + ISA = "" + ISE = "" + def __init__(self, show=False, tools=None): self._set_model_configs(system='You are an AI-Agent developed by LazyLLM.', sos='<|start_system|>', soh='<|Human|>:', soa='<|Assistant|>:', eos='<|end_system|>', eoh='', eoa='') @@ -73,21 +77,25 @@ def _get_instruction_and_input(self, input): assert len(prompt_keys) == 0 return self._instruction_template, input assert isinstance(input, dict) + input = input.copy() kwargs = {k: input.pop(k) for k in prompt_keys} - assert len(input) <= 1, f'Unexpected keys found in input: {list(input.keys())}' - return (self._instruction_template.format(**kwargs) if len(kwargs) > 0 else self._instruction_template, - list(input.values())[0] if input else '') + assert len(input) <= 1, f"Unexpected keys found in input: {list(input.keys())}" + return (reduce(lambda s, kv: s.replace(f"{{{kv[0]}}}", kv[1]), + kwargs.items(), + self._instruction_template) + if len(kwargs) > 0 else self._instruction_template, + list(input.values())[0] if input else "") def _check_values(self, instruction, input, history, tools): pass # Used for TrainableModule(local deployed) - def _generate_prompt_impl(self, instruction, input, history, tools, label): - params = dict(system=self._system, instruction=instruction, input=input, history=history, tools=tools, + def _generate_prompt_impl(self, instruction, input, user, history, tools, label): + params = dict(system=self._system, instruction=instruction, input=input, user=user, history=history, tools=tools, sos=self._sos, eos=self._eos, soh=self._soh, eoh=self._eoh, soa=self._soa, eoa=self._eoa) return self._template.format(**params) + (label if label else '') # Used for OnlineChatModule - def _generate_prompt_dict_impl(self, instruction, input, history, tools, label): + def _generate_prompt_dict_impl(self, instruction, input, user, history, tools, label): if not history: history = [] if isinstance(input, str): history.append({"role": "user", "content": input}) @@ -96,6 +104,9 @@ def _generate_prompt_dict_impl(self, instruction, input, history, tools, label): else: raise TypeError("input must be a string or a dict") + if user: + history[-1]["content"].insert(0, user) + history.insert(0, {"role": "system", "content": self._system + "\n" + instruction if instruction else self._system}) @@ -105,6 +116,18 @@ def pre_hook(self, func: Optional[Callable] = None): self._pre_hook = func return self + def _split_instruction(self, instruction: str): + system_instruction = instruction + user_instruction = "" + if LazyLLMPrompterBase.ISA in instruction and LazyLLMPrompterBase.ISE in instruction: + # The instruction includes system prompts and/or user prompts + pattern = re.compile(r"%s(.*)%s" % (LazyLLMPrompterBase.ISA, LazyLLMPrompterBase.ISE)) + ret = re.split(pattern, instruction) + system_instruction = ret[0] + user_instruction = ret[1] + + return system_instruction, user_instruction + def generate_prompt(self, input: Union[str, Dict[str, str], None] = None, history: List[Union[List[str], Dict[str, Any]]] = None, tools: Union[List[Dict[str, Any]], None] = None, @@ -116,8 +139,9 @@ def generate_prompt(self, input: Union[str, Dict[str, str], None] = None, history = self._get_histories(history, return_dict=return_dict) tools = self._get_tools(tools, return_dict=return_dict) self._check_values(instruction, input, history, tools) + instruction, user_instruction = self._split_instruction(instruction) func = self._generate_prompt_dict_impl if return_dict else self._generate_prompt_impl - result = func(instruction, input, history, tools, label) + result = func(instruction, input, user_instruction, history, tools, label) if self._show or show: LOG.info(result) return result diff --git a/lazyllm/components/prompter/chatPrompter.py b/lazyllm/components/prompter/chatPrompter.py index dff1dd29..d7e828b2 100644 --- a/lazyllm/components/prompter/chatPrompter.py +++ b/lazyllm/components/prompter/chatPrompter.py @@ -1,13 +1,17 @@ -from typing import List, Union, Optional +from typing import List, Union, Optional, Dict from .builtinPrompt import LazyLLMPrompterBase class ChatPrompter(LazyLLMPrompterBase): - def __init__(self, instruction: Union[None, str] = None, + def __init__(self, instruction: Union[None, str, Dict[str, str]] = None, extro_keys: Union[None, List[str]] = None, show: bool = False, tools: Optional[List] = None): super(__class__, self).__init__(show, tools=tools) + if isinstance(instruction, dict): + splice_instruction = instruction.get("system", "") + \ + ChatPrompter.ISA + instruction.get("user", "") + ChatPrompter.ISE + instruction = splice_instruction instruction_template = f'{instruction}\n{{extro_keys}}\n'.replace( '{extro_keys}', LazyLLMPrompterBase._get_extro_key_template(extro_keys)) - self._init_prompt("{sos}{system}{instruction}{tools}{eos}\n\n{history}\n{soh}\n{input}\n{eoh}{soa}\n", + self._init_prompt("{sos}{system}{instruction}{tools}{eos}\n\n{history}\n{soh}\n{user}{input}\n{eoh}{soa}\n", instruction_template) @property diff --git a/lazyllm/docs/components.py b/lazyllm/docs/components.py index 4cbe8054..3368bedc 100644 --- a/lazyllm/docs/components.py +++ b/lazyllm/docs/components.py @@ -520,20 +520,84 @@ def test_prompter(): >>> downloader.download('GLM3-6B') ''') +# ============= Formatter + +# FormatterBase +add_chinese_doc('formatter.FormatterBase', '''\ +此类是格式化器的基类,格式化器是模型输出结果的格式化器,用户可以自定义格式化器,也可以使用LazyLLM提供的格式化器。 +主要方法:_parse_formatter:解析索引内容。_load:解析str对象,其中包含python对象的部分被解析出来,比如list,dict等对象。_parse_py_data_by_formatter:根据自定义的格式化器和索引对python对象进行格式化。format:对传入的内容进行格式化,如果内容是字符串类型,先将字符串转化为python对象,再进行格式化。如果内容是python对象,直接进行格式化。 +''') + +add_english_doc('formatter.FormatterBase', '''\ +This class is the base class of the formatter. The formatter is the formatter of the model output result. Users can customize the formatter or use the formatter provided by LazyLLM. +Main methods: _parse_formatter: parse the index content. _load: Parse the str object, and the part containing Python objects is parsed out, such as list, dict and other objects. _parse_py_data_by_formatter: format the python object according to the custom formatter and index. format: format the passed content. If the content is a string type, convert the string into a python object first, and then format it. If the content is a python object, format it directly. +''') + +add_example('formatter.FormatterBase', '''\ +>>> from lazyllm.components.formatter import FormatterBase +>>> class MyFormatter(LazyLLMFormatterBase): +... def _load(self, data): +... return str_to_list(data) +... +... def _parse_py_data_by_formatter(self, data, formatter): +... return extract_data_by_formatter(data, formatter) +... +>>> fmt = MyFormatter("[1:3]") # 取列表中索引为1和2的元素 +>>> fmt.format("[1,2,3,4,5]") # 输入为字符串"[1,2,3,4,5]" +[2,3] +''') + +# JsonFormatter +add_chinese_doc('JsonFormatter', '''\ +此类是JSON格式化器,即用户希望模型输出的内容格式为JSON,还可以通过索引方式对输出内容中的某个字段进行选择。 +''') + +add_english_doc('JsonFormatter', '''\ +This class is a JSON formatter, that is, the user wants the model to output content is JSON format, and can also select a field in the output content by indexing. +''') + +add_example('JsonFormatter', '''\ +>>> from lazyllm.components import JsonFormatter +>>> # Assume that the model output without specifying a formatter is as follows: +"Based on your input, here is the corresponding list of nested dictionaries:\\n\\n```python\\n[\\n {\\n \"title\": \"# Introduction\",\\n \"describe\": \"Provide an overview of the topic and set the stage for the article. Discuss what the reader can expect to learn from this article.\"\\n },\\n {\\n \"title\": \"## What is Artificial Intelligence?\",\\n \"describe\": \"Define Artificial Intelligence and discuss its importance in various fields, including the medical industry.\"\\n },\\n {\\n \"title\": \"## Applications of AI in Medical Field\",\\n \"describe\": \"Outline the ways AI is used in the medical field, such as diagnosis, drug discovery, and patient treatment.\"\\n },\\n {\\n \"title\": \"### Medical Image Analysis\",\\n \"describe\": \"Discuss how AI-powered image analysis tools help in detecting diseases and analyzing medical images, such as X-rays, MRIs, and CT scans.\"\\n },\\n {\\n \"title\": \"### Personalized Medicine\",\\n \"describe\": \"Explain how AI algorithms can assist in genetic testing and tailor treatment plans based on an individual's genetic makeup.\"\\n },\\n {\\n \"title\": \"### Electronic Health Records (EHRs) and Medical Data Management\",\\n \"describe\": \"Discuss the role of AI in managing and analyzing large amounts of medical data, such as electronic health records, for improved patient care and population health management.\"\\n },\\n {\\n \"title\": \"## Challenges in AI Adoption\",\\n \"describe\": \"Highlight potential challenges to AI implementation, including data privacy, ethical concerns, and regulatory issues.\"\\n },\\n {\\n \"title\": \"## Future of AI in Medicine\",\\n \"describe\": \"Investigate the evolving role of AI in medicine, the anticipated advancements in the field, and their potential impact on medical professionals and patients.\"\\n },\\n {\\n \"title\": \"# Conclusion\",\\n \"describe\": \"Summarize the key points of the article and emphasize the potential for AI in revolutionizing the medical field.\"\\n }\\n]\\n```\\n\\nPlease use the provided `title` and `describe` information to write your article, leveraging Markdown format to denote hierarchical levels. Each `title` should reflect its corresponding level in a Markdown format, including \"#\" for level 1, \"##\" for level 2, and \"###\" for level 3. The `describe` text provides a guide for developing each section of the article, ensuring it aligns with the overarching discussion on the application of AI in the medical field." +>>> jsonFormatter=JsonFormatter("[:, title]") # ":" represents all elements in a list. "title" represents the "title" field in the json data. +>>> model.formatter(jsonFormatter) +>>> # The model output of the specified formatter is as follows +["# Introduction", "## What is Artificial Intelligence?", "## Applications of AI in Medical Field", "### Medical Image Analysis", "### Personalized Medicine", "### Electronic Health Records (EHRs) and Medical Data Management", "## Challenges in AI Adoption", "## Future of AI in Medicine", "# Conclusion"] +''') + +# EmptyFormatter +add_chinese_doc('EmptyFormatter', '''\ +此类是空的格式化器,即用户希望对模型的输出不做格式化,用户可以对模型指定该格式化器,也可以不指定(模型默认的格式化器就是空格式化器) +''') + +add_english_doc('EmptyFormatter', '''\ +This type is the system default formatter. When the user does not specify anything or does not want to format the model output, this type is selected. The model output will be in the same format. +''') + +add_example('EmptyFormatter', '''\ +>>> from lazyllm.components import EmptyFormatter +>>> # Assume that the model output without specifying a formatter is as follows: +"Here's a nested list of dictionaries based on your user input:\\n\\n```json\\n[\\n {\\n \"title\": \"# AI in Medical Field\",\\n \"describe\": \"Please provide a detailed introduction to the use of artificial intelligence in the medical field, emphasizing its potential benefits and challenges.\"\\n },\\n {\\n \"title\": \"## Applications of AI in Medical Diagnosis\",\\n \"describe\": \"Please discuss the utilization of AI in medical diagnosis, including its advantages over traditional methods and notable achievements.\"\\n },\\n {\\n \"title\": \"### AI-assisted Diagnosis Tools\",\\n \"describe\": \"Please elaborate on specific AI-assisted diagnostic tools used in medical practice, such as image analysis, predictive analytics, and decision support systems.\"\\n },\\n {\\n \"title\": \"#### Image Analysis Tools\",\\n \"describe\": \"Please provide a comprehensive overview of AI-powered image analysis tools and their role in enhancing disease detection and treatment planning.\"\\n },\\n {\\n \"title\": \"#### Predictive Analytics\",\\n \"describe\": \"Please explain how predictive analytics leverages AI to forecast diseases, identify risk factors, and develop personalized treatment protocols.\"\\n },\\n {\\n \"title\": \"#### Decision Support Systems\",\\n \"describe\": \"Please discuss the role of decision support systems in facilitating clinical decision-making and improving patient outcomes.\"\\n },\\n {\\n \"title\": \"## Advantages and Limitations of AI in Medical Field\",\\n \"describe\": \"Please identify and elaborate on the key advantages and limitations of employing AI in the medical field, including ethical, legal, and practical considerations.\"\\n },\\n {\\n \"title\": \"## Future Perspectives and Innovations\",\\n \"describe\": \"Please provide a forward-looking view of the progression of AI in the healthcare sector, predicting future developments, and discussing the potential impact on medical professionals and patients.\"\\n },\\n {\\n \"title\": \"### New AI Technologies\",\\n \"describe\": \"Please discuss emerging AI technologies that could reshape the medical field, such as machine learning, natural language processing, and robotics.\"\\n },\\n {\\n \"title\": \"#### Machine Learning\",\\n \"describe\": \"Please explain how machine learning algorithms are being used to enhance medical research, such as in drug discovery, genomics, and epidemiology.\"\\n },\\n {\\n \"title\": \"#### Natural Language Processing\",\\n \"describe\": \"Please discuss the role of natural language processing in extracting and analyzing medical data from various sources, such as electronic health records and scientific literature.\"\\n },\\n {\\n \"title\": \"#### Robotics\",\\n \"describe\": \"Please elaborate on the incorporation of AI-driven robots in medical procedures and surgeries, emphasizing their potential to revolutionize patient care and treatment options.\"\\n },\\n {\\n \"title\": \"### Ethical Considerations\",\\n \"describe\": \"Please address the ethical concerns surrounding AI in healthcare, such as data privacy, transparency, and patient autonomy, and discuss potential methods to mitigate these issues.\"\\n }\\n]\\n```\\n\\nThis outline provides a comprehensive structure for your article, addressing various aspects of the application of AI in the medical field, from specific AI technologies to ethical considerations. You can start by elaborating on each section, providing detailed descriptions, examples, and relevant information. Be sure to include scientific research, case studies, and expert opinions to support your arguments and provide a comprehensive understanding of the subject." +>>> emptyFormatter = EmptyFormatter() +>>> model.formatter(emptyFormatter) +>>> # The model output of the specified formatter is as follows +"Here's a nested list of dictionaries based on your user input:\\n\\n```json\\n[\\n {\\n \"title\": \"# AI in Medical Field\",\\n \"describe\": \"Please provide a detailed introduction to the use of artificial intelligence in the medical field, emphasizing its potential benefits and challenges.\"\\n },\\n {\\n \"title\": \"## Applications of AI in Medical Diagnosis\",\\n \"describe\": \"Please discuss the utilization of AI in medical diagnosis, including its advantages over traditional methods and notable achievements.\"\\n },\\n {\\n \"title\": \"### AI-assisted Diagnosis Tools\",\\n \"describe\": \"Please elaborate on specific AI-assisted diagnostic tools used in medical practice, such as image analysis, predictive analytics, and decision support systems.\"\\n },\\n {\\n \"title\": \"#### Image Analysis Tools\",\\n \"describe\": \"Please provide a comprehensive overview of AI-powered image analysis tools and their role in enhancing disease detection and treatment planning.\"\\n },\\n {\\n \"title\": \"#### Predictive Analytics\",\\n \"describe\": \"Please explain how predictive analytics leverages AI to forecast diseases, identify risk factors, and develop personalized treatment protocols.\"\\n },\\n {\\n \"title\": \"#### Decision Support Systems\",\\n \"describe\": \"Please discuss the role of decision support systems in facilitating clinical decision-making and improving patient outcomes.\"\\n },\\n {\\n \"title\": \"## Advantages and Limitations of AI in Medical Field\",\\n \"describe\": \"Please identify and elaborate on the key advantages and limitations of employing AI in the medical field, including ethical, legal, and practical considerations.\"\\n },\\n {\\n \"title\": \"## Future Perspectives and Innovations\",\\n \"describe\": \"Please provide a forward-looking view of the progression of AI in the healthcare sector, predicting future developments, and discussing the potential impact on medical professionals and patients.\"\\n },\\n {\\n \"title\": \"### New AI Technologies\",\\n \"describe\": \"Please discuss emerging AI technologies that could reshape the medical field, such as machine learning, natural language processing, and robotics.\"\\n },\\n {\\n \"title\": \"#### Machine Learning\",\\n \"describe\": \"Please explain how machine learning algorithms are being used to enhance medical research, such as in drug discovery, genomics, and epidemiology.\"\\n },\\n {\\n \"title\": \"#### Natural Language Processing\",\\n \"describe\": \"Please discuss the role of natural language processing in extracting and analyzing medical data from various sources, such as electronic health records and scientific literature.\"\\n },\\n {\\n \"title\": \"#### Robotics\",\\n \"describe\": \"Please elaborate on the incorporation of AI-driven robots in medical procedures and surgeries, emphasizing their potential to revolutionize patient care and treatment options.\"\\n },\\n {\\n \"title\": \"### Ethical Considerations\",\\n \"describe\": \"Please address the ethical concerns surrounding AI in healthcare, such as data privacy, transparency, and patient autonomy, and discuss potential methods to mitigate these issues.\"\\n }\\n]\\n```\\n\\nThis outline provides a comprehensive structure for your article, addressing various aspects of the application of AI in the medical field, from specific AI technologies to ethical considerations. You can start by elaborating on each section, providing detailed descriptions, examples, and relevant information. Be sure to include scientific research, case studies, and expert opinions to support your arguments and provide a comprehensive understanding of the subject." +''') # ============= Prompter add_chinese_doc('prompter.PrompterBase', '''\ Prompter的基类,自定义的Prompter需要继承此基类,并通过基类提供的 ``_init_prompt`` 函数来设置Prompt模板和Instruction的模板,以及截取结果所使用的字符串。可以查看 :doc:`/best_practice/prompt` 进一步了解Prompt的设计思想和使用方式。 -Prompt模板和Instruction模板都用 ``{}`` 表示要填充的字段,其中Prompt可包含的字段有 ``system``, ``history``, ``tools``等,而instruction_template可包含的字段为 ``instruction`` 和 ``extro_keys`` 。 -``instruction`` 由应用的开发者传入, ``instruction`` 中也可以带有 ``{}`` 用于让定义可填充的字段,方便用户填入额外的信息。 +Prompt模板和Instruction模板都用 ``{}`` 表示要填充的字段,其中Prompt可包含的字段有 ``system``, ``history``, ``tools``, ``user`` 等,而instruction_template可包含的字段为 ``instruction`` 和 ``extro_keys`` 。 +``instruction`` 由应用的开发者传入, ``instruction`` 中也可以带有 ``{}`` 用于让定义可填充的字段,方便用户填入额外的信息。如果 ``instruction`` 字段为字符串,则认为是系统instruction;如果是字典,则它包含的key只能是 ``user`` 和 ``system`` 两种选择。 ``user`` 表示用户输入的instruction,在prompt中放在用户输入前面, ``system`` 表示系统instruction,在prompt中凡在system prompt后面。 ''') add_english_doc('prompter.PrompterBase', '''\ The base class of Prompter. A custom Prompter needs to inherit from this base class and set the Prompt template and the Instruction template using the `_init_prompt` function provided by the base class, as well as the string used to capture results. Refer to :doc:`/best_practice/prompt.rst` for further understanding of the design philosophy and usage of Prompts. -Both the Prompt template and the Instruction template use ``{}`` to indicate the fields to be filled in. The fields that can be included in the Prompt are `system`, `history`, `tools`, etc., while the fields that can be included in the instruction_template are `instruction` and `extro_keys`. +Both the Prompt template and the Instruction template use ``{}`` to indicate the fields to be filled in. The fields that can be included in the Prompt are `system`, `history`, `tools`, `user` etc., while the fields that can be included in the instruction_template are `instruction` and `extro_keys`. If the ``instruction`` field is a string, it is considered as a system instruction; if it is a dictionary, it can only contain the keys ``user`` and ``system``. ``user`` represents the user input instruction, which is placed before the user input in the prompt, and ``system`` represents the system instruction, which is placed after the system prompt in the prompt. ``instruction`` is passed in by the application developer, and the ``instruction`` can also contain ``{}`` to define fillable fields, making it convenient for users to input additional information. ''') @@ -598,7 +662,7 @@ def test_prompter(): Alpaca格式的Prompter,支持工具调用,不支持历史对话。 Args: - instruction (Option[str]): 大模型的任务指令,至少带一个可填充的槽位(如 ``{instruction}``)。 + instruction (Option[str]): 大模型的任务指令,至少带一个可填充的槽位(如 ``{instruction}``)。或者使用字典指定 ``system`` 和 ``user`` 的指令。 extro_keys (Option[List]): 额外的字段,用户的输入会填充这些字段。 show (bool): 标志是否打印生成的Prompt,默认为False tools (Option[list]): 大模型可以使用的工具集合,默认为None @@ -610,7 +674,7 @@ def test_prompter(): Sure! Here is the translation, keeping the original format: Args: - instruction (Option[str]): Task instructions for the large model, with at least one fillable slot (e.g. ``{instruction}``). + instruction (Option[str]): Task instructions for the large model, with at least one fillable slot (e.g. ``{instruction}``). Or use a dictionary to specify the ``system`` and ``user`` instructions. extro_keys (Option[List]): Additional fields that will be filled with user input. show (bool): Flag indicating whether to print the generated Prompt, default is False. tools (Option[list]): Tool-set which is provived for LLMs, default is None. @@ -629,13 +693,20 @@ def test_prompter(): 'You are an AI-Agent developed by LazyLLM.\\nBelow is an instruction that describes a task, paired with extra messages such as input that provides further context if possible. Write a response that appropriately completes the request.\\n\\n ### Instruction:\\nhello world hello world, my input\\n\\nHere are some extra messages you can referred to:\\n\\n### knowledge:\\nlazyllm\\n\\n\\n### Response:\\n' >>> p.generate_prompt(dict(instruction='hello world', input='my input', knowledge='lazyllm'), return_dict=True) {'messages': [{'role': 'system', 'content': 'You are an AI-Agent developed by LazyLLM.\\nBelow is an instruction that describes a task, paired with extra messages such as input that provides further context if possible. Write a response that appropriately completes the request.\\n\\n ### Instruction:\\nhello world hello world, my input\\n\\nHere are some extra messages you can referred to:\\n\\n### knowledge:\\nlazyllm\\n\\n'}, {'role': 'user', 'content': ''}]} +>>> +>>> p = AlpacaPrompter(dict(system="hello world", user="this is user instruction {input}")) +>>> p.generate_prompt(dict(input="my input")) +'You are an AI-Agent developed by LazyLLM.\nBelow is an instruction that describes a task, paired with extra messages such as input that provides further context if possible. Write a response that appropriately completes the request.\\n\\n ### Instruction:\\nhello word\\n\\n\\n\\nthis is user instruction my input### Response:\\n' +>>> p.generate_prompt(dict(input="my input"), return_dict=True) +{'messages': [{'role': 'system', 'content': 'You are an AI-Agent developed by LazyLLM.\\nBelow is an instruction that describes a task, paired with extra messages such as input that provides further context if possible. Write a response that appropriately completes the request.\\n\\n ### Instruction:\\nhello world'}, {'role': 'user', 'content': 'this is user instruction my input'}]} + ''') add_chinese_doc('ChatPrompter', '''\ 多轮对话的Prompt,支持工具调用和历史对话 Args: - instruction (Option[str]): 大模型的任务指令,可以带0到多个待填充的槽位,用 ``{}`` 表示。 + instruction (Option[str]): 大模型的任务指令,可以带0到多个待填充的槽位,用 ``{}`` 表示。针对用户instruction可以通过字典传递,字段为 ``user`` 和 ``system`` 。 extro_keys (Option[List]): 额外的字段,用户的输入会填充这些字段。 show (bool): 标志是否打印生成的Prompt,默认为False ''') @@ -644,7 +715,7 @@ def test_prompter(): chat prompt, supports tool calls and historical dialogue. Args: - instruction (Option[str]): Task instructions for the large model, with 0 to multiple fillable slot, represented by ``{}``. + instruction (Option[str]): Task instructions for the large model, with 0 to multiple fillable slot, represented by ``{}``. For user instructions, you can pass a dictionary with fields ``user`` and ``system``. extro_keys (Option[List]): Additional fields that will be filled with user input. show (bool): Flag indicating whether to print the generated Prompt, default is False. ''') @@ -664,6 +735,12 @@ def test_prompter(): {'messages': [{'role': 'system', 'content': 'You are an AI-Agent developed by LazyLLM.\\nhello world this is my ins\\nHere are some extra messages you can referred to:\\n\\n### knowledge:\\nLazyLLM-Knowledge\\n\\n\\n'}, {'role': 'user', 'content': 'this is my inp'}]} >>> p.generate_prompt(dict(instruction='this is my ins', input='this is my inp', knowledge='LazyLLM-Knowledge'), history=[['s1', 'e1'], ['s2', 'e2']]) '<|start_system|>You are an AI-Agent developed by LazyLLM.hello world this is my ins\\nHere are some extra messages you can referred to:\\n\\n### knowledge:\\nLazyLLM-Knowledge\\n\\n\\n<|end_system|>\\n\\n<|Human|>:s1<|Assistant|>:e1<|Human|>:s2<|Assistant|>:e2\\n<|Human|>:\\nthis is my inp\\n<|Assistant|>:\\n' +>>> +>>> p = ChatPrompter(dict(system="hello world", user="this is user instruction {input} ")) +>>> p.generate_prompt(dict(input="my input", query="this is user query")) +'<|start_system|>You are an AI-Agent developed by LazyLLM.hello world\\n\\n<|end_system|>\\n\\n\\n<|Human|>:\\nthis is user instruction my input this is user query\\n<|Assistant|>:\\n' +>>> p.generate_prompt(dict(input="my input", query="this is user query"), return_dict=True) +{'messages': [{'role': 'system', 'content': 'You are an AI-Agent developed by LazyLLM.\\nhello world\\n\\n'}, {'role': 'user', 'content': 'this is user instruction my input this is user query'}]} ''') # ============= Launcher diff --git a/lazyllm/module/module.py b/lazyllm/module/module.py index 3ebc6496..bdbfd1bd 100644 --- a/lazyllm/module/module.py +++ b/lazyllm/module/module.py @@ -273,7 +273,7 @@ def prompt(self, prompt=None): self._prompt = EmptyPrompter() elif isinstance(prompt, PrompterBase): self._prompt = prompt - elif isinstance(prompt, str): + elif isinstance(prompt, (str, dict)): self._prompt = ChatPrompter(prompt) return self diff --git a/lazyllm/module/onlineChatModule/onlineChatModule.py b/lazyllm/module/onlineChatModule/onlineChatModule.py index a9a6cba5..201ca97f 100644 --- a/lazyllm/module/onlineChatModule/onlineChatModule.py +++ b/lazyllm/module/onlineChatModule/onlineChatModule.py @@ -20,7 +20,6 @@ class OnlineChatModule(metaclass=_ChatModuleMeta): @staticmethod def _encapsulate_parameters(base_url: str, model: str, - system_prompt: str, stream: bool, return_trace: bool, **kwargs) -> Dict[str, Any]: @@ -29,8 +28,6 @@ def _encapsulate_parameters(base_url: str, params['base_url'] = base_url if model is not None: params['model'] = model - if system_prompt is not None: - params['system_prompt'] = system_prompt params.update(kwargs) return params @@ -39,11 +36,10 @@ def __new__(self, source: str, base_url: str = None, model: str = None, - system_prompt: str = None, stream: bool = True, return_trace: bool = False, **kwargs): - params = OnlineChatModule._encapsulate_parameters(base_url, model, system_prompt, stream, return_trace, **kwargs) + params = OnlineChatModule._encapsulate_parameters(base_url, model, stream, return_trace, **kwargs) if source.lower() == "openai": return OpenAIModule(**params) diff --git a/lazyllm/module/onlineChatModule/onlineChatModuleBase.py b/lazyllm/module/onlineChatModule/onlineChatModuleBase.py index 85a2fd10..edb75ce4 100644 --- a/lazyllm/module/onlineChatModule/onlineChatModuleBase.py +++ b/lazyllm/module/onlineChatModule/onlineChatModuleBase.py @@ -1,10 +1,12 @@ import json import os import requests +import re from typing import Tuple, List, Dict, Union, Any import time import lazyllm from lazyllm.components.prompter import PrompterBase, ChatPrompter +from lazyllm.components.formatter import FormatterBase, EmptyFormatter from ..module import ModuleBase, Pipeline class OnlineChatModuleBase(ModuleBase): @@ -31,13 +33,16 @@ def __init__(self, self._set_chat_url() self.prompt() self._is_trained = False + self.formatter() + self.field_extractor() + self._stream_end_token = "[DONE]" def prompt(self, prompt=None): if prompt is None: self._prompt = ChatPrompter() elif isinstance(prompt, PrompterBase): self._prompt = prompt - elif isinstance(prompt, str): + elif isinstance(prompt, (str, dict)): self._prompt = ChatPrompter(prompt) else: raise TypeError(f"{prompt} type is not supported.") @@ -70,14 +75,80 @@ def _get_models_list(self): res_json = r.json() return res_json + def _parse_output_by_key(self, key: str, data: Dict[str, Any]): + if "choices" in data and isinstance(data["choices"], list): + item = data['choices'][0] + data = item.get("delta", {}) if "delta" in item else item.get("message", {}) + return data if not key else data.get(key, "") + else: + raise ValueError(f"The response {data} does not contain a 'choices' field.") + + def _synthetic_output(self, response: Dict[str, Any]): + if len(self._extractor_fields) == 1: + key = self._extractor_fields[0] + content = self._parse_output_by_key(key, response) if key else "" + return self._formatter.format(content) if content else "" + elif len(self._extractor_fields) > 1: + res = {} + for key in self._extractor_fields: + content = self._parse_output_by_key(key, response) if key else "" + res[key] = self._formatter.format(content) if content else "" + return res + else: + content = self._parse_output_by_key(".", response) + return self._formatter.format(content) if content else "" + + def _stream_post_process(self, response: str) -> Dict[str, Any]: + try: + chunk = json.loads(response) + return chunk + except ValueError: + return response + except Exception as e: + lazyllm.LOG.error(e) + return "" + def _parse_response_stream(self, response: str) -> str: - chunk = response.decode('utf-8')[6:] - return chunk + pattern = re.compile(r"^data:\s*") + response = re.sub(pattern, "", response.decode('utf-8')) + chunk = self._stream_post_process(response) + if self._stream_end_token == chunk: return self._stream_end_token + return self._synthetic_output(chunk) + + def _nonstream_post_process(self, response: str) -> Dict[str, Any]: + try: + chunk = json.loads(response) + return chunk + except Exception as e: + lazyllm.LOG.error(e) + return "" def _parse_response_non_stream(self, response: str) -> Dict[str, Any]: """Parse the response from the interface""" - cur_msg = json.loads(response)["choices"][0] - return cur_msg + cur_msg = self._nonstream_post_process(response) + return self._synthetic_output(cur_msg) + + def formatter(self, format: FormatterBase = None): + if isinstance(format, FormatterBase): + self._formatter = format + elif format is None: + self._formatter = EmptyFormatter() + else: + raise TypeError("format must be a FormatterBase") + + return self + + def field_extractor(self, key: Union[str, List[str]] = None): + if key is None: + self._extractor_fields = ["content"] + elif isinstance(key, str): + self._extractor_fields = [key] + elif isinstance(key, list): + self._extractor_fields = key + else: + raise TypeError(f"Unsupported type: {type(key)}") + + return self def forward(self, __input: Union[Dict, str] = None, llm_chat_history: List[List[str]] = None, tools: List[Dict[str, Any]] = None, **kw): # noqa C901 """LLM inference interface""" @@ -101,8 +172,9 @@ def _impl_stream(): for line in r.iter_lines(): if len(line) == 0: continue + chunk = self._parse_response_stream(line) - if chunk == "[DONE]": return + if self._stream_end_token == chunk: return yield chunk def _impl_non_stream(): diff --git a/lazyllm/module/onlineChatModule/sensenovaModule.py b/lazyllm/module/onlineChatModule/sensenovaModule.py index ba4566f0..5d6aef16 100644 --- a/lazyllm/module/onlineChatModule/sensenovaModule.py +++ b/lazyllm/module/onlineChatModule/sensenovaModule.py @@ -1,7 +1,7 @@ import json import os import requests -from typing import Tuple +from typing import Tuple, Dict, Any import uuid import lazyllm from .onlineChatModuleBase import OnlineChatModuleBase @@ -53,27 +53,34 @@ def encode_jwt_token(ak: str, sk: str) -> str: def _set_chat_url(self): self._url = os.path.join(self._base_url, 'chat-completions') - def _parse_response_stream(self, response: str) -> str: - chunk = response.decode('utf-8')[5:] + def _stream_post_process(self, response: str) -> Dict[str, Any]: try: - chunk = json.loads(chunk)["data"] + chunk = json.loads(response)["data"] content = chunk['choices'][0]['delta'] - chunk['choices'][0]['delta'] = {"content": content} - return json.dumps(chunk, ensure_ascii=False) - except Exception: + role = chunk['choices'][0].pop("role") + chunk['choices'][0]['delta'] = {"content": content, "role": role} + if "tool_calls" in chunk["choices"][0]: + tool_calls = chunk["choices"][0].pop("tool_calls") + chunk["choices"][0]["delta"]["tool_calls"] = tool_calls + chunk["model"] = self._model_name return chunk + except ValueError: + return chunk + except Exception as e: + lazyllm.LOG.error(e) + return "" - def _parse_response_non_stream(self, response: str) -> str: + def _nonstream_post_process(self, response: str) -> Dict[str, Any]: try: resp = json.loads(response)['data'] - content = resp["choices"][0].get("message", "") + content = resp['choices'][0].get('message', '') msg = {"role": resp['choices'][0].pop("role"), "content": content} - resp['choices'][0]['message'] = msg - if 'tool_calls' in resp['choices'][0]: - tool_calls = resp['choices'][0].pop("tool_calls") - resp['choices'][0]['message']['tool_calls'] = tool_calls - resp['model'] = self._model_name - return resp["choices"][0] + resp["choices"][0]["message"] = msg + if "tool_calls" in resp["choices"][0]: + tool_calls = resp["choices"][0].pop("tool_calls") + resp["choices"][0]["message"]["tool_calls"] = tool_calls + resp["model"] = self._model_name + return resp except Exception as e: lazyllm.LOG.error(e) return ""