diff --git a/middleware/llm_task.py b/middleware/llm_task.py index fa999b5..3952be8 100644 --- a/middleware/llm_task.py +++ b/middleware/llm_task.py @@ -73,10 +73,8 @@ def create_message(self, write_back=True): for i, message in enumerate(history_messages): _history.append(message) # 刮削器合并消息,这里评价简写了。 - _total = 0 for i, _msg in enumerate(_history): - _total += 1 - self.scraper.add_message(_msg, score=len(str(_msg)), order=_total) + self.scraper.add_message(_msg, score=len(str(_msg))) # 处理附带的任何原始消息 if write_back: @@ -88,8 +86,7 @@ def create_message(self, write_back=True): _buffer.append(Message(role="user", content=message.text)) # 新消息的分数比较高 for i, _msg in enumerate(_buffer): - _total += 1 - self.scraper.add_message(_msg, score=len(str(_msg)) + 100, order=_total) + self.scraper.add_message(_msg, score=len(str(_msg)) + 50) # save to history for _msg in _buffer: self.message_history.add_message(message=_msg) diff --git a/plugins/search.py b/plugins/search.py index dc91af4..550b20f 100644 --- a/plugins/search.py +++ b/plugins/search.py @@ -28,7 +28,7 @@ def search_on_duckduckgo(search_sentence: str, key_words: str = None): from duckduckgo_search import DDGS - from middleware.filter.sublimate import Sublimate + from sdk.filter import Sublimate with DDGS(timeout=20) as ddgs: _text = [] for r in ddgs.text(search_sentence): @@ -76,7 +76,7 @@ class SearchTool(BaseTool): def pre_check(self): try: from duckduckgo_search import DDGS - from middleware.filter.sublimate import Sublimate + from sdk.filter import Sublimate return True except ImportError as e: logger.warning(f"plugin:package not found,please install it first:{e}") diff --git a/sdk/endpoint/openai/action.py b/sdk/endpoint/openai/action.py index 55c511a..31cfced 100644 --- a/sdk/endpoint/openai/action.py +++ b/sdk/endpoint/openai/action.py @@ -11,6 +11,7 @@ from loguru import logger from pydantic import BaseModel +from sdk.filter.evaluate import Sim from sdk.schema import Message @@ -99,15 +100,19 @@ class Sorter(BaseModel): # 最大消息数 max_messages: int = 12 + # 计数器 + tick: int = 0 + # 方法:添加消息 - def add_message(self, message: Message, score: float, order: int): + def add_message(self, message: Message, score: float): if hasattr(message, "function_call"): return None - self.messages.append(self.Sorter(message=message, score=score, order=order)) + self.messages.append(self.Sorter(message=message, score=score, order=self.tick)) + self.tick += 1 # 按照顺序排序 self.messages.sort(key=lambda x: x.order) - if len(self.messages) > self.max_messages: - self.messages.pop() + while len(self.messages) > self.max_messages: + self.messages.pop(0) # 方法:获取消息 def get_messages(self) -> List[Message]: @@ -120,14 +125,37 @@ def get_messages(self) -> List[Message]: logger.debug(_message) return _message + def build_messages(self): + # 只取三个,末位匹配 + _message = self.get_messages() + if len(_message) < 3: + return _message + _build = [] + _must = _message[-3:] + _check_list = _message[:-3] + _match_sentence = _message[-1:][0].content + for item_obj in _check_list: + if Sim.cosion_similarity(pre=_match_sentence, aft=item_obj.content) < 0.9: + _build.append(item_obj) + else: + pass + # logger.warning(f"ignore sim item {item_obj}") + _build.extend(_must) + return _build + # 方法:获取消息数 def get_num_messages(self) -> int: return len(self.messages) # 方法:清除消息到负载 def reduce_messages(self, limit: int = 2048): - if limit > 100: + # 预留位 + if limit > 1000: + limit = limit - 250 + else: limit = limit - 70 + + # 执行删除操作 if TokenizerObj.num_tokens_from_messages(self.get_messages()) > limit: # 从最旧开始删除 self.messages.sort(key=lambda x: x.order) diff --git a/middleware/filter/__init__.py b/sdk/filter/__init__.py similarity index 81% rename from middleware/filter/__init__.py rename to sdk/filter/__init__.py index 37c5797..ecf4536 100644 --- a/middleware/filter/__init__.py +++ b/sdk/filter/__init__.py @@ -5,3 +5,4 @@ # @Software: PyCharm # 去噪声算法 +from .sublimate import Sublimate diff --git a/middleware/filter/api/LICENSE b/sdk/filter/api/LICENSE similarity index 100% rename from middleware/filter/api/LICENSE rename to sdk/filter/api/LICENSE diff --git a/middleware/filter/api/README.md b/sdk/filter/api/README.md similarity index 100% rename from middleware/filter/api/README.md rename to sdk/filter/api/README.md diff --git a/middleware/filter/api/__init__.py b/sdk/filter/api/__init__.py similarity index 100% rename from middleware/filter/api/__init__.py rename to sdk/filter/api/__init__.py diff --git a/middleware/filter/api/data/stop_words.txt b/sdk/filter/api/data/stop_words.txt similarity index 100% rename from middleware/filter/api/data/stop_words.txt rename to sdk/filter/api/data/stop_words.txt diff --git a/middleware/filter/api/data/synonym.json b/sdk/filter/api/data/synonym.json similarity index 100% rename from middleware/filter/api/data/synonym.json rename to sdk/filter/api/data/synonym.json diff --git a/middleware/filter/api/keyphrase/__init__.py b/sdk/filter/api/keyphrase/__init__.py similarity index 100% rename from middleware/filter/api/keyphrase/__init__.py rename to sdk/filter/api/keyphrase/__init__.py diff --git a/middleware/filter/api/keyphrase/keyphrase.py b/sdk/filter/api/keyphrase/keyphrase.py similarity index 100% rename from middleware/filter/api/keyphrase/keyphrase.py rename to sdk/filter/api/keyphrase/keyphrase.py diff --git a/middleware/filter/api/keywords/__init__.py b/sdk/filter/api/keywords/__init__.py similarity index 100% rename from middleware/filter/api/keywords/__init__.py rename to sdk/filter/api/keywords/__init__.py diff --git a/middleware/filter/api/keywords/tfidf.py b/sdk/filter/api/keywords/tfidf.py similarity index 100% rename from middleware/filter/api/keywords/tfidf.py rename to sdk/filter/api/keywords/tfidf.py diff --git a/middleware/filter/api/requirements.txt b/sdk/filter/api/requirements.txt similarity index 100% rename from middleware/filter/api/requirements.txt rename to sdk/filter/api/requirements.txt diff --git a/middleware/filter/api/solo.py b/sdk/filter/api/solo.py similarity index 100% rename from middleware/filter/api/solo.py rename to sdk/filter/api/solo.py diff --git a/middleware/filter/api/summarization/__init__.py b/sdk/filter/api/summarization/__init__.py similarity index 100% rename from middleware/filter/api/summarization/__init__.py rename to sdk/filter/api/summarization/__init__.py diff --git a/middleware/filter/api/summarization/textrank_summarization.py b/sdk/filter/api/summarization/textrank_summarization.py similarity index 100% rename from middleware/filter/api/summarization/textrank_summarization.py rename to sdk/filter/api/summarization/textrank_summarization.py diff --git a/middleware/filter/api/summarization/tfidf_summarization.py b/sdk/filter/api/summarization/tfidf_summarization.py similarity index 100% rename from middleware/filter/api/summarization/tfidf_summarization.py rename to sdk/filter/api/summarization/tfidf_summarization.py diff --git a/middleware/filter/api/text_similarity/__init__.py b/sdk/filter/api/text_similarity/__init__.py similarity index 100% rename from middleware/filter/api/text_similarity/__init__.py rename to sdk/filter/api/text_similarity/__init__.py diff --git a/middleware/filter/api/text_similarity/cosion.py b/sdk/filter/api/text_similarity/cosion.py similarity index 100% rename from middleware/filter/api/text_similarity/cosion.py rename to sdk/filter/api/text_similarity/cosion.py diff --git a/middleware/filter/api/text_similarity/edit.py b/sdk/filter/api/text_similarity/edit.py similarity index 100% rename from middleware/filter/api/text_similarity/edit.py rename to sdk/filter/api/text_similarity/edit.py diff --git a/middleware/filter/api/text_similarity/simhash.py b/sdk/filter/api/text_similarity/simhash.py similarity index 100% rename from middleware/filter/api/text_similarity/simhash.py rename to sdk/filter/api/text_similarity/simhash.py diff --git a/middleware/filter/evaluate.py b/sdk/filter/evaluate.py similarity index 100% rename from middleware/filter/evaluate.py rename to sdk/filter/evaluate.py diff --git a/middleware/filter/langdetect_fasttext/LICENSE b/sdk/filter/langdetect_fasttext/LICENSE similarity index 100% rename from middleware/filter/langdetect_fasttext/LICENSE rename to sdk/filter/langdetect_fasttext/LICENSE diff --git a/middleware/filter/langdetect_fasttext/README.md b/sdk/filter/langdetect_fasttext/README.md similarity index 100% rename from middleware/filter/langdetect_fasttext/README.md rename to sdk/filter/langdetect_fasttext/README.md diff --git a/middleware/filter/langdetect_fasttext/__init__.py b/sdk/filter/langdetect_fasttext/__init__.py similarity index 100% rename from middleware/filter/langdetect_fasttext/__init__.py rename to sdk/filter/langdetect_fasttext/__init__.py diff --git a/middleware/filter/langdetect_fasttext/detect.py b/sdk/filter/langdetect_fasttext/detect.py similarity index 100% rename from middleware/filter/langdetect_fasttext/detect.py rename to sdk/filter/langdetect_fasttext/detect.py diff --git a/middleware/filter/langdetect_unicode/LICENSE b/sdk/filter/langdetect_unicode/LICENSE similarity index 100% rename from middleware/filter/langdetect_unicode/LICENSE rename to sdk/filter/langdetect_unicode/LICENSE diff --git a/middleware/filter/langdetect_unicode/__init__.py b/sdk/filter/langdetect_unicode/__init__.py similarity index 100% rename from middleware/filter/langdetect_unicode/__init__.py rename to sdk/filter/langdetect_unicode/__init__.py diff --git a/middleware/filter/langdetect_unicode/langdetect.py b/sdk/filter/langdetect_unicode/langdetect.py similarity index 100% rename from middleware/filter/langdetect_unicode/langdetect.py rename to sdk/filter/langdetect_unicode/langdetect.py diff --git a/middleware/filter/reduce.py b/sdk/filter/reduce.py similarity index 100% rename from middleware/filter/reduce.py rename to sdk/filter/reduce.py diff --git a/middleware/filter/sublimate.py b/sdk/filter/sublimate.py similarity index 100% rename from middleware/filter/sublimate.py rename to sdk/filter/sublimate.py diff --git a/sender/app.py b/sender/app.py index d14b21f..d17110a 100644 --- a/sender/app.py +++ b/sender/app.py @@ -27,8 +27,7 @@ rss_app = RssApp() func = [ - telegram_bot.polling( - non_stop=True, + telegram_bot.infinity_polling( allowed_updates=util.update_types, skip_pending=True, timeout=60,