fix long term mem

middleware/llm_task.py

@@ -73,10 +73,8 @@ def create_message(self, write_back=True):
         for i, message in enumerate(history_messages):
             _history.append(message)
         # 刮削器合并消息，这里评价简写了。
-        _total = 0
         for i, _msg in enumerate(_history):
-            _total += 1
-            self.scraper.add_message(_msg, score=len(str(_msg)), order=_total)
+            self.scraper.add_message(_msg, score=len(str(_msg)))
 
         # 处理附带的任何原始消息
         if write_back:
@@ -88,8 +86,7 @@ def create_message(self, write_back=True):
                 _buffer.append(Message(role="user", content=message.text))
             # 新消息的分数比较高
             for i, _msg in enumerate(_buffer):
-                _total += 1
-                self.scraper.add_message(_msg, score=len(str(_msg)) + 100, order=_total)
+                self.scraper.add_message(_msg, score=len(str(_msg)) + 50)
             # save to history
             for _msg in _buffer:
                 self.message_history.add_message(message=_msg)

plugins/search.py

@@ -28,7 +28,7 @@
 
 def search_on_duckduckgo(search_sentence: str, key_words: str = None):
     from duckduckgo_search import DDGS
-    from middleware.filter.sublimate import Sublimate
+    from sdk.filter import Sublimate
     with DDGS(timeout=20) as ddgs:
         _text = []
         for r in ddgs.text(search_sentence):
@@ -76,7 +76,7 @@ class SearchTool(BaseTool):
     def pre_check(self):
         try:
             from duckduckgo_search import DDGS
-            from middleware.filter.sublimate import Sublimate
+            from sdk.filter import Sublimate
             return True
         except ImportError as e:
             logger.warning(f"plugin:package <duckduckgo_search> not found,please install it first:{e}")

sdk/endpoint/openai/action.py

@@ -11,6 +11,7 @@
 from loguru import logger
 from pydantic import BaseModel
 
+from sdk.filter.evaluate import Sim
 from sdk.schema import Message
 
 
@@ -99,15 +100,19 @@ class Sorter(BaseModel):
     # 最大消息数
     max_messages: int = 12
 
+    # 计数器
+    tick: int = 0
+
     # 方法：添加消息
-    def add_message(self, message: Message, score: float, order: int):
+    def add_message(self, message: Message, score: float):
         if hasattr(message, "function_call"):
             return None
-        self.messages.append(self.Sorter(message=message, score=score, order=order))
+        self.messages.append(self.Sorter(message=message, score=score, order=self.tick))
+        self.tick += 1
         # 按照顺序排序
         self.messages.sort(key=lambda x: x.order)
-        if len(self.messages) > self.max_messages:
-            self.messages.pop()
+        while len(self.messages) > self.max_messages:
+            self.messages.pop(0)
 
     # 方法：获取消息
     def get_messages(self) -> List[Message]:
@@ -120,14 +125,37 @@ def get_messages(self) -> List[Message]:
         logger.debug(_message)
         return _message
 
+    def build_messages(self):
+        # 只取三个，末位匹配
+        _message = self.get_messages()
+        if len(_message) < 3:
+            return _message
+        _build = []
+        _must = _message[-3:]
+        _check_list = _message[:-3]
+        _match_sentence = _message[-1:][0].content
+        for item_obj in _check_list:
+            if Sim.cosion_similarity(pre=_match_sentence, aft=item_obj.content) < 0.9:
+                _build.append(item_obj)
+            else:
+                pass
+                # logger.warning(f"ignore sim item {item_obj}")
+        _build.extend(_must)
+        return _build
+
     # 方法：获取消息数
     def get_num_messages(self) -> int:
         return len(self.messages)
 
     # 方法：清除消息到负载
     def reduce_messages(self, limit: int = 2048):
-        if limit > 100:
+        # 预留位
+        if limit > 1000:
+            limit = limit - 250
+        else:
             limit = limit - 70
+
+        # 执行删除操作
         if TokenizerObj.num_tokens_from_messages(self.get_messages()) > limit:
             # 从最旧开始删除
             self.messages.sort(key=lambda x: x.order)

middleware/filter/__init__.py → sdk/filter/__init__.py

@@ -5,3 +5,4 @@
 # @Software: PyCharm
 
 # 去噪声算法
+from .sublimate import Sublimate

sender/app.py

@@ -27,8 +27,7 @@
 rss_app = RssApp()
 
 func = [
-    telegram_bot.polling(
-        non_stop=True,
+    telegram_bot.infinity_polling(
         allowed_updates=util.update_types,
         skip_pending=True,
         timeout=60,