store local tokenizer for tests to avoid connection to the HF hub clo…

…ud. Disable stack view cache

tapeagents/llms.py

@@ -104,6 +104,8 @@ def log_completion(self, prompt: Prompt, message: LLMMessage, cached: bool = Fal
 # Use this variable to force all LLMs to use cache from the sqlite DB
 # This is meant to be used for testing purposes only
 _REPLAY_SQLITE: str = ""
+# force replacement of the tokenizer during testing
+_MOCK_TOKENIZER: str = ""
 
 
 class CachedLLM(LLM):
@@ -423,7 +425,8 @@ def load_tokenizer(self):
         if self.tokenizer is None:
             import transformers
 
-            self.tokenizer = transformers.AutoTokenizer.from_pretrained(self.tokenizer_name or self.model_name)
+            name = _MOCK_TOKENIZER if _MOCK_TOKENIZER else (self.tokenizer_name or self.model_name)
+            self.tokenizer = transformers.AutoTokenizer.from_pretrained(name)
 
     def make_training_text(self, prompt: Prompt, completion: Completion) -> TrainingText:
         self.load_tokenizer()

tapeagents/view.py

@@ -164,8 +164,8 @@ def put_new_view_on_stack(self, step):
     @staticmethod
     def compute(tape: Tape) -> TapeViewStack[StepType]:
         # TODO: retrieve view from a prefix of the tape, recompute from the prefix
-        if (cached_view_stack := _view_stack_cache.get(id(tape))) is not None:
-            return cached_view_stack
+        # if (cached_view_stack := _view_stack_cache.get(id(tape))) is not None:
+        #     return cached_view_stack
         stack = TapeViewStack(stack=[TapeView(agent_name="root", agent_full_name="root")])
         for step in tape.steps:
             stack.update(step)

tests/res/meta_llama_3_70b_tokenizer/special_tokens_map.json

@@ -0,0 +1,4 @@
+{
+  "bos_token": "<|begin_of_text|>",
+  "eos_token": "<|end_of_text|>"
+}