Ensure the tokenizer initialises its state

CHANGELOG.md

@@ -5,6 +5,14 @@ Changelog
 
 --------
 
+## 3.1.1 - 2024-10-18
+
+### Fixed
+
+* [Tokenizer] Correct a state bug that made it impossible to tokenize/parse multiple messages ([#14](https://github.com/estratocloud/edifact/issues/14)).
+
+--------
+
 ## 3.1.0 - 2024-01-19
 
 ### Added

src/Tokenizer.php

@@ -57,9 +57,11 @@ final class Tokenizer implements TokenizerInterface
     public function getTokens(string $message, ControlCharactersInterface $characters): array
     {
         $this->message = $message;
+        $this->position = 0;
         $this->characters = $characters;
         $this->char = "";
         $this->string = "";
+        $this->isEscaped = false;
 
         $this->readNextChar();
 

tests/TokenizerTest.php

@@ -47,6 +47,30 @@ public function testBasic(): void
     }
 
 
+    /**
+     * Regression test for https://github.com/estratocloud/edifact/issues/14
+     */
+    public function testMultiple(): void
+    {
+        $this->assertTokens("RFF+PD:50515", [
+            new Token(Token::CONTENT, "RFF"),
+            new Token(Token::DATA_SEPARATOR, "+"),
+            new Token(Token::CONTENT, "PD"),
+            new Token(Token::COMPONENT_SEPARATOR, ":"),
+            new Token(Token::CONTENT, "50515"),
+        ]);
+
+        # Ensure we can use the same tokenizer instance for multiple messages
+        $this->assertTokens("RFF+PD:50515", [
+            new Token(Token::CONTENT, "RFF"),
+            new Token(Token::DATA_SEPARATOR, "+"),
+            new Token(Token::CONTENT, "PD"),
+            new Token(Token::COMPONENT_SEPARATOR, ":"),
+            new Token(Token::CONTENT, "50515"),
+        ]);
+    }
+
+
     public function testEscape(): void
     {
         $this->assertTokens("RFF+PD?:5", [