minor refactor

fastn-stack · Feb 20, 2024 · 7b8d6b2 · 7b8d6b2
1 parent 4c77b17
commit 7b8d6b2
Showing 1 changed file with 22 additions and 13 deletions.
diff --git a/fastn-expr/src/tokenizer.rs b/fastn-expr/src/tokenizer.rs
@@ -48,20 +48,29 @@ pub fn tokenize(input: &str) -> Result<Vec<Token>, TokenizerError> {
                 tokens.push(get_token(&current_token));
                 current_token.clear();
             }
-        } else if (((c == '.' || c == '_') && !current_token.is_empty()) || c.is_alphanumeric())
-            || (c == '-' && current_token.is_empty())
-        {
-            current_token.push(c);
-        } else if c == '"' {
-            in_string_literal = true;
-        } else if !current_token.is_empty() {
-            tokens.push(get_token(&current_token));
-            current_token.clear();
         } else {
-            return Err(TokenizerError::UnexpectedToken {
-                token: c,
-                position: pos,
-            });
+            match c {
+                '.' | '_' if !current_token.is_empty() => {
+                    current_token.push(c);
+                }
+                '-' if current_token.is_empty() => {
+                    current_token.push(c);
+                }
+                '"' => in_string_literal = true,
+                _ => {
+                    if c.is_alphanumeric() {
+                        current_token.push(c);
+                    } else if !current_token.is_empty() {
+                        tokens.push(get_token(&current_token));
+                        current_token.clear();
+                    } else {
+                        return Err(TokenizerError::UnexpectedToken {
+                            token: c,
+                            position: pos,
+                        });
+                    }
+                }
+            }
         }
     }