tokenizer: fix 0 byte following invalid (ziglang#21482)

closes ziglang#21481
DivergentClouds · Sep 24, 2024 · 9df598a · 9df598a
1 parent 10c0dec
commit 9df598a
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/lib/std/zig/tokenizer.zig b/lib/std/zig/tokenizer.zig
@@ -535,6 +535,8 @@ pub const Tokenizer = struct {
                 switch (self.buffer[self.index]) {
                     0 => if (self.index == self.buffer.len) {
                         result.tag = .invalid;
+                    } else {
+                        continue :state .invalid;
                     },
                     '\n' => result.tag = .invalid,
                     else => continue :state .invalid,
@@ -1250,6 +1252,7 @@ test "invalid token characters" {
 
 test "invalid literal/comment characters" {
     try testTokenize("\"\x00\"", &.{.invalid});
+    try testTokenize("`\x00`", &.{.invalid});
     try testTokenize("//\x00", &.{.invalid});
     try testTokenize("//\x1f", &.{.invalid});
     try testTokenize("//\x7f", &.{.invalid});