replace backslashes in path to fix windows extraction, check unrecogn…

…ized content when reading strings
jshmrtn · Aug 21, 2024 · 8f1ff2d · 8f1ff2d
1 parent a32b979
commit 8f1ff2d
Show file tree

Hide file tree

Showing 2 changed files with 7 additions and 4 deletions.
diff --git a/scripts/gettext_extract.ts b/scripts/gettext_extract.ts
@@ -22,10 +22,10 @@ try {
   process.exit(1);
 }
 
-var getFiles = async (config: GettextConfig) => {
+const getFiles = async (config: GettextConfig) => {
   const allFiles = await Promise.all(
     config.input?.include.map((pattern) => {
-      const searchPath = path.join(config.input.path, pattern);
+      const searchPath = path.join(config.input.path, pattern).replace(/\\/g, "/");
       console.info(`Searching: ${chalk.blueBright(searchPath)}`);
       return glob(searchPath);
     }),

diff --git a/src/extract/tokenizer.ts b/src/extract/tokenizer.ts
@@ -38,7 +38,7 @@ export function tokenize(mapping: KeywordMapping, src: string): Token[] {
 
   function addToken(kind: TokenKind, charIndex: number, value?: string) {
     if (unrecognizedContent.trim()) {
-      tokens.push({ kind: TokenKind.Unrecognized, idx });
+      tokens.push({ kind: TokenKind.Unrecognized, idx, value: unrecognizedContent });
       unrecognizedContent = "";
     }
     if (value) {
@@ -99,7 +99,10 @@ export function tokenize(mapping: KeywordMapping, src: string): Token[] {
         // improves robustness as it prevents issues with odd numbers
         // but will also parse calls within string literals
         const prevTokenKind = tokens[tokens.length - 1]?.kind;
-        if (prevTokenKind === TokenKind.ParenLeft || prevTokenKind === TokenKind.Comma) {
+        if (
+          !unrecognizedContent.trim() &&
+          (prevTokenKind === TokenKind.ParenLeft || prevTokenKind === TokenKind.Comma)
+        ) {
           addToken(TokenKind.String, idx, readString(c));
           break;
         }