Merge pull request #60 from jansorg/jansorg/path-antiquotations

Support antiquotations in paths

src/main/java/org/nixos/idea/lang/NixParserDefinition.java

@@ -77,6 +77,14 @@ public SpaceRequirements spaceExistenceTypeBetweenTokens(ASTNode left, ASTNode r
         if (leftType == NixTypes.DOLLAR && rightType == NixTypes.LCURLY) {
             return SpaceRequirements.MUST_NOT;
         }
+        else if (leftType == NixTypes.PATH_SEGMENT) {
+            // path segment, antiquotation or PATH_END on the right
+            return SpaceRequirements.MUST_NOT;
+        }
+        else if (rightType == NixTypes.PATH_END) {
+            // path segment or antiquotation on the left
+            return SpaceRequirements.MUST_NOT;
+        }
         else if (NixTypeUtil.MIGHT_COLLAPSE_WITH_ID.contains(leftType) &&
                  NixTypeUtil.MIGHT_COLLAPSE_WITH_ID.contains(rightType)) {
             return SpaceRequirements.MUST;

src/main/java/org/nixos/idea/lang/highlighter/NixSyntaxHighlighter.java

@@ -66,8 +66,7 @@ public class NixSyntaxHighlighter extends SyntaxHighlighterBase {
             // Literals
             entry(NixTypes.INT, NixTextAttributes.NUMBER),
             entry(NixTypes.FLOAT, NixTextAttributes.NUMBER),
-            entry(NixTypes.PATH, NixTextAttributes.PATH),
-            entry(NixTypes.HPATH, NixTextAttributes.PATH),
+            entry(NixTypes.PATH_SEGMENT, NixTextAttributes.PATH),
             entry(NixTypes.SPATH, NixTextAttributes.PATH),
             entry(NixTypes.URI, NixTextAttributes.URI),
             // String literals

src/main/java/org/nixos/idea/psi/NixTypeUtil.java

@@ -24,9 +24,9 @@ public final class NixTypeUtil {
                     NixTypes.ID,
                     NixTypes.INT,
                     NixTypes.FLOAT,
-                    NixTypes.PATH,
-                    NixTypes.HPATH,
                     NixTypes.SPATH,
+                    NixTypes.PATH_SEGMENT,
+                    NixTypes.PATH_END,
                     NixTypes.URI));
 
     private NixTypeUtil() {} // Cannot be instantiated

src/main/java/org/nixos/idea/psi/impl/NixParserUtil.java

@@ -3,6 +3,7 @@
 import com.intellij.lang.PsiBuilder;
 import com.intellij.lang.parser.GeneratedParserUtilBase;
 import com.intellij.openapi.util.Key;
+import com.intellij.psi.TokenType;
 import org.jetbrains.annotations.NotNull;
 
 public final class NixParserUtil extends GeneratedParserUtilBase {

src/main/lang/Nix.bnf

@@ -169,7 +169,9 @@ expr_simple ::=
   | list
   | legacy_let
 identifier ::= ID
-literal ::= INT | FLOAT | PATH | HPATH | SPATH | URI
+literal ::= INT | FLOAT | URI | path
+;{ extends("path")=literal }
+path ::= SPATH | PATH_SEGMENT (PATH_SEGMENT | antiquotation)* PATH_END
 parens ::= LPAREN expr recover_parens RPAREN { pin=1 }
 set ::= [ REC ] LCURLY recover_set (bind recover_set)* RCURLY { pin=2 }
 list ::= LBRAC recover_list (expr_select recover_list)* RBRAC { pin=1 }

src/main/lang/Nix.flex

@@ -43,15 +43,17 @@ import static org.nixos.idea.psi.NixTypes.*;
 %function advance
 %type IElementType
 %unicode
-%state BLOCK STRING IND_STRING ANTIQUOTATION_START ANTIQUOTATION
+%state BLOCK STRING IND_STRING ANTIQUOTATION_START ANTIQUOTATION PATH
 
 ANY=[^]
 ID=[a-zA-Z_][a-zA-Z0-9_'-]*
 INT=[0-9]+
 FLOAT=(([1-9][0-9]*\.[0-9]*)|(0?\.[0-9]+))([Ee][+-]?[0-9]+)?
-PATH=[a-zA-Z0-9._+-]*(\/[a-zA-Z0-9._+-]+)+\/?
-HPATH=\~(\/[a-zA-Z0-9._+-]+)+\/?
-SPATH=\<[a-zA-Z0-9._+-]+(\/[a-zA-Z0-9._+-]+)*\>
+PATH_CHAR=[a-zA-Z0-9\.\_\-\+]
+PATH={PATH_CHAR}*(\/{PATH_CHAR}+)+\/?
+PATH_SEG={PATH_CHAR}*\/
+HPATH_START=\~\/
+SPATH=\<{PATH_CHAR}+(\/{PATH_CHAR}+)*\>
 URI=[a-zA-Z][a-zA-Z0-9.+-]*\:[a-zA-Z0-9%/?:@&=+$,\-_.!~*']+
 
 WHITE_SPACE=[\ \t\r\n]+
@@ -91,6 +93,16 @@ MCOMMENT=\/\*([^*]|\*[^\/])*\*\/
   "}"                   { popState(BLOCK); return RCURLY; }
 }
 
+<PATH> {
+  "$"/"{"               { pushState(ANTIQUOTATION_START); return DOLLAR; }
+  {PATH_SEG}            { return PATH_SEGMENT; }
+  {PATH_CHAR}+          { return PATH_SEGMENT; }
+  // anything else, e.g. whitespace, stops lexing of a PATH
+  // we're delegating back to the parent state
+  // PATH_END is an empty-length token to signal the end of the path
+  [^]                   { popState(PATH); yypushback(yylength()); return PATH_END; }
+}
+
 <YYINITIAL, BLOCK, ANTIQUOTATION> {
   "if"                  { return IF; }
   "then"                { return THEN; }
@@ -146,8 +158,10 @@ MCOMMENT=\/\*([^*]|\*[^\/])*\*\/
   {ID}                  { return ID; }
   {INT}                 { return INT; }
   {FLOAT}               { return FLOAT; }
-  {PATH}                { return PATH; }
-  {HPATH}               { return HPATH; }
+  "/" / "${"            { pushState(PATH); return PATH_SEGMENT; }
+  {PATH}
+  | {PATH_SEG}
+  | {HPATH_START}       { pushState(PATH); return PATH_SEGMENT; }
   {SPATH}               { return SPATH; }
   {URI}                 { return URI; }
 

src/test/java/org/nixos/idea/lang/ParsingTest.java

@@ -145,10 +145,13 @@ public void testWith() {
 
   @Override
   protected void tearDown() throws Exception {
-    // Ensure that the parser does not generate errors even when the errors have
-    // accidentally been added to the expected result.
-    ensureNoErrorElements();
-    super.tearDown();
+    try {
+      // Ensure that the parser does not generate errors even when the errors have
+      // accidentally been added to the expected result.
+      ensureNoErrorElements();
+    } finally {
+      super.tearDown();
+    }
   }
 
   @Override

src/test/java/org/nixos/idea/lang/highlighter/NixSyntaxHighlighterTest.java

@@ -3,6 +3,7 @@
 import com.intellij.openapi.editor.colors.TextAttributesKey;
 import com.intellij.psi.TokenType;
 import com.intellij.psi.tree.IElementType;
+import com.intellij.psi.tree.TokenSet;
 import org.jetbrains.annotations.NotNull;
 import org.junit.jupiter.api.Named;
 import org.junit.jupiter.api.Test;
@@ -15,12 +16,12 @@
 import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
-import static org.junit.jupiter.api.Assertions.assertAll;
-import static org.junit.jupiter.api.Assertions.assertEquals;
-import static org.junit.jupiter.api.Assertions.assertNotEquals;
-import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static java.util.function.Predicate.not;
+import static org.junit.jupiter.api.Assertions.*;
 
 final class NixSyntaxHighlighterTest {
+    private static final TokenSet EMPTY_LENGTH_TOKENS = TokenSet.create(NixTypes.PATH_END);
+
     @Test
     void testAttributesKeysForUnknownTokenType() {
         TextAttributesKey[] tokenHighlights = new NixSyntaxHighlighter().getTokenHighlights(TokenType.CODE_FRAGMENT);
@@ -38,9 +39,10 @@ void testAttributesKeysForKnownTokenTypes(@NotNull IElementType tokenType) {
                 () -> assertNotNull(tokenHighlights[index], String.format("tokenHighlights[%d]", index))));
     }
 
-    static @NotNull Stream<Named<IElementType>> testAttributesKeysForKnownTokenTypes() {
+    static @NotNull Stream<Named<? extends IElementType>> testAttributesKeysForKnownTokenTypes() {
         return Stream.concat(
                 Stream.of(Named.of("TokenType.BAD_CHARACTER", TokenType.BAD_CHARACTER)),
-                ReflectionUtils.getPublicStaticFieldValues(NixTypes.class, NixTokenType.class));
+                ReflectionUtils.getPublicStaticFieldValues(NixTypes.class, NixTokenType.class)
+        ).filter(not(named -> EMPTY_LENGTH_TOKENS.contains(named.getPayload())));
     }
 }

src/test/testData/ParsingTest/List.lexer.txt

@@ -2,7 +2,8 @@
 WHITE_SPACE (' ')
 INT ('123')
 WHITE_SPACE (' ')
-PATH ('./foo.nix')
+PATH_SEGMENT ('./foo.nix')
+PATH_END ('')
 WHITE_SPACE (' ')
 STRING_OPEN ('"')
 STR ('abc')

src/test/testData/ParsingTest/List.txt

@@ -5,8 +5,8 @@ Nix File(0,36)
     NixLiteralImpl(LITERAL)(2,5)
       PsiElement(INT)('123')(2,5)
     PsiWhiteSpace(' ')(5,6)
-    NixLiteralImpl(LITERAL)(6,15)
-      PsiElement(PATH)('./foo.nix')(6,15)
+    NixPathImpl(PATH)(6,15)
+      PsiElement(PATH_SEGMENT)('./foo.nix')(6,15)
     PsiWhiteSpace(' ')(15,16)
     NixStdStringImpl(STD_STRING)(16,21)
       PsiElement(STRING_OPEN)('"')(16,17)
@@ -33,4 +33,4 @@ Nix File(0,36)
       PsiWhiteSpace(' ')(32,33)
       PsiElement(})('}')(33,34)
     PsiWhiteSpace(' ')(34,35)
-    PsiElement(])(']')(35,36)
+    PsiElement(])(']')(35,36)

src/test/testData/ParsingTest/ListWithFunction.lexer.txt

@@ -2,7 +2,8 @@
 WHITE_SPACE (' ')
 INT ('123')
 WHITE_SPACE (' ')
-PATH ('./foo.nix')
+PATH_SEGMENT ('./foo.nix')
+PATH_END ('')
 WHITE_SPACE (' ')
 STRING_OPEN ('"')
 STR ('abc')

src/test/testData/ParsingTest/ListWithFunction.txt

@@ -5,8 +5,8 @@ Nix File(0,38)
     NixLiteralImpl(LITERAL)(2,5)
       PsiElement(INT)('123')(2,5)
     PsiWhiteSpace(' ')(5,6)
-    NixLiteralImpl(LITERAL)(6,15)
-      PsiElement(PATH)('./foo.nix')(6,15)
+    NixPathImpl(PATH)(6,15)
+      PsiElement(PATH_SEGMENT)('./foo.nix')(6,15)
     PsiWhiteSpace(' ')(15,16)
     NixStdStringImpl(STD_STRING)(16,21)
       PsiElement(STRING_OPEN)('"')(16,17)
@@ -37,4 +37,4 @@ Nix File(0,38)
           PsiElement(})('}')(34,35)
       PsiElement())(')')(35,36)
     PsiWhiteSpace(' ')(36,37)
-    PsiElement(])(']')(37,38)
+    PsiElement(])(']')(37,38)

src/test/testData/ParsingTest/Path.lexer.txt

@@ -1,11 +1,123 @@
 [ ('[')
 WHITE_SPACE ('\n')
-PATH ('/bin/sh')
+PATH_SEGMENT ('/bin/sh')
+PATH_END ('')
 WHITE_SPACE ('\n')
-PATH ('./builder.sh')
+PATH_SEGMENT ('./builder.sh')
+PATH_END ('')
 WHITE_SPACE ('\n')
-HPATH ('~/foo')
+PATH_SEGMENT ('~/')
+PATH_SEGMENT ('foo')
+PATH_END ('')
 WHITE_SPACE ('\n')
 SPATH ('<nixpkgs>')
 WHITE_SPACE ('\n')
-] (']')
+SCOMMENT ('# antiquotations')
+WHITE_SPACE ('\n')
+PATH_SEGMENT ('/')
+$ ('$')
+{ ('{')
+ID ('fileName')
+} ('}')
+PATH_END ('')
+WHITE_SPACE ('\n')
+PATH_SEGMENT ('/')
+$ ('$')
+{ ('{')
+ID ('fileName')
+} ('}')
+PATH_SEGMENT ('/')
+PATH_END ('')
+WHITE_SPACE ('\n')
+PATH_SEGMENT ('./')
+$ ('$')
+{ ('{')
+ID ('foo')
+} ('}')
+PATH_SEGMENT ('-')
+$ ('$')
+{ ('{')
+ID ('bar')
+} ('}')
+PATH_SEGMENT ('.nix')
+PATH_END ('')
+WHITE_SPACE ('\n')
+PATH_SEGMENT ('./')
+$ ('$')
+{ ('{')
+ID ('foo')
+} ('}')
+PATH_SEGMENT ('-')
+$ ('$')
+{ ('{')
+ID ('bar')
+} ('}')
+PATH_SEGMENT ('/')
+PATH_END ('')
+WHITE_SPACE ('\n')
+STRING_OPEN ('"')
+$ ('$')
+{ ('{')
+PATH_SEGMENT ('./foo.txt')
+PATH_END ('')
+} ('}')
+STRING_CLOSE ('"')
+WHITE_SPACE ('\n\n')
+SCOMMENT ('# whitespace must not be part of paths')
+WHITE_SPACE ('\n')
+PATH_SEGMENT ('prefix/dir/file.txt')
+PATH_END ('')
+WHITE_SPACE (' ')
+PATH_SEGMENT ('next/path/element')
+PATH_END ('')
+WHITE_SPACE ('\n\n')
+SCOMMENT ('# At least one slash (/) must appear before any interpolated expression for the result to be recognized as a path.')
+WHITE_SPACE ('\n')
+PATH_SEGMENT ('a/b')
+PATH_END ('')
+WHITE_SPACE ('\n\n')
+SCOMMENT ('# https://nixos.org/manual/nix/stable/language/values.html#type-path')
+WHITE_SPACE ('\n')
+SCOMMENT ('#   a.${foo}/b.${bar} is a syntactically valid division operation.')
+WHITE_SPACE ('\n')
+SCOMMENT ('# but the Nix parser seems to handle this differently:')
+WHITE_SPACE ('\n')
+SCOMMENT ('#   https://github.com/NixOS/nix-idea/issues/59#issuecomment-1494786812')
+WHITE_SPACE ('\n')
+ID ('a')
+. ('.')
+$ ('$')
+{ ('{')
+ID ('foo')
+} ('}')
+PATH_SEGMENT ('/b.')
+$ ('$')
+{ ('{')
+ID ('bar')
+} ('}')
+PATH_END ('')
+WHITE_SPACE ('\n\n')
+SCOMMENT ('# ./a.${foo}/b.${bar} is a path.')
+WHITE_SPACE ('\n')
+PATH_SEGMENT ('./a.')
+$ ('$')
+{ ('{')
+ID ('foo')
+} ('}')
+PATH_SEGMENT ('/')
+PATH_SEGMENT ('b.')
+$ ('$')
+{ ('{')
+ID ('bar')
+} ('}')
+PATH_END ('')
+WHITE_SPACE ('\n\n')
+SCOMMENT ('# trailing slashes')
+WHITE_SPACE ('\n')
+PATH_SEGMENT ('/dir/subdir/')
+PATH_END ('')
+WHITE_SPACE ('\n')
+PATH_SEGMENT ('./dir/subdir/')
+PATH_END ('')
+WHITE_SPACE ('\n')
+] (']')

src/test/testData/ParsingTest/Path.nix

@@ -3,4 +3,29 @@
 ./builder.sh
 ~/foo
 <nixpkgs>
+# antiquotations
+/${fileName}
+/${fileName}/
+./${foo}-${bar}.nix
+./${foo}-${bar}/
+"${./foo.txt}"
+
+# whitespace must not be part of paths
+prefix/dir/file.txt next/path/element
+
+# At least one slash (/) must appear before any interpolated expression for the result to be recognized as a path.
+a/b
+
+# https://nixos.org/manual/nix/stable/language/values.html#type-path
+#   a.${foo}/b.${bar} is a syntactically valid division operation.
+# but the Nix parser seems to handle this differently:
+#   https://github.com/NixOS/nix-idea/issues/59#issuecomment-1494786812
+a.${foo}/b.${bar}
+
+# ./a.${foo}/b.${bar} is a path.
+./a.${foo}/b.${bar}
+
+# trailing slashes
+/dir/subdir/
+./dir/subdir/
 ]