From 1f42304fb72bda1a598a76011d50a64ac8fdd70f Mon Sep 17 00:00:00 2001 From: Yingchi Long Date: Wed, 10 Apr 2024 06:12:06 +0800 Subject: [PATCH] nixd: support `textDocument/semanticTokens/full` (#408) --- nixd/include/nixd/Controller/Controller.h | 3 + nixd/lib/Controller/LifeTime.cpp | 34 +++ nixd/lib/Controller/SemanticTokens.cpp | 241 ++++++++++++++++++++++ nixd/lib/Controller/Support.cpp | 2 + nixd/meson.build | 1 + nixd/tools/nixd/test/initialize.md | 26 +++ nixd/tools/nixd/test/semantic-tokens.md | 175 ++++++++++++++++ 7 files changed, 482 insertions(+) create mode 100644 nixd/lib/Controller/SemanticTokens.cpp create mode 100644 nixd/tools/nixd/test/semantic-tokens.md diff --git a/nixd/include/nixd/Controller/Controller.h b/nixd/include/nixd/Controller/Controller.h index 17f01fe8e..1f0df49ec 100644 --- a/nixd/include/nixd/Controller/Controller.h +++ b/nixd/include/nixd/Controller/Controller.h @@ -81,6 +81,9 @@ class Controller : public lspserver::LSPServer { const lspserver::DocumentSymbolParams &Params, lspserver::Callback> Reply); + void onSemanticTokens(const lspserver::SemanticTokensParams &Params, + lspserver::Callback Reply); + void onDefinition(const lspserver::TextDocumentPositionParams &Params, lspserver::Callback Reply); diff --git a/nixd/lib/Controller/LifeTime.cpp b/nixd/lib/Controller/LifeTime.cpp index 7a83081c0..7d563d4b9 100644 --- a/nixd/lib/Controller/LifeTime.cpp +++ b/nixd/lib/Controller/LifeTime.cpp @@ -38,6 +38,40 @@ void Controller:: }, {"definitionProvider", true}, {"documentSymbolProvider", true}, + { + "semanticTokensProvider", + Object{ + { + "legend", + Object{ + {"tokenTypes", + Array{ + "function", // function + "string", // string + "number", // number + "type", // select + "keyword", // builtin + "variable", // constant + "interface", // fromWith + "variable", // variable + "regexp", // null + "macro", // bool + "method", // attrname + "regexp", // lambdaArg + "regexp", // lambdaFormal + }}, + {"tokenModifiers", + Array{ + "static", // builtin + "abstract", // deprecated + "async", // dynamic + }}, + }, + }, + {"range", false}, + {"full", true}, + }, + }, {"referencesProvider", true}, {"documentHighlightProvider", true}, {"hoverProvider", true}, diff --git a/nixd/lib/Controller/SemanticTokens.cpp b/nixd/lib/Controller/SemanticTokens.cpp new file mode 100644 index 000000000..6c4a57e7d --- /dev/null +++ b/nixd/lib/Controller/SemanticTokens.cpp @@ -0,0 +1,241 @@ +/// \file +/// \brief Implementation of [Semantic Tokens]. +/// [Semantic Tokens]: +/// https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_semanticTokens + +#include "nixd/Controller/Controller.h" + +#include +#include +#include +#include +#include +#include +#include + +using namespace nixd; +using namespace lspserver; +using namespace nixf; + +namespace { + +enum SemaType { + ST_Function, + ST_String, + ST_Number, + ST_Select, + ST_Builtin, + ST_Defined, + ST_FromWith, + ST_Undefined, + ST_Null, + ST_Bool, + ST_AttrName, + ST_LambdaArg, + ST_LambdaFormal, +}; + +enum SemaModifiers { + SM_Builtin = 1 << 0, + SM_Deprecated = 1 << 1, + SM_Dynamic = 1 << 2, +}; + +struct RawSemanticToken { + lspserver::Position Pos; + bool operator<(const RawSemanticToken &Other) const { + return Pos < Other.Pos; + } + unsigned Length; + unsigned TokenType; + unsigned TokenModifiers; +}; + +class SemanticTokenBuilder { + + const VariableLookupAnalysis &VLA; + + nixf::Position Previous = {0, 0}; + + std::vector Raw; + +public: + SemanticTokenBuilder(const VariableLookupAnalysis &VLA) : VLA(VLA) {} + void addImpl(nixf::LexerCursor Pos, unsigned Length, unsigned TokenType, + unsigned TokenModifiers) { + Raw.emplace_back(RawSemanticToken{ + {static_cast(Pos.line()), static_cast(Pos.column())}, + Length, + TokenType, + TokenModifiers}); + } + + void add(const Node &N, unsigned TokenType, unsigned TokenModifiers) { + if (skip(N)) + return; + addImpl(N.lCur(), len(N), TokenType, TokenModifiers); + } + + static bool skip(const Node &N) { + // Skip cross-line strings. + return N.range().lCur().line() != N.range().rCur().line(); + } + + static unsigned len(const Node &N) { + return N.range().rCur().offset() - N.range().lCur().offset(); + } + + void dfs(const ExprString &Str) { + unsigned Modifers = 0; + if (!Str.isLiteral()) + return; + add(Str, ST_String, Modifers); + } + + void dfs(const ExprVar &Var) { + if (Var.id().name() == "true" || Var.id().name() == "false") { + add(Var, ST_Bool, SM_Builtin); + return; + } + + if (Var.id().name() == "null") { + add(Var, ST_Null, 0); + return; + } + + auto Result = VLA.query(Var); + using ResultKind = VariableLookupAnalysis::LookupResultKind; + if (Result.Def && Result.Def->isBuiltin()) { + add(Var, ST_Builtin, SM_Builtin); + return; + } + if (Result.Kind == ResultKind::Defined) { + add(Var, ST_Defined, 0); + return; + } + if (Result.Kind == ResultKind::FromWith) { + add(Var, ST_FromWith, SM_Dynamic); + return; + } + + add(Var, ST_Defined, SM_Deprecated); + } + + void dfs(const ExprSelect &Select) { + dfs(&Select.expr()); + dfs(Select.defaultExpr()); + if (!Select.path()) + return; + for (const std::shared_ptr &Name : Select.path()->names()) { + if (!Name) + continue; + const AttrName &AN = *Name; + if (AN.isStatic()) { + if (AN.kind() == AttrName::ANK_ID) { + add(AN, ST_Select, 0); + } + } + } + } + + void dfs(const SemaAttrs &SA) { + for (const auto &[Name, Attr] : SA.staticAttrs()) { + if (!Attr.value()) + continue; + add(Attr.key(), ST_AttrName, 0); + dfs(Attr.value()); + } + for (const auto &Attr : SA.dynamicAttrs()) { + dfs(Attr.value()); + } + } + + void dfs(const LambdaArg &Arg) { + if (Arg.id()) + add(*Arg.id(), ST_LambdaArg, 0); + // Color deduplicated formals. + if (Arg.formals()) + for (const auto &[_, Formal] : Arg.formals()->dedup()) { + if (Formal->id()) { + add(*Formal->id(), ST_LambdaFormal, 0); + } + } + } + + void dfs(const ExprLambda &Lambda) { + if (Lambda.arg()) { + dfs(*Lambda.arg()); + } + dfs(Lambda.body()); + } + + void dfs(const Node *AST) { + if (!AST) + return; + switch (AST->kind()) { + case Node::NK_ExprLambda: { + const auto &Lambda = static_cast(*AST); + dfs(Lambda); + break; + } + case Node::NK_ExprString: { + const auto &Str = static_cast(*AST); + dfs(Str); + break; + } + case Node::NK_ExprVar: { + const auto &Var = static_cast(*AST); + dfs(Var); + break; + } + case Node::NK_ExprSelect: { + const auto &Select = static_cast(*AST); + dfs(Select); + break; + } + case Node::NK_ExprAttrs: { + const SemaAttrs &SA = static_cast(*AST).sema(); + dfs(SA); + break; + } + default: + for (const Node *Ch : AST->children()) { + dfs(Ch); + } + } + }; + + std::vector finish() { + std::vector Tokens; + std::sort(Raw.begin(), Raw.end()); + lspserver::Position Prev{0, 0}; + for (auto Elm : Raw) { + assert(Elm.Pos.line - Prev.line >= 0); + unsigned DeltaLine = Elm.Pos.line - Prev.line; + unsigned DeltaCol = + DeltaLine ? Elm.Pos.character : Elm.Pos.character - Prev.character; + Prev = Elm.Pos; + Tokens.emplace_back(DeltaLine, DeltaCol, Elm.Length, Elm.TokenType, + Elm.TokenModifiers); + } + return Tokens; + } +}; + +} // namespace + +void Controller::onSemanticTokens(const SemanticTokensParams &Params, + Callback Reply) { + auto Action = [Reply = std::move(Reply), URI = Params.textDocument.uri, + this]() mutable { + if (std::shared_ptr TU = + getTU(URI.file().str(), Reply, /*Ignore=*/true)) { + if (std::shared_ptr AST = getAST(*TU, Reply)) { + SemanticTokenBuilder Builder(*TU->variableLookup()); + Builder.dfs(AST.get()); + Reply(SemanticTokens{.tokens = Builder.finish()}); + } + } + }; + boost::asio::post(Pool, std::move(Action)); +} diff --git a/nixd/lib/Controller/Support.cpp b/nixd/lib/Controller/Support.cpp index 91049d0d7..d57247ced 100644 --- a/nixd/lib/Controller/Support.cpp +++ b/nixd/lib/Controller/Support.cpp @@ -127,6 +127,8 @@ Controller::Controller(std::unique_ptr In, &Controller::onDefinition); Registry.addMethod("textDocument/documentSymbol", this, &Controller::onDocumentSymbol); + Registry.addMethod("textDocument/semanticTokens/full", this, + &Controller::onSemanticTokens); Registry.addMethod("textDocument/references", this, &Controller::onReferences); Registry.addMethod("textDocument/documentHighlight", this, diff --git a/nixd/meson.build b/nixd/meson.build index c38471951..7d13af9eb 100644 --- a/nixd/meson.build +++ b/nixd/meson.build @@ -16,6 +16,7 @@ libnixd_lib = library( 'lib/Controller/LifeTime.cpp', 'lib/Controller/NixTU.cpp', 'lib/Controller/Rename.cpp', + 'lib/Controller/SemanticTokens.cpp', 'lib/Controller/Support.cpp', 'lib/Controller/TextDocumentSync.cpp', 'lib/Eval/EvalProvider.cpp', diff --git a/nixd/tools/nixd/test/initialize.md b/nixd/tools/nixd/test/initialize.md index 3012c4ebc..f8bd4f5f2 100644 --- a/nixd/tools/nixd/test/initialize.md +++ b/nixd/tools/nixd/test/initialize.md @@ -40,6 +40,32 @@ CHECK-NEXT: "referencesProvider": true, CHECK-NEXT: "renameProvider": { CHECK-NEXT: "prepareProvider": true CHECK-NEXT: }, +CHECK-NEXT: "semanticTokensProvider": { +CHECK-NEXT: "full": true, +CHECK-NEXT: "legend": { +CHECK-NEXT: "tokenModifiers": [ +CHECK-NEXT: "static", +CHECK-NEXT: "abstract", +CHECK-NEXT: "async" +CHECK-NEXT: ], +CHECK-NEXT: "tokenTypes": [ +CHECK-NEXT: "function", +CHECK-NEXT: "string", +CHECK-NEXT: "number", +CHECK-NEXT: "type", +CHECK-NEXT: "keyword", +CHECK-NEXT: "variable", +CHECK-NEXT: "interface", +CHECK-NEXT: "variable", +CHECK-NEXT: "regexp", +CHECK-NEXT: "macro", +CHECK-NEXT: "method", +CHECK-NEXT: "regexp", +CHECK-NEXT: "regexp" +CHECK-NEXT: ] +CHECK-NEXT: }, +CHECK-NEXT: "range": false +CHECK-NEXT: }, CHECK-NEXT: "textDocumentSync": { CHECK-NEXT: "change": 2, CHECK-NEXT: "openClose": true, diff --git a/nixd/tools/nixd/test/semantic-tokens.md b/nixd/tools/nixd/test/semantic-tokens.md new file mode 100644 index 000000000..1236da2fc --- /dev/null +++ b/nixd/tools/nixd/test/semantic-tokens.md @@ -0,0 +1,175 @@ +# RUN: nixd --lit-test < %s | FileCheck %s + +<-- initialize(0) + +```json +{ + "jsonrpc":"2.0", + "id":0, + "method":"initialize", + "params":{ + "processId":123, + "rootPath":"", + "capabilities":{ + }, + "trace":"off" + } +} +``` + + +<-- textDocument/didOpen + + +```nix +{ + x = 1; + anonymousLambda = { a }: a; + namedLambda = a: a; + numbers = 1 + 2.0; + bool = true; + bool2= false; + string = "1"; + string2 = "${builtins.foo}"; + undefined = x; + list = [ 1 2 3 ]; + null = null; + wit = with builtins; [ a b c]; +} +``` + +```json +{ + "jsonrpc":"2.0", + "method":"textDocument/didOpen", + "params":{ + "textDocument":{ + "uri":"file:///basic.nix", + "languageId":"nix", + "version":1, + "text":"{\n x = 1;\n anonymousLambda = { a }: a;\n namedLambda = a: a;\n numbers = 1 + 2.0;\n bool = true;\n bool2= false;\n string = \"1\";\n string2 = \"${builtins.foo}\";\n undefined = x;\n list = [ 1 2 3 ];\n}\n" + } + } +} +``` + +<-- textDocument/semanticTokens(2) + + +```json +{ + "jsonrpc":"2.0", + "id":2, + "method":"textDocument/semanticTokens/full", + "params":{ + "textDocument":{ + "uri":"file:///basic.nix" + } + } +} +``` + +``` + CHECK: "id": 2, +CHECK-NEXT: "jsonrpc": "2.0", +CHECK-NEXT: "result": { +CHECK-NEXT: "data": [ +CHECK-NEXT: 1, +CHECK-NEXT: 4, +CHECK-NEXT: 1, +CHECK-NEXT: 10, +CHECK-NEXT: 0, +CHECK-NEXT: 1, +CHECK-NEXT: 4, +CHECK-NEXT: 15, +CHECK-NEXT: 10, +CHECK-NEXT: 0, +CHECK-NEXT: 0, +CHECK-NEXT: 20, +CHECK-NEXT: 1, +CHECK-NEXT: 12, +CHECK-NEXT: 0, +CHECK-NEXT: 0, +CHECK-NEXT: 5, +CHECK-NEXT: 1, +CHECK-NEXT: 5, +CHECK-NEXT: 0, +CHECK-NEXT: 1, +CHECK-NEXT: 4, +CHECK-NEXT: 11, +CHECK-NEXT: 10, +CHECK-NEXT: 0, +CHECK-NEXT: 0, +CHECK-NEXT: 14, +CHECK-NEXT: 1, +CHECK-NEXT: 11, +CHECK-NEXT: 0, +CHECK-NEXT: 0, +CHECK-NEXT: 3, +CHECK-NEXT: 1, +CHECK-NEXT: 5, +CHECK-NEXT: 0, +CHECK-NEXT: 1, +CHECK-NEXT: 4, +CHECK-NEXT: 7, +CHECK-NEXT: 10, +CHECK-NEXT: 0, +CHECK-NEXT: 1, +CHECK-NEXT: 4, +CHECK-NEXT: 4, +CHECK-NEXT: 10, +CHECK-NEXT: 0, +CHECK-NEXT: 0, +CHECK-NEXT: 7, +CHECK-NEXT: 4, +CHECK-NEXT: 9, +CHECK-NEXT: 1, +CHECK-NEXT: 1, +CHECK-NEXT: 4, +CHECK-NEXT: 5, +CHECK-NEXT: 10, +CHECK-NEXT: 0, +CHECK-NEXT: 0, +CHECK-NEXT: 7, +CHECK-NEXT: 5, +CHECK-NEXT: 9, +CHECK-NEXT: 1, +CHECK-NEXT: 1, +CHECK-NEXT: 4, +CHECK-NEXT: 6, +CHECK-NEXT: 10, +CHECK-NEXT: 0, +CHECK-NEXT: 0, +CHECK-NEXT: 9, +CHECK-NEXT: 3, +CHECK-NEXT: 1, +CHECK-NEXT: 0, +CHECK-NEXT: 1, +CHECK-NEXT: 4, +CHECK-NEXT: 7, +CHECK-NEXT: 10, +CHECK-NEXT: 0, +CHECK-NEXT: 1, +CHECK-NEXT: 4, +CHECK-NEXT: 9, +CHECK-NEXT: 10, +CHECK-NEXT: 0, +CHECK-NEXT: 0, +CHECK-NEXT: 12, +CHECK-NEXT: 1, +CHECK-NEXT: 5, +CHECK-NEXT: 2, +CHECK-NEXT: 1, +CHECK-NEXT: 4, +CHECK-NEXT: 4, +CHECK-NEXT: 10, +CHECK-NEXT: 0 +CHECK-NEXT: ], +CHECK-NEXT: "resultId": "" +CHECK-NEXT: } +CHECK-NEXT: } +``` + +```json +{"jsonrpc":"2.0","method":"exit"} +```