From 8727445656db4afe60d571bd80baeaf464ce6334 Mon Sep 17 00:00:00 2001 From: Jon Ross-Perkins Date: Mon, 27 Jan 2025 08:15:04 -0800 Subject: [PATCH] Add a framework for LSP testing. (#4841) Also adds tests for exit and initialize, just basic things. --- testing/file_test/README.md | 6 + testing/file_test/file_test_base.cpp | 108 +++++++++++++++--- testing/file_test/file_test_base.h | 6 + testing/file_test/file_test_base_test.cpp | 28 ++--- testing/file_test/testdata/lsp_calls.carbon | 48 ++++++++ testing/file_test/testdata/stdin.carbon | 1 + toolchain/language_server/BUILD | 5 + toolchain/language_server/language_server.cpp | 8 +- .../language_server/testdata/exit.carbon | 16 +++ .../testdata/fail_empty_stdin.carbon | 16 +++ .../testdata/initialize.carbon | 28 +++++ toolchain/testing/BUILD | 1 + toolchain/testing/file_test.cpp | 20 +++- 13 files changed, 255 insertions(+), 36 deletions(-) create mode 100644 testing/file_test/testdata/lsp_calls.carbon create mode 100644 toolchain/language_server/testdata/exit.carbon create mode 100644 toolchain/language_server/testdata/fail_empty_stdin.carbon create mode 100644 toolchain/language_server/testdata/initialize.carbon diff --git a/testing/file_test/README.md b/testing/file_test/README.md index ba44db73605d0..f1231d21ed42b 100644 --- a/testing/file_test/README.md +++ b/testing/file_test/README.md @@ -86,6 +86,12 @@ they have an associated error. An exception is that the main test file may omit Some keywords can be inserted for content: +- ``` + [[@LSP::]] + ``` + + Produces JSON for an LSP method call, complete with `Content-Length` header. + - ``` [[@TEST_NAME]] ``` diff --git a/testing/file_test/file_test_base.cpp b/testing/file_test/file_test_base.cpp index fd5b7a23e09fc..fe589227c2efd 100644 --- a/testing/file_test/file_test_base.cpp +++ b/testing/file_test/file_test_base.cpp @@ -344,13 +344,16 @@ auto FileTestBase::ProcessTestFileAndRun(TestContext& context) llvm::PrettyStackTraceProgram stack_trace_entry( test_argv_for_stack_trace.size() - 1, test_argv_for_stack_trace.data()); + // Execution must be serialized for either serial tests or console output. + std::unique_lock output_lock; + if (output_mutex_ && + (context.capture_console_output || !AllowParallelRun())) { + output_lock = std::unique_lock(*output_mutex_); + } + // Conditionally capture console output. We use a scope exit to ensure the // captures terminate even on run failures. - std::unique_lock output_lock; if (context.capture_console_output) { - if (output_mutex_) { - output_lock = std::unique_lock(*output_mutex_); - } CaptureStderr(); CaptureStdout(); } @@ -514,6 +517,84 @@ struct SplitState { int file_index = 0; }; +// Replaces the keyword at the given position. Returns the position to start a +// find for the next keyword. +static auto ReplaceContentKeywordAt(std::string* content, size_t keyword_pos, + llvm::StringRef test_name, int* lsp_id) + -> ErrorOr { + auto keyword = llvm::StringRef(*content).substr(keyword_pos); + + // Line replacements aren't handled here. + static constexpr llvm::StringLiteral Line = "[[@LINE"; + if (keyword.starts_with(Line)) { + // Just move past the prefix to find the next one. + return keyword_pos + Line.size(); + } + + // Replaced with the actual test name. + static constexpr llvm::StringLiteral TestName = "[[@TEST_NAME]]"; + if (keyword.starts_with(TestName)) { + content->replace(keyword_pos, TestName.size(), test_name); + return keyword_pos + test_name.size(); + } + + // Reformatted as an LSP call with headers. + static constexpr llvm::StringLiteral Lsp = "[[@LSP:"; + if (keyword.starts_with(Lsp)) { + auto method_start = keyword_pos + Lsp.size(); + + static constexpr llvm::StringLiteral LspEnd = "]]"; + auto keyword_end = content->find("]]", method_start); + if (keyword_end == std::string::npos) { + return ErrorBuilder() + << "Missing `" << LspEnd << "` after `" << Lsp << "`"; + } + + auto method_end = content->find(":", method_start); + auto extra_content_start = method_end + 1; + if (method_end == std::string::npos || method_end > keyword_end) { + method_end = keyword_end; + extra_content_start = keyword_end; + } + auto method = content->substr(method_start, method_end - method_start); + + auto extra_content = + content->substr(extra_content_start, keyword_end - extra_content_start); + std::string extra_content_sep; + if (!extra_content.empty()) { + extra_content_sep = ","; + if (!extra_content.starts_with("\n")) { + extra_content_sep += " "; + } + } + + // Form the JSON. + std::string json; + if (method == "exit") { + if (!extra_content.empty()) { + return Error("`[[@LSP:exit:` cannot include extra content"); + } + json = R"({"jsonrpc": "2.0", "method": "exit"})"; + } else { + json = llvm::formatv( + R"({{"jsonrpc": "2.0", "id": "{0}", "method": "{1}"{2}{3}})", + ++(*lsp_id), method, extra_content_sep, extra_content) + .str(); + } + // Add the Content-Length header. The `2` accounts for extra newlines. + auto json_with_header = + llvm::formatv("Content-Length: {0}\n\n{1}\n", json.size() + 2, json) + .str(); + // Insert the content. + content->replace(keyword_pos, keyword_end + 2 - keyword_pos, + json_with_header); + return keyword_pos + json_with_header.size(); + } + + return ErrorBuilder() << "Unexpected use of `[[@` at `" + << keyword.substr(0, 5) << "`"; +} + // Replaces the content keywords. // // TEST_NAME is the only content keyword at present, but we do validate that @@ -543,20 +624,13 @@ static auto ReplaceContentKeywords(llvm::StringRef filename, test_name.consume_front("fail_"); test_name.consume_front("todo_"); + // A counter for LSP calls. + int lsp_id = 0; while (keyword_pos != std::string::npos) { - static constexpr llvm::StringLiteral TestName = "[[@TEST_NAME]]"; - auto keyword = llvm::StringRef(*content).substr(keyword_pos); - if (keyword.starts_with(TestName)) { - content->replace(keyword_pos, TestName.size(), test_name); - keyword_pos += test_name.size(); - } else if (keyword.starts_with("[[@LINE")) { - // Just move past the prefix to find the next one. - keyword_pos += Prefix.size(); - } else { - return ErrorBuilder() - << "Unexpected use of `[[@` at `" << keyword.substr(0, 5) << "`"; - } - keyword_pos = content->find(Prefix, keyword_pos); + CARBON_ASSIGN_OR_RETURN( + auto keyword_end, + ReplaceContentKeywordAt(content, keyword_pos, test_name, &lsp_id)); + keyword_pos = content->find(Prefix, keyword_end); } return Success(); } diff --git a/testing/file_test/file_test_base.h b/testing/file_test/file_test_base.h index ca27369b98c0f..e54e1e1ea855a 100644 --- a/testing/file_test/file_test_base.h +++ b/testing/file_test/file_test_base.h @@ -116,6 +116,12 @@ class FileTestBase : public testing::Test { // Optionally allows children to provide extra replacements for autoupdate. virtual auto DoExtraCheckReplacements(std::string& /*check_line*/) -> void {} + // Whether to allow running the test in parallel, particularly for autoupdate. + // This can be overridden to force some tests to be run serially. At any given + // time, all parallel tests and a single non-parallel test will be allowed to + // run. + virtual auto AllowParallelRun() const -> bool { return true; } + // Runs a test and compares output. This keeps output split by line so that // issues are a little easier to identify by the different line. auto TestBody() -> void final; diff --git a/testing/file_test/file_test_base_test.cpp b/testing/file_test/file_test_base_test.cpp index 4a2c49fe21115..262e3644be206 100644 --- a/testing/file_test/file_test_base_test.cpp +++ b/testing/file_test/file_test_base_test.cpp @@ -198,21 +198,6 @@ static auto TestEscaping(TestParams& params) return {{.success = true}}; } -// Prints and returns expected results for stdin.carbon. -static auto TestStdin(TestParams& params) - -> ErrorOr { - CARBON_CHECK(params.input_stream); - constexpr int ReadSize = 256; - char buf[ReadSize]; - while (feof(params.input_stream) == 0) { - auto read = fread(&buf, sizeof(char), ReadSize, params.input_stream); - if (read > 0) { - params.error_stream.write(buf, read); - } - } - return {{.success = true}}; -} - // Prints and returns expected results for unattached_multi_file.carbon. static auto TestUnattachedMultiFile(TestParams& params) -> ErrorOr { @@ -258,6 +243,17 @@ static auto EchoFileContent(TestParams& params) buffer = remainder; } } + if (params.input_stream) { + params.error_stream << "--- STDIN:\n"; + constexpr int ReadSize = 1024; + char buf[ReadSize]; + while (feof(params.input_stream) == 0) { + auto read = fread(&buf, sizeof(char), ReadSize, params.input_stream); + if (read > 0) { + params.error_stream.write(buf, read); + } + } + } return {{.success = true}}; } @@ -287,8 +283,6 @@ auto FileTestBaseTest::Run( .Case("file_only_re_one_file.carbon", &TestFileOnlyREOneFile) .Case("file_only_re_multi_file.carbon", &TestFileOnlyREMultiFile) .Case("no_line_number.carbon", &TestNoLineNumber) - .Case("stdin.carbon", &TestStdin) - .Case("stdin_and_autoupdate_split.carbon", &TestStdin) .Case("unattached_multi_file.carbon", &TestUnattachedMultiFile) .Case("fail_multi_success_overall_fail.carbon", [&](TestParams&) { diff --git a/testing/file_test/testdata/lsp_calls.carbon b/testing/file_test/testdata/lsp_calls.carbon new file mode 100644 index 0000000000000..d3bb7b80e3bf4 --- /dev/null +++ b/testing/file_test/testdata/lsp_calls.carbon @@ -0,0 +1,48 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception + +// AUTOUPDATE +// TIP: To test this file alone, run: +// TIP: bazel test //testing/file_test:file_test_base_test --test_arg=--file_tests=testing/file_test/testdata/lsp_calls.carbon +// TIP: To dump output, run: +// TIP: bazel run //testing/file_test:file_test_base_test -- --dump_output --file_tests=testing/file_test/testdata/lsp_calls.carbon + +// --- STDIN +[[@LSP:foo:]] +[[@LSP:foo]] +[[@LSP:bar:content]] +[[@LSP:baz: +multi +line +]] +[[@LSP:exit]] + +// --- AUTOUPDATE-SPLIT + +// CHECK:STDERR: --- STDIN: +// CHECK:STDERR: Content-Length: 48 +// CHECK:STDERR: +// CHECK:STDERR: {"jsonrpc": "2.0", "id": "1", "method": "foo"} +// CHECK:STDERR: +// CHECK:STDERR: Content-Length: 48 +// CHECK:STDERR: +// CHECK:STDERR: {"jsonrpc": "2.0", "id": "2", "method": "foo"} +// CHECK:STDERR: +// CHECK:STDERR: Content-Length: 57 +// CHECK:STDERR: +// CHECK:STDERR: {"jsonrpc": "2.0", "id": "3", "method": "bar", content} +// CHECK:STDERR: +// CHECK:STDERR: Content-Length: 61 +// CHECK:STDERR: +// CHECK:STDERR: {"jsonrpc": "2.0", "id": "4", "method": "baz", +// CHECK:STDERR: multi +// CHECK:STDERR: line +// CHECK:STDERR: } +// CHECK:STDERR: +// CHECK:STDERR: Content-Length: 38 +// CHECK:STDERR: +// CHECK:STDERR: {"jsonrpc": "2.0", "method": "exit"} +// CHECK:STDERR: +// CHECK:STDERR: +// CHECK:STDOUT: 1 args: `default_args` diff --git a/testing/file_test/testdata/stdin.carbon b/testing/file_test/testdata/stdin.carbon index a2a8cc0ef1869..0a23acdf195a2 100644 --- a/testing/file_test/testdata/stdin.carbon +++ b/testing/file_test/testdata/stdin.carbon @@ -20,6 +20,7 @@ n // --- AUTOUPDATE-SPLIT +// CHECK:STDERR: --- STDIN: // CHECK:STDERR: // CHECK:STDERR: S // CHECK:STDERR: t diff --git a/toolchain/language_server/BUILD b/toolchain/language_server/BUILD index ecde8af8ff572..24684c1014b0a 100644 --- a/toolchain/language_server/BUILD +++ b/toolchain/language_server/BUILD @@ -6,6 +6,11 @@ load("@rules_cc//cc:defs.bzl", "cc_library") package(default_visibility = ["//visibility:public"]) +filegroup( + name = "testdata", + data = glob(["testdata/**/*.carbon"]), +) + cc_library( name = "language_server", srcs = ["language_server.cpp"], diff --git a/toolchain/language_server/language_server.cpp b/toolchain/language_server/language_server.cpp index ad51e7e7417bc..aff99f614f415 100644 --- a/toolchain/language_server/language_server.cpp +++ b/toolchain/language_server/language_server.cpp @@ -6,6 +6,7 @@ #include "clang-tools-extra/clangd/LSPBinder.h" #include "clang-tools-extra/clangd/Transport.h" +#include "clang-tools-extra/clangd/support/Logger.h" #include "common/raw_string_ostream.h" #include "toolchain/language_server/context.h" #include "toolchain/language_server/incoming_messages.h" @@ -14,7 +15,12 @@ namespace Carbon::LanguageServer { auto Run(FILE* input_stream, llvm::raw_ostream& output_stream, - llvm::raw_ostream& /*error_stream*/) -> ErrorOr { + llvm::raw_ostream& error_stream) -> ErrorOr { + // TODO: Consider implementing a custom logger that splits vlog to + // vlog_stream when provided. For now, this disables verbose logging. + clang::clangd::StreamLogger logger(error_stream, clang::clangd::Logger::Info); + clang::clangd::LoggingSession logging_session(logger); + // Set up the connection. std::unique_ptr transport( clang::clangd::newJSONTransport(input_stream, output_stream, diff --git a/toolchain/language_server/testdata/exit.carbon b/toolchain/language_server/testdata/exit.carbon new file mode 100644 index 0000000000000..c35774965b28f --- /dev/null +++ b/toolchain/language_server/testdata/exit.carbon @@ -0,0 +1,16 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// AUTOUPDATE +// TIP: To test this file alone, run: +// TIP: bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/exit.carbon +// TIP: To dump output, run: +// TIP: bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/exit.carbon + +// --- STDIN +[[@LSP:exit]] + +// --- AUTOUPDATE-SPLIT + +// CHECK:STDOUT: diff --git a/toolchain/language_server/testdata/fail_empty_stdin.carbon b/toolchain/language_server/testdata/fail_empty_stdin.carbon new file mode 100644 index 0000000000000..fed9f4a557ac1 --- /dev/null +++ b/toolchain/language_server/testdata/fail_empty_stdin.carbon @@ -0,0 +1,16 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// AUTOUPDATE +// TIP: To test this file alone, run: +// TIP: bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/fail_empty_stdin.carbon +// TIP: To dump output, run: +// TIP: bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/fail_empty_stdin.carbon + +// --- STDIN +// --- AUTOUPDATE-SPLIT + +// CHECK:STDERR: error: Input/output error +// CHECK:STDERR: +// CHECK:STDOUT: diff --git a/toolchain/language_server/testdata/initialize.carbon b/toolchain/language_server/testdata/initialize.carbon new file mode 100644 index 0000000000000..7c4afb3449957 --- /dev/null +++ b/toolchain/language_server/testdata/initialize.carbon @@ -0,0 +1,28 @@ +// Part of the Carbon Language project, under the Apache License v2.0 with LLVM +// Exceptions. See /LICENSE for license information. +// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +// +// AUTOUPDATE +// TIP: To test this file alone, run: +// TIP: bazel test //toolchain/testing:file_test --test_arg=--file_tests=toolchain/language_server/testdata/initialize.carbon +// TIP: To dump output, run: +// TIP: bazel run //toolchain/testing:file_test -- --dump_output --file_tests=toolchain/language_server/testdata/initialize.carbon + +// --- STDIN +[[@LSP:initialize]] +[[@LSP:exit]] + +// --- AUTOUPDATE-SPLIT + +// CHECK:STDOUT: Content-Length: 148{{\r}} +// CHECK:STDOUT: {{\r}} +// CHECK:STDOUT: { +// CHECK:STDOUT: "id": "1", +// CHECK:STDOUT: "jsonrpc": "2.0", +// CHECK:STDOUT: "result": { +// CHECK:STDOUT: "capabilities": { +// CHECK:STDOUT: "documentSymbolProvider": true, +// CHECK:STDOUT: "textDocumentSync": 1 +// CHECK:STDOUT: } +// CHECK:STDOUT: } +// CHECK:STDOUT: } diff --git a/toolchain/testing/BUILD b/toolchain/testing/BUILD index 6c5b3d7214d93..c3ab3e713e0f8 100644 --- a/toolchain/testing/BUILD +++ b/toolchain/testing/BUILD @@ -16,6 +16,7 @@ filegroup( "//toolchain/diagnostics:testdata", "//toolchain/driver:testdata", "//toolchain/format:testdata", + "//toolchain/language_server:testdata", "//toolchain/lex:testdata", "//toolchain/lower:testdata", "//toolchain/parse:testdata", diff --git a/toolchain/testing/file_test.cpp b/toolchain/testing/file_test.cpp index 89e702692422b..6d1ec274fbc83 100644 --- a/toolchain/testing/file_test.cpp +++ b/toolchain/testing/file_test.cpp @@ -20,7 +20,7 @@ namespace Carbon::Testing { namespace { // Provides common test support for the driver. This is used by file tests in -// phase subdirectories. +// component subdirectories. class ToolchainFileTest : public FileTestBase { public: explicit ToolchainFileTest(llvm::StringRef exe_path, std::mutex* output_mutex, @@ -51,6 +51,12 @@ class ToolchainFileTest : public FileTestBase { // driver. auto DoExtraCheckReplacements(std::string& check_line) -> void override; + // Most tests can be run in parallel, but clangd has a global for its logging + // system so we need language-server tests to be run in serial. + auto AllowParallelRun() const -> bool override { + return component_ != "language_server"; + } + private: // Adds a file to the fs. auto AddFile(llvm::vfs::InMemoryFileSystem& fs, llvm::StringRef path) @@ -98,6 +104,10 @@ auto ToolchainFileTest::Run( llvm::IntrusiveRefCntPtr& fs, FILE* input_stream, llvm::raw_pwrite_stream& output_stream, llvm::raw_pwrite_stream& error_stream) -> ErrorOr { + if (component_ == "language_server" && !input_stream) { + return Error("language_server tests must provide STDIN"); + } + CARBON_ASSIGN_OR_RETURN(auto prelude, installation_.ReadPreludeManifest()); if (!is_no_prelude()) { for (const auto& file : prelude) { @@ -130,12 +140,20 @@ auto ToolchainFileTest::Run( entry.first.starts_with("not_file") || llvm::is_contained(prelude, entry.first); }); + + if (component_ == "language_server") { + // The language server doesn't always add a suffix newline, so add one for + // tests to be happy. + output_stream << "\n"; + } return result; } auto ToolchainFileTest::GetDefaultArgs() -> llvm::SmallVector { if (component_ == "format") { return {"format", "%s"}; + } else if (component_ == "language_server") { + return {"language-server"}; } llvm::SmallVector args = {"compile", "--include-diagnostic-kind",