Added 'built-in' validation category, made build phases more consistent

packages/langium/src/references/linker.ts

@@ -19,14 +19,16 @@ import { DocumentState } from '../workspace/documents';
  * Language-specific service for resolving cross-references in the AST.
  */
 export interface Linker {
+
     /**
      * Links all cross-references within the specified document. The default implementation loads only target
-     * elements from documents that are present in the `LangiumDocuments` service.
+     * elements from documents that are present in the `LangiumDocuments` service. The linked references are
+     * stored in the document's `references` property.
      *
      * @param document A LangiumDocument that shall be linked.
      * @param cancelToken A token for cancelling the operation.
      */
-    link(document: LangiumDocument, cancelToken?: CancellationToken): Promise<void>;
+    link(document: LangiumDocument, cancelToken?: CancellationToken): Promise<Reference[]>;
 
     /**
      * Unlinks all references within the specified document and removes them from the list of `references`.
@@ -62,6 +64,7 @@ export interface Linker {
      * @returns the desired Reference node, whose behavior wrt. resolving the cross reference is implementation specific.
      */
     buildReference(node: AstNode, property: string, refNode: CstNode | undefined, refText: string): Reference;
+
 }
 
 interface DefaultReference extends Reference {
@@ -82,12 +85,12 @@ export class DefaultLinker implements Linker {
         this.astNodeLocator = services.workspace.AstNodeLocator;
     }
 
-    async link(document: LangiumDocument, cancelToken = CancellationToken.None): Promise<void> {
+    async link(document: LangiumDocument, cancelToken = CancellationToken.None): Promise<Reference[]> {
         for (const node of streamAst(document.parseResult.value)) {
             await interruptAndCheck(cancelToken);
             streamReferences(node).forEach(ref => this.doLink(ref, document));
         }
-        document.state = DocumentState.Linked;
+        return document.references;
     }
 
     protected doLink(refInfo: ReferenceInfo, document: LangiumDocument): void {

packages/langium/src/references/scope-computation.ts

@@ -40,6 +40,7 @@ export interface ScopeComputation {
      * resolving references to symbols in the same document. The result is a multimap assigning a
      * set of AST node descriptions to every level of the AST. These data are used by the `ScopeProvider`
      * service to determine which target nodes are visible in the context of a specific cross-reference.
+     * The multimap is stored in the document's `precomputedScopes` property.
      *
      * _Note:_ You should not resolve any cross-references in this service method. Cross-reference
      * resolution depends on the scope computation phase to be completed.
@@ -119,6 +120,7 @@ export class DefaultScopeComputation implements ScopeComputation {
             await interruptAndCheck(cancelToken);
             this.processNode(node, document, scopes);
         }
+        document.precomputedScopes = scopes;
         return scopes;
     }
 

packages/langium/src/validation/document-validator.ts

@@ -20,10 +20,10 @@ import { interruptAndCheck, isOperationCancelled } from '../utils/promise-util';
 
 export interface ValidationOptions {
     /**
-     * If this is set, only the checks associated with this category are executed; otherwise
-     * all checks are executed. The default category if not specified is `'fast'`.
+     * If this is set, only the checks associated with these categories are executed; otherwise
+     * all checks are executed. The default category if not specified to the registry is `'fast'`.
      */
-    category?: ValidationCategory
+    categories?: ValidationCategory[];
     /** If true, no further diagnostics are reported if there are lexing errors. */
     stopAfterLexingErrors?: boolean
     /** If true, no further diagnostics are reported if there are parsing errors. */
@@ -63,19 +63,21 @@ export class DefaultDocumentValidator implements DocumentValidator {
 
         await interruptAndCheck(cancelToken);
 
-        this.processLexingErrors(parseResult, diagnostics, options);
-        if (options.stopAfterLexingErrors && diagnostics.some(d => d.code === DocumentValidator.LexingError)) {
-            return diagnostics;
-        }
+        if (!options.categories || options.categories.includes('built-in')) {
+            this.processLexingErrors(parseResult, diagnostics, options);
+            if (options.stopAfterLexingErrors && diagnostics.some(d => d.code === DocumentValidator.LexingError)) {
+                return diagnostics;
+            }
 
-        this.processParsingErrors(parseResult, diagnostics, options);
-        if (options.stopAfterParsingErrors && diagnostics.some(d => d.code === DocumentValidator.ParsingError)) {
-            return diagnostics;
-        }
+            this.processParsingErrors(parseResult, diagnostics, options);
+            if (options.stopAfterParsingErrors && diagnostics.some(d => d.code === DocumentValidator.ParsingError)) {
+                return diagnostics;
+            }
 
-        this.processLinkingErrors(document, diagnostics, options);
-        if (options.stopAfterLinkingErrors && diagnostics.some(d => d.code === DocumentValidator.LinkingError)) {
-            return diagnostics;
+            this.processLinkingErrors(document, diagnostics, options);
+            if (options.stopAfterLinkingErrors && diagnostics.some(d => d.code === DocumentValidator.LinkingError)) {
+                return diagnostics;
+            }
         }
 
         // Process custom validations
@@ -180,7 +182,7 @@ export class DefaultDocumentValidator implements DocumentValidator {
 
         await Promise.all(streamAst(rootNode).map(async node => {
             await interruptAndCheck(cancelToken);
-            const checks = this.validationRegistry.getChecks(node.$type, options.category);
+            const checks = this.validationRegistry.getChecks(node.$type, options.categories);
             for (const check of checks) {
                 await check(node, acceptor, cancelToken);
             }

packages/langium/src/validation/validation-registry.ts

@@ -65,8 +65,15 @@ export type ValidationChecks<T> = {
  * it as `slow`, it will be skipped for normal as-you-type validation. Then it's up to you when to
  * schedule these long-running checks: after the fast checks are done, or after saving a document,
  * or with an explicit command, etc.
+ *
+ * `built-in` checks are errors produced by the lexer, the parser, or the linker. They cannot be used
+ * for custom validation checks.
  */
-export type ValidationCategory = 'fast' | 'slow'
+export type ValidationCategory = 'fast' | 'slow' | 'built-in'
+
+export namespace ValidationCategory {
+    export const all: ValidationCategory[] = ['fast', 'slow', 'built-in'];
+}
 
 type ValidationCheckEntry = {
     check: ValidationCheck
@@ -93,6 +100,9 @@ export class ValidationRegistry {
      * @param thisObj Optional object to be used as `this` when calling the validation check functions.
      */
     register<T>(checksRecord: ValidationChecks<T>, thisObj: ThisParameterType<unknown> = this, category: ValidationCategory = 'fast'): void {
+        if (category === 'built-in') {
+            throw new Error("The 'built-in' category is reserved for lexer, parser, and linker errors.");
+        }
         for (const [type, ch] of Object.entries(checksRecord)) {
             const callbacks = ch as ValidationCheck | ValidationCheck[];
             if (Array.isArray(callbacks)) {
@@ -141,11 +151,11 @@ export class ValidationRegistry {
         }
     }
 
-    getChecks(type: string, category?: ValidationCategory): Stream<ValidationCheck> {
+    getChecks(type: string, categories?: ValidationCategory[]): Stream<ValidationCheck> {
         let checks = stream(this.entries.get(type))
             .concat(this.entries.get('AstNode'));
-        if (category) {
-            checks = checks.filter(entry => entry.category === category);
+        if (categories) {
+            checks = checks.filter(entry => categories.includes(entry.category));
         }
         return checks.map(entry => entry.check);
     }

packages/langium/src/workspace/document-builder.ts

@@ -16,14 +16,15 @@ import { CancellationToken, Disposable } from 'vscode-languageserver';
 import { MultiMap } from '../utils/collections';
 import { interruptAndCheck } from '../utils/promise-util';
 import { stream } from '../utils/stream';
+import { ValidationCategory } from '../validation/validation-registry';
 import { DocumentState } from './documents';
 
 export interface BuildOptions {
     /**
      * Control the validation phase with this option:
-     *  - `true` enables all validation checks
-     *  - An object with the `category` property enables a subset of validation checks
+     *  - `true` enables all validation checks and forces revalidating the documents
      *  - `false` or `undefined` disables all validation checks
+     *  - An object runs only the necessary validation checks; the `categories` property restricts this to a specific subset
      */
     validation?: boolean | ValidationOptions
 }
@@ -81,9 +82,9 @@ export type DocumentBuildListener = (built: LangiumDocument[], cancelToken: Canc
 export class DefaultDocumentBuilder implements DocumentBuilder {
 
     updateBuildOptions: BuildOptions = {
-        // Default: run only the validation checks in the _fast_ category (includes those without category)
+        // Default: run only the built-in validation checks and those in the _fast_ category (includes those without category)
         validation: {
-            category: 'fast'
+            categories: ['built-in', 'fast']
         }
     };
 
@@ -104,7 +105,38 @@ export class DefaultDocumentBuilder implements DocumentBuilder {
 
     async build<T extends AstNode>(documents: Array<LangiumDocument<T>>, options: BuildOptions = {}, cancelToken = CancellationToken.None): Promise<void> {
         for (const document of documents) {
-            this.buildState.delete(document.uri.toString());
+            const key = document.uri.toString();
+            if (document.state === DocumentState.Validated) {
+                if (typeof options.validation === 'boolean' && options.validation) {
+                    // Force re-running all validation checks
+                    document.state = DocumentState.IndexedReferences;
+                    document.diagnostics = undefined;
+                    this.buildState.delete(key);
+                } else if (typeof options.validation === 'object') {
+                    const state = this.buildState.get(key);
+                    const previousValidationOptions = state?.options?.validation;
+                    if (typeof previousValidationOptions === 'object' && previousValidationOptions.categories) {
+                        // Validation with explicit options was requested for a document that has already been partly validated.
+                        // In this case, we need to merge the previous validation categories with the new ones.
+                        const newCategories = options.validation.categories ?? ValidationCategory.all;
+                        const previousCategories = previousValidationOptions.categories;
+                        const categories = newCategories.filter(c => !previousCategories.includes(c));
+                        this.buildState.set(key, {
+                            completed: false,
+                            options: {
+                                validation: {
+                                    ...options.validation,
+                                    categories
+                                }
+                            }
+                        });
+                        document.state = DocumentState.IndexedReferences;
+                    } // In the other case, all validation checks are already done and we can skip this document.
+                }
+            } else {
+                // Default: forget any previous build options
+                this.buildState.delete(key);
+            }
         }
         await this.buildDocuments(documents, options, cancelToken);
     }
@@ -132,6 +164,7 @@ export class DefaultDocumentBuilder implements DocumentBuilder {
                 const linker = this.serviceRegistry.getServices(doc.uri).references.Linker;
                 linker.unlink(doc);
                 doc.state = Math.min(doc.state, DocumentState.ComputedScopes);
+                doc.diagnostics = undefined;
             });
         // Notify listeners of the update
         for (const listener of this.updateListeners) {
@@ -179,20 +212,7 @@ export class DefaultDocumentBuilder implements DocumentBuilder {
      * that a certain build phase is already done, the phase is skipped for that document.
      */
     protected async buildDocuments(documents: LangiumDocument[], options: BuildOptions, cancelToken: CancellationToken): Promise<void> {
-        for (const doc of documents) {
-            const key = doc.uri.toString();
-            const state = this.buildState.get(key);
-            // If the document has no previous build state, we set it. If it has one, but it's already marked
-            // as completed, we overwrite it. If the previous build was not completed, we keep its state
-            // and continue where it was cancelled.
-            if (!state || state.completed) {
-                this.buildState.set(key, {
-                    completed: false,
-                    options
-                });
-            }
-        }
-
+        this.prepareBuild(documents, options);
         // 0. Parse content
         await this.runCancelable(documents, DocumentState.Parsed, cancelToken, doc =>
             this.langiumDocumentFactory.update(doc)
@@ -203,11 +223,13 @@ export class DefaultDocumentBuilder implements DocumentBuilder {
         );
         // 2. Compute scopes
         await this.runCancelable(documents, DocumentState.ComputedScopes, cancelToken, doc =>
-            this.computeScopes(doc, cancelToken)
+            this.serviceRegistry.getServices(doc.uri)
+                .references.ScopeComputation.computeLocalScopes(doc, cancelToken)
         );
         // 3. Linking
         await this.runCancelable(documents, DocumentState.Linked, cancelToken, doc =>
-            this.serviceRegistry.getServices(doc.uri).references.Linker.link(doc, cancelToken)
+            this.serviceRegistry.getServices(doc.uri)
+                .references.Linker.link(doc, cancelToken)
         );
         // 4. Index references
         await this.runCancelable(documents, DocumentState.IndexedReferences, cancelToken, doc =>
@@ -228,11 +250,28 @@ export class DefaultDocumentBuilder implements DocumentBuilder {
         }
     }
 
+    protected prepareBuild(documents: LangiumDocument[], options: BuildOptions): void {
+        for (const doc of documents) {
+            const key = doc.uri.toString();
+            const state = this.buildState.get(key);
+            // If the document has no previous build state, we set it. If it has one, but it's already marked
+            // as completed, we overwrite it. If the previous build was not completed, we keep its state
+            // and continue where it was cancelled.
+            if (!state || state.completed) {
+                this.buildState.set(key, {
+                    completed: false,
+                    options
+                });
+            }
+        }
+    }
+
     protected async runCancelable(documents: LangiumDocument[], targetState: DocumentState, cancelToken: CancellationToken, callback: (document: LangiumDocument) => MaybePromise<unknown>): Promise<void> {
         const filtered = documents.filter(e => e.state < targetState);
         for (const document of filtered) {
             await interruptAndCheck(cancelToken);
             await callback(document);
+            document.state = targetState;
         }
         await this.notifyBuildPhase(filtered, targetState, cancelToken);
     }
@@ -256,20 +295,6 @@ export class DefaultDocumentBuilder implements DocumentBuilder {
         }
     }
 
-    /**
-     * Precompute the local scopes of the given document. The resulting data structure is used by
-     * the `ScopeProvider` service to determine the visible scope of any cross-reference.
-     *
-     * _Note:_ You should not resolve any cross-references during this phase. Once the phase is completed,
-     * you may follow the `ref` property of a reference, which triggers lazy resolution. The result is
-     * either the respective target AST node or `undefined` in case the target is not in scope.
-     */
-    protected async computeScopes(document: LangiumDocument, cancelToken: CancellationToken): Promise<void> {
-        const scopeComputation = this.serviceRegistry.getServices(document.uri).references.ScopeComputation;
-        document.precomputedScopes = await scopeComputation.computeLocalScopes(document, cancelToken);
-        document.state = DocumentState.ComputedScopes;
-    }
-
     /**
      * Determine whether the given document should be validated during a build. The default
      * implementation checks the `validation` property of the build options. If it's set to `true`
@@ -281,14 +306,18 @@ export class DefaultDocumentBuilder implements DocumentBuilder {
 
     /**
      * Run validation checks on the given document and store the resulting diagnostics in the document.
+     * If the document already contains diagnostics, the new ones are added to the list.
      */
     protected async validate(document: LangiumDocument, cancelToken: CancellationToken): Promise<void> {
         const validator = this.serviceRegistry.getServices(document.uri).validation.DocumentValidator;
         const validationSetting = this.getBuildOptions(document).validation;
         const options = typeof validationSetting === 'object' ? validationSetting : undefined;
         const diagnostics = await validator.validateDocument(document, options, cancelToken);
-        document.diagnostics = diagnostics;
-        document.state = DocumentState.Validated;
+        if (document.diagnostics) {
+            document.diagnostics.push(...diagnostics);
+        } else {
+            document.diagnostics = diagnostics;
+        }
     }
 
     protected getBuildOptions(document: LangiumDocument): BuildOptions {

packages/langium/src/workspace/documents.ts

@@ -242,7 +242,6 @@ export class DefaultLangiumDocumentFactory implements LangiumDocumentFactory {
 
         document.parseResult = this.parse(document.uri, text);
         (document.parseResult.value as Mutable<AstNode>).$document = document;
-        document.state = DocumentState.Parsed;
         return document;
     }
 
@@ -357,7 +356,7 @@ export class DefaultLangiumDocuments implements LangiumDocuments {
             langiumDoc.state = DocumentState.Changed;
             langiumDoc.precomputedScopes = undefined;
             langiumDoc.references = [];
-            langiumDoc.diagnostics = [];
+            langiumDoc.diagnostics = undefined;
         }
         return langiumDoc;
     }

packages/langium/src/workspace/index-manager.ts

@@ -15,7 +15,6 @@ import { CancellationToken } from 'vscode-languageserver';
 import { getDocument } from '../utils/ast-util';
 import { stream } from '../utils/stream';
 import { equalURI } from '../utils/uri-util';
-import { DocumentState } from './documents';
 
 /**
  * The index manager is responsible for keeping metadata about symbols and cross-references
@@ -139,14 +138,12 @@ export class DefaultIndexManager implements IndexManager {
             data.node = undefined; // clear reference to the AST Node
         }
         this.simpleIndex.set(document.uri.toString(), exports);
-        document.state = DocumentState.IndexedContent;
     }
 
     async updateReferences(document: LangiumDocument, cancelToken = CancellationToken.None): Promise<void> {
         const services = this.serviceRegistry.getServices(document.uri);
         const indexData: ReferenceDescription[] = await services.workspace.ReferenceDescriptionProvider.createDescriptions(document, cancelToken);
         this.referenceIndex.set(document.uri.toString(), indexData);
-        document.state = DocumentState.IndexedReferences;
     }
 
     isAffected(document: LangiumDocument, changedUris: Set<string>): boolean {