Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Refactored document builder and affected analysis #1094

Merged
merged 7 commits into from
Jul 10, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion examples/arithmetics/src/cli/cli-util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export async function extractDocument<T extends AstNode>(fileName: string, exten
}

const document = services.shared.workspace.LangiumDocuments.getOrCreateDocument(URI.file(path.resolve(fileName)));
await services.shared.workspace.DocumentBuilder.build([document], { validationChecks: 'all' });
await services.shared.workspace.DocumentBuilder.build([document], { validation: true });

const validationErrors = (document.diagnostics ?? []).filter(e => e.severity === 1);
if (validationErrors.length > 0) {
Expand Down
2 changes: 1 addition & 1 deletion examples/domainmodel/src/cli/cli-util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ export async function extractDocument<T extends AstNode>(fileName: string, exten
}

const document = services.shared.workspace.LangiumDocuments.getOrCreateDocument(URI.file(path.resolve(fileName)));
await services.shared.workspace.DocumentBuilder.build([document], { validationChecks: 'all' });
await services.shared.workspace.DocumentBuilder.build([document], { validation: true });

const validationErrors = (document.diagnostics ?? []).filter(e => e.severity === 1);
if (validationErrors.length > 0) {
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ export class QualifiedNameProvider {
prefix = (isPackageDeclaration(prefix.$container)
? this.getQualifiedName(prefix.$container, prefix.name) : prefix.name);
}
return (prefix ? prefix + '.' : '') + name;
return prefix ? prefix + '.' + name : name;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ export class DomainModelScopeComputation extends DefaultScopeComputation {
const localDescriptions: AstNodeDescription[] = [];
for (const element of container.elements) {
await interruptAndCheck(cancelToken);
if (isType(element)) {
if (isType(element) && element.name) {
const description = this.descriptions.createDescription(element, element.name, document);
localDescriptions.push(description);
} else if (isPackageDeclaration(element)) {
Expand Down
2 changes: 1 addition & 1 deletion examples/requirements/src/cli/cli-util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,7 @@ export async function extractDocuments(fileName: string, services: LangiumServic
await services.shared.workspace.WorkspaceManager.initializeWorkspace(folders);

const documents = services.shared.workspace.LangiumDocuments.all.toArray();
await services.shared.workspace.DocumentBuilder.build(documents, { validationChecks: 'all' });
await services.shared.workspace.DocumentBuilder.build(documents, { validation: true });

documents.forEach(document => {
const validationErrors = (document.diagnostics ?? []).filter(e => e.severity === 1);
Expand Down
2 changes: 1 addition & 1 deletion examples/statemachine/src/cli/cli-util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -22,7 +22,7 @@ export async function extractDocument(fileName: string, extensions: string[], se
}

const document = services.shared.workspace.LangiumDocuments.getOrCreateDocument(URI.file(path.resolve(fileName)));
await services.shared.workspace.DocumentBuilder.build([document], { validationChecks: 'all' });
await services.shared.workspace.DocumentBuilder.build([document], { validation: true });

const validationErrors = (document.diagnostics ?? []).filter(e => e.severity === 1);
if (validationErrors.length > 0) {
Expand Down
4 changes: 2 additions & 2 deletions packages/langium-cli/src/generate.ts
Original file line number Diff line number Diff line change
Expand Up @@ -136,7 +136,7 @@ async function relinkGrammars(grammars: Grammar[]): Promise<void> {
return newDoc;
});
newDocuments.forEach(e => langiumDocuments.addDocument(e));
await documentBuilder.build(newDocuments, { validationChecks: 'none' });
await documentBuilder.build(newDocuments, { validation: false });
}

async function buildAll(config: LangiumConfig): Promise<Map<string, LangiumDocument>> {
Expand All @@ -155,7 +155,7 @@ async function buildAll(config: LangiumConfig): Promise<Map<string, LangiumDocum
map.set(doc.uri.fsPath, doc);
}
await sharedServices.workspace.DocumentBuilder.build(documents.all.toArray(), {
validationChecks: 'all'
validation: true
});
return map;
}
Expand Down
6 changes: 4 additions & 2 deletions packages/langium/src/references/linker.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,9 +19,11 @@ import { DocumentState } from '../workspace/documents';
* Language-specific service for resolving cross-references in the AST.
*/
export interface Linker {

/**
* Links all cross-references within the specified document. The default implementation loads only target
* elements from documents that are present in the `LangiumDocuments` service.
* elements from documents that are present in the `LangiumDocuments` service. The linked references are
* stored in the document's `references` property.
*
* @param document A LangiumDocument that shall be linked.
* @param cancelToken A token for cancelling the operation.
Expand Down Expand Up @@ -62,6 +64,7 @@ export interface Linker {
* @returns the desired Reference node, whose behavior wrt. resolving the cross reference is implementation specific.
*/
buildReference(node: AstNode, property: string, refNode: CstNode | undefined, refText: string): Reference;

}

interface DefaultReference extends Reference {
Expand All @@ -87,7 +90,6 @@ export class DefaultLinker implements Linker {
await interruptAndCheck(cancelToken);
streamReferences(node).forEach(ref => this.doLink(ref, document));
}
document.state = DocumentState.Linked;
}

protected doLink(refInfo: ReferenceInfo, document: LangiumDocument): void {
Expand Down
2 changes: 1 addition & 1 deletion packages/langium/src/test/langium-test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -449,7 +449,7 @@ export interface ValidationResult<T extends AstNode = AstNode> {
export function validationHelper<T extends AstNode = AstNode>(services: LangiumServices): (input: string) => Promise<ValidationResult<T>> {
const parse = parseHelper<T>(services);
return async (input) => {
const document = await parse(input, { validationChecks: 'all' });
const document = await parse(input, { validation: true });
return { document, diagnostics: document.diagnostics ?? [] };
};
}
Expand Down
2 changes: 1 addition & 1 deletion packages/langium/src/utils/grammar-util.ts
Original file line number Diff line number Diff line change
Expand Up @@ -312,7 +312,7 @@ export async function createServicesForGrammar(config: {
: getDocument(config.grammar);
const grammarNode = grammarDocument.parseResult.value as ast.Grammar;
const documentBuilder = grammarServices.shared.workspace.DocumentBuilder;
await documentBuilder.build([grammarDocument], { validationChecks: 'none' });
await documentBuilder.build([grammarDocument], { validation: false });

const parserConfig = config.parserConfig ?? {
skipValidations: false
Expand Down
82 changes: 60 additions & 22 deletions packages/langium/src/validation/document-validator.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,30 +7,48 @@
import type { MismatchedTokenException } from 'chevrotain';
import type { Diagnostic } from 'vscode-languageserver';
import type { LanguageMetaData } from '../grammar/language-meta-data';
import type { ParseResult } from '../parser/langium-parser';
import type { LangiumServices } from '../services';
import type { AstNode, CstNode } from '../syntax-tree';
import type { LangiumDocument } from '../workspace/documents';
import type { DiagnosticInfo, ValidationAcceptor, ValidationRegistry } from './validation-registry';
import type { DiagnosticInfo, ValidationAcceptor, ValidationCategory, ValidationRegistry } from './validation-registry';
import { CancellationToken, DiagnosticSeverity, Position, Range } from 'vscode-languageserver';
import { findNodeForKeyword, findNodeForProperty } from '../utils/grammar-util';
import { streamAst } from '../utils/ast-util';
import { tokenToRange } from '../utils/cst-util';
import { interruptAndCheck, isOperationCancelled } from '../utils/promise-util';

export interface ValidationOptions {
/**
* If this is set, only the checks associated with these categories are executed; otherwise
* all checks are executed. The default category if not specified to the registry is `'fast'`.
*/
categories?: ValidationCategory[];
/** If true, no further diagnostics are reported if there are lexing errors. */
stopAfterLexingErrors?: boolean
/** If true, no further diagnostics are reported if there are parsing errors. */
stopAfterParsingErrors?: boolean
/** If true, no further diagnostics are reported if there are linking errors. */
stopAfterLinkingErrors?: boolean
}

/**
* Language-specific service for validating `LangiumDocument`s.
*/
export interface DocumentValidator {
/**
* Validates the whole specified document.
*
* @param document specified document to validate
* @param options options to control the validation process
* @param cancelToken allows to cancel the current operation
* @throws `OperationCanceled` if a user action occurs during execution
*/
validateDocument(document: LangiumDocument, cancelToken?: CancellationToken): Promise<Diagnostic[]>;
validateDocument(document: LangiumDocument, options?: ValidationOptions, cancelToken?: CancellationToken): Promise<Diagnostic[]>;
}

export class DefaultDocumentValidator implements DocumentValidator {

protected readonly validationRegistry: ValidationRegistry;
protected readonly metadata: LanguageMetaData;

Expand All @@ -39,13 +57,45 @@ export class DefaultDocumentValidator implements DocumentValidator {
this.metadata = services.LanguageMetaData;
}

async validateDocument(document: LangiumDocument, cancelToken = CancellationToken.None): Promise<Diagnostic[]> {
async validateDocument(document: LangiumDocument, options: ValidationOptions = {}, cancelToken = CancellationToken.None): Promise<Diagnostic[]> {
const parseResult = document.parseResult;
const diagnostics: Diagnostic[] = [];

await interruptAndCheck(cancelToken);

// Process lexing errors
if (!options.categories || options.categories.includes('built-in')) {
this.processLexingErrors(parseResult, diagnostics, options);
if (options.stopAfterLexingErrors && diagnostics.some(d => d.code === DocumentValidator.LexingError)) {
return diagnostics;
}

this.processParsingErrors(parseResult, diagnostics, options);
if (options.stopAfterParsingErrors && diagnostics.some(d => d.code === DocumentValidator.ParsingError)) {
return diagnostics;
}

this.processLinkingErrors(document, diagnostics, options);
if (options.stopAfterLinkingErrors && diagnostics.some(d => d.code === DocumentValidator.LinkingError)) {
return diagnostics;
}
}

// Process custom validations
try {
diagnostics.push(...await this.validateAst(parseResult.value, options, cancelToken));
} catch (err) {
if (isOperationCancelled(err)) {
throw err;
}
console.error('An error occurred during validation:', err);
}

await interruptAndCheck(cancelToken);

return diagnostics;
}

protected processLexingErrors(parseResult: ParseResult, diagnostics: Diagnostic[], _options: ValidationOptions): void {
for (const lexerError of parseResult.lexerErrors) {
const diagnostic: Diagnostic = {
severity: DiagnosticSeverity.Error,
Expand All @@ -65,8 +115,9 @@ export class DefaultDocumentValidator implements DocumentValidator {
};
diagnostics.push(diagnostic);
}
}

// Process parsing errors
protected processParsingErrors(parseResult: ParseResult, diagnostics: Diagnostic[], _options: ValidationOptions): void {
for (const parserError of parseResult.parserErrors) {
let range: Range | undefined = undefined;
// We can run into the chevrotain error recovery here
Expand Down Expand Up @@ -100,8 +151,9 @@ export class DefaultDocumentValidator implements DocumentValidator {
diagnostics.push(diagnostic);
}
}
}

// Process unresolved references
protected processLinkingErrors(document: LangiumDocument, diagnostics: Diagnostic[], _options: ValidationOptions): void {
for (const reference of document.references) {
const linkingError = reference.error;
if (linkingError) {
Expand All @@ -120,31 +172,17 @@ export class DefaultDocumentValidator implements DocumentValidator {
diagnostics.push(this.toDiagnostic('error', linkingError.message, info));
}
}

// Process custom validations
try {
diagnostics.push(...await this.validateAst(parseResult.value, document, cancelToken));
} catch (err) {
if (isOperationCancelled(err)) {
throw err;
}
console.error('An error occurred during validation:', err);
}

await interruptAndCheck(cancelToken);

return diagnostics;
}

protected async validateAst(rootNode: AstNode, document: LangiumDocument, cancelToken = CancellationToken.None): Promise<Diagnostic[]> {
protected async validateAst(rootNode: AstNode, options: ValidationOptions, cancelToken = CancellationToken.None): Promise<Diagnostic[]> {
const validationItems: Diagnostic[] = [];
const acceptor: ValidationAcceptor = <N extends AstNode>(severity: 'error' | 'warning' | 'info' | 'hint', message: string, info: DiagnosticInfo<N>) => {
validationItems.push(this.toDiagnostic(severity, message, info));
};

await Promise.all(streamAst(rootNode).map(async node => {
await interruptAndCheck(cancelToken);
const checks = this.validationRegistry.getChecks(node.$type);
const checks = this.validationRegistry.getChecks(node.$type, options.categories);
for (const check of checks) {
await check(node, acceptor, cancelToken);
}
Expand Down
65 changes: 56 additions & 9 deletions packages/langium/src/validation/validation-registry.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,8 +8,10 @@ import type { CancellationToken, CodeDescription, DiagnosticRelatedInformation,
import type { LangiumServices } from '../services';
import type { AstNode, AstReflection, Properties } from '../syntax-tree';
import type { MaybePromise } from '../utils/promise-util';
import type { Stream } from '../utils/stream';
import { MultiMap } from '../utils/collections';
import { isOperationCancelled } from '../utils/promise-util';
import { stream } from '../utils/stream';

export type DiagnosticInfo<N extends AstNode, P = Properties<N>> = {
/** The AST node to which the diagnostic is attached. */
Expand Down Expand Up @@ -57,26 +59,66 @@ export type ValidationChecks<T> = {
AstNode?: ValidationCheck<AstNode>;
}

/**
* `fast` checks can be executed after every document change (i.e. as the user is typing). If a check
* is too slow it can delay the response to document changes, yielding bad user experience. By marking
* it as `slow`, it will be skipped for normal as-you-type validation. Then it's up to you when to
* schedule these long-running checks: after the fast checks are done, or after saving a document,
* or with an explicit command, etc.
*
* `built-in` checks are errors produced by the lexer, the parser, or the linker. They cannot be used
* for custom validation checks.
*/
export type ValidationCategory = 'fast' | 'slow' | 'built-in'

export namespace ValidationCategory {
export const all: readonly ValidationCategory[] = ['fast', 'slow', 'built-in'];
}

type ValidationCheckEntry = {
check: ValidationCheck
category: ValidationCategory
}

/**
* Manages a set of `ValidationCheck`s to be applied when documents are validated.
*/
export class ValidationRegistry {
private readonly validationChecks = new MultiMap<string, ValidationCheck>();
private readonly entries = new MultiMap<string, ValidationCheckEntry>();
private readonly reflection: AstReflection;

constructor(services: LangiumServices) {
this.reflection = services.shared.AstReflection;
}

register<T>(checksRecord: ValidationChecks<T>, thisObj: ThisParameterType<unknown> = this): void {
/**
* Register a set of validation checks. Each value in the record can be either a single validation check (i.e. a function)
* or an array of validation checks.
*
* @param checksRecord Set of validation checks to register.
* @param category Optional category for the validation checks (defaults to `'fast'`).
* @param thisObj Optional object to be used as `this` when calling the validation check functions.
*/
register<T>(checksRecord: ValidationChecks<T>, thisObj: ThisParameterType<unknown> = this, category: ValidationCategory = 'fast'): void {
if (category === 'built-in') {
throw new Error("The 'built-in' category is reserved for lexer, parser, and linker errors.");
}
for (const [type, ch] of Object.entries(checksRecord)) {
const callbacks = ch as ValidationCheck | ValidationCheck[];
if (Array.isArray(callbacks)) {
for (const check of callbacks) {
this.doRegister(type, this.wrapValidationException(check, thisObj));
const entry: ValidationCheckEntry = {
check: this.wrapValidationException(check, thisObj),
category
};
this.addEntry(type, entry);
}
} else if (typeof callbacks === 'function') {
this.doRegister(type, this.wrapValidationException(callbacks, thisObj));
const entry: ValidationCheckEntry = {
check: this.wrapValidationException(callbacks, thisObj),
category
};
this.addEntry(type, entry);
}
}
}
Expand All @@ -99,18 +141,23 @@ export class ValidationRegistry {
};
}

protected doRegister(type: string, check: ValidationCheck): void {
protected addEntry(type: string, entry: ValidationCheckEntry): void {
if (type === 'AstNode') {
this.validationChecks.add('AstNode', check);
this.entries.add('AstNode', entry);
return;
}
for (const subtype of this.reflection.getAllSubTypes(type)) {
this.validationChecks.add(subtype, check);
this.entries.add(subtype, entry);
}
}

getChecks(type: string): readonly ValidationCheck[] {
return this.validationChecks.get(type).concat(this.validationChecks.get('AstNode'));
getChecks(type: string, categories?: ValidationCategory[]): Stream<ValidationCheck> {
let checks = stream(this.entries.get(type))
.concat(this.entries.get('AstNode'));
if (categories) {
checks = checks.filter(entry => categories.includes(entry.category));
}
return checks.map(entry => entry.check);
}

}
Loading