From 78c4f261c50a207e00d327050bdf67cbfe381918 Mon Sep 17 00:00:00 2001 From: Lee Yi Date: Sun, 14 Apr 2024 19:24:21 -0400 Subject: [PATCH] Unify Runner Execution (#1672) * Refactor more runner logic into SourceRunner * Allow source to run from the desktop * Run format * Made it so that all runners need to go through sourceFilesRunner * Use fetch for docs loading exclusively * Run format * Add new tests to make sure that runners load modules * Make repl work with local imports and add tests * Fix potential issue where repl could return an error * bumping version --------- Co-authored-by: Martin Henz --- package.json | 7 +- src/cse-machine/__tests__/cse-machine-heap.ts | 13 +- .../__tests__/cse-machine-runtime-context.ts | 6 +- .../__tests__/cse-machine-unique-id.ts | 22 +- src/cse-machine/interpreter.ts | 2 +- src/errors/moduleErrors.ts | 51 ---- src/index.ts | 89 ++----- src/infiniteLoops/instrument.ts | 6 +- src/modules/moduleTypes.ts | 2 +- .../preprocessor/__tests__/analyzer.ts | 2 +- src/modules/preprocessor/__tests__/linker.ts | 11 +- .../preprocessor/__tests__/preprocessor.ts | 62 +++-- .../preprocessor/__tests__/resolver.ts | 34 ++- src/modules/preprocessor/analyzer.ts | 3 +- src/modules/preprocessor/index.ts | 43 ++- src/modules/preprocessor/linker.ts | 86 ++++-- src/modules/utils.ts | 4 +- src/repl/__tests__/repl.ts | 250 ++++++++++++++++++ src/repl/index.ts | 11 + src/repl/repl-non-det.ts | 33 ++- src/repl/repl.ts | 205 ++++++-------- src/repl/transpiler.ts | 159 ++++++----- src/repl/utils.ts | 80 ++++++ src/runner/__tests__/files.ts | 50 ++-- src/runner/__tests__/modules.ts | 55 ++++ src/runner/fullJSRunner.ts | 4 - src/runner/sourceRunner.ts | 96 +++++-- src/runner/utils.ts | 17 +- src/stepper/stepper.ts | 2 +- src/transpiler/transpiler.ts | 44 ++- src/utils/ast/dummyAstCreator.ts | 4 +- src/utils/ast/helpers.ts | 57 ++-- src/utils/testing.ts | 14 +- yarn.lock | 15 +- 34 files changed, 986 insertions(+), 553 deletions(-) delete mode 100644 src/errors/moduleErrors.ts create mode 100644 src/repl/__tests__/repl.ts create mode 100644 src/repl/index.ts create mode 100644 src/repl/utils.ts create mode 100644 src/runner/__tests__/modules.ts diff --git a/package.json b/package.json index 948805365..246c902a7 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "js-slang", - "version": "1.0.68", + "version": "1.0.69", "license": "Apache-2.0", "description": "Javascript-based implementations of Source, written in Typescript", "keywords": [ @@ -28,10 +28,11 @@ "dist" ], "bin": { - "js-slang": "dist/repl/repl.js" + "js-slang": "dist/repl/index.js" }, "dependencies": { "@babel/parser": "^7.19.4", + "@commander-js/extra-typings": "^12.0.1", "@joeychenofficial/alt-ergo-modified": "^2.4.0", "@ts-morph/bootstrap": "^0.18.0", "@types/estree": "0.0.52", @@ -40,10 +41,10 @@ "acorn-loose": "^8.0.0", "acorn-walk": "^8.0.0", "astring": "^1.4.3", + "commander": "^12.0.0", "gpu.js": "^2.16.0", "js-base64": "^3.7.5", "lodash": "^4.17.21", - "node-getopt": "^0.3.2", "source-map": "0.7.3" }, "resolutions": { diff --git a/src/cse-machine/__tests__/cse-machine-heap.ts b/src/cse-machine/__tests__/cse-machine-heap.ts index ba5ac8b9c..3565ce6d7 100644 --- a/src/cse-machine/__tests__/cse-machine-heap.ts +++ b/src/cse-machine/__tests__/cse-machine-heap.ts @@ -1,10 +1,9 @@ import { mockClosure, mockContext } from '../../mocks/context' -import { parse } from '../../parser/parser' +import { runCodeInSource } from '../../runner' import { Chapter } from '../../types' import { stripIndent } from '../../utils/formatters' -import { sourceRunner } from '../../runner' import Heap from '../heap' -import { EnvArray } from '../types' +import type { EnvArray } from '../types' test('Heap works correctly', () => { const heap1 = new Heap() @@ -50,11 +49,11 @@ test('Heap works correctly', () => { const expectEnvTreeFrom = (code: string, hasPrelude = true) => { const context = mockContext(Chapter.SOURCE_4) if (!hasPrelude) context.prelude = null - const parsed = parse(code, context) + return expect( - sourceRunner(parsed!, context, false, { executionMethod: 'cse-machine' }).then( - () => context.runtime.environmentTree - ) + runCodeInSource(code, context, { + executionMethod: 'cse-machine' + }).then(() => context.runtime.environmentTree) ).resolves } diff --git a/src/cse-machine/__tests__/cse-machine-runtime-context.ts b/src/cse-machine/__tests__/cse-machine-runtime-context.ts index f80e1b8da..61e3919b9 100644 --- a/src/cse-machine/__tests__/cse-machine-runtime-context.ts +++ b/src/cse-machine/__tests__/cse-machine-runtime-context.ts @@ -1,13 +1,11 @@ import { mockContext } from '../../mocks/context' -import { parse } from '../../parser/parser' +import { runCodeInSource } from '../../runner' import { Chapter } from '../../types' import { stripIndent } from '../../utils/formatters' -import { sourceRunner } from '../../runner' const getContextFrom = async (code: string) => { const context = mockContext(Chapter.SOURCE_4) - const parsed = parse(code, context) - await sourceRunner(parsed!, context, false, { executionMethod: 'cse-machine' }) + await runCodeInSource(code, context, { executionMethod: 'cse-machine' }) return context } diff --git a/src/cse-machine/__tests__/cse-machine-unique-id.ts b/src/cse-machine/__tests__/cse-machine-unique-id.ts index 30ddc2023..75a972fef 100644 --- a/src/cse-machine/__tests__/cse-machine-unique-id.ts +++ b/src/cse-machine/__tests__/cse-machine-unique-id.ts @@ -1,14 +1,15 @@ import { mockContext } from '../../mocks/context' -import { parse } from '../../parser/parser' -import { Chapter, Context, Environment } from '../../types' +import { Chapter, type Context, type Environment } from '../../types' import { stripIndent } from '../../utils/formatters' -import { sourceRunner } from '../../runner' +import { runCodeInSource } from '../../runner' import { createProgramEnvironment } from '../utils' const getContextFrom = async (code: string, envSteps?: number) => { const context = mockContext(Chapter.SOURCE_4) - const parsed = parse(code, context) - await sourceRunner(parsed!, context, false, { envSteps, executionMethod: 'cse-machine' }) + await runCodeInSource(code, context, { + envSteps, + executionMethod: 'cse-machine' + }) return context } @@ -87,21 +88,18 @@ const getProgramEnv = (context: Context) => { } test('Program environment id stays the same regardless of amount of steps', async () => { - const parsed = parse( - stripIndent` + const code = stripIndent` let x = 0; for (let i = 0; i < 10; i = i + 1) { x = [x]; } - `, - mockContext(Chapter.SOURCE_4) - ) + ` + let programEnvId = '47' // The above program has a total of 335 steps // Start from steps = 1 so that the program environment always exists for (let steps = 1; steps < 336; steps++) { - const context = mockContext(Chapter.SOURCE_4) - await sourceRunner(parsed!, context, false, { envSteps: steps, executionMethod: 'cse-machine' }) + const context = await getContextFrom(code, steps) const programEnv = getProgramEnv(context)! if (programEnv.id !== programEnvId) { programEnvId = programEnv.id diff --git a/src/cse-machine/interpreter.ts b/src/cse-machine/interpreter.ts index 9d439e3db..ff1b357c9 100644 --- a/src/cse-machine/interpreter.ts +++ b/src/cse-machine/interpreter.ts @@ -214,7 +214,7 @@ function evaluateImports(program: es.Program, context: Context) { const [importNodeMap] = filterImportDeclarations(program) const environment = currentEnvironment(context) - for (const [moduleName, nodes] of Object.entries(importNodeMap)) { + for (const [moduleName, nodes] of importNodeMap) { const functions = context.nativeStorage.loadedModules[moduleName] for (const node of nodes) { for (const spec of node.specifiers) { diff --git a/src/errors/moduleErrors.ts b/src/errors/moduleErrors.ts deleted file mode 100644 index 94214d4ee..000000000 --- a/src/errors/moduleErrors.ts +++ /dev/null @@ -1,51 +0,0 @@ -/* tslint:disable: max-classes-per-file */ -import { Node } from '../types' -import { RuntimeSourceError } from './runtimeSourceError' - -export class ModuleConnectionError extends RuntimeSourceError { - private static message: string = `Unable to get modules.` - private static elaboration: string = `You should check your Internet connection, and ensure you have used the correct module path.` - constructor(node?: Node) { - super(node) - } - - public explain() { - return ModuleConnectionError.message - } - - public elaborate() { - return ModuleConnectionError.elaboration - } -} - -export class ModuleNotFoundError extends RuntimeSourceError { - constructor(public moduleName: string, node?: Node) { - super(node) - } - - public explain() { - return `Module "${this.moduleName}" not found.` - } - - public elaborate() { - return ` - You should check your import declarations, and ensure that all are valid modules. - ` - } -} - -export class ModuleInternalError extends RuntimeSourceError { - constructor(public moduleName: string, public error?: any, node?: Node) { - super(node) - } - - public explain() { - return `Error(s) occured when executing the module "${this.moduleName}".` - } - - public elaborate() { - return ` - You may need to contact with the author for this module to fix this error. - ` - } -} diff --git a/src/index.ts b/src/index.ts index cc5445e5a..70262b05e 100644 --- a/src/index.ts +++ b/src/index.ts @@ -27,20 +27,11 @@ export { SourceDocumentation } from './editors/ace/docTooltip' import { CSEResultPromise, resumeEvaluate } from './cse-machine/interpreter' import { ModuleNotFoundError } from './modules/errors' -import type { ImportOptions, SourceFiles } from './modules/moduleTypes' +import type { ImportOptions } from './modules/moduleTypes' import preprocessFileImports from './modules/preprocessor' import { validateFilePath } from './modules/preprocessor/filePaths' -import { mergeImportOptions } from './modules/utils' import { getKeywords, getProgramNames, NameDeclaration } from './name-extractor' -import { parse } from './parser/parser' -import { - fullJSRunner, - hasVerboseErrors, - htmlRunner, - resolvedErrorPromise, - sourceFilesRunner -} from './runner' -import { mapResult } from './alt-langs/mapper' +import { htmlRunner, resolvedErrorPromise, sourceFilesRunner } from './runner' export interface IOptions { scheduler: 'preemptive' | 'async' @@ -225,56 +216,38 @@ export async function runFilesInContext( context: Context, options: RecursivePartial = {} ): Promise { - async function runFilesInContextHelper( - files: Partial>, - entrypointFilePath: string, - context: Context, - options: RecursivePartial = {} - ): Promise { - for (const filePath in files) { - const filePathError = validateFilePath(filePath) - if (filePathError !== null) { - context.errors.push(filePathError) - return resolvedErrorPromise - } + for (const filePath in files) { + const filePathError = validateFilePath(filePath) + if (filePathError !== null) { + context.errors.push(filePathError) + return resolvedErrorPromise } + } + let result: Result + if (context.chapter === Chapter.HTML) { const code = files[entrypointFilePath] if (code === undefined) { context.errors.push(new ModuleNotFoundError(entrypointFilePath)) return resolvedErrorPromise } - - if ( - context.chapter === Chapter.FULL_JS || - context.chapter === Chapter.FULL_TS || - context.chapter === Chapter.PYTHON_1 - ) { - const program = parse(code, context) - if (program === null) { - return resolvedErrorPromise - } - - const fullImportOptions = mergeImportOptions(options.importOptions) - return fullJSRunner(program, context, fullImportOptions) - } - - if (context.chapter === Chapter.HTML) { - return htmlRunner(code, context, options) - } - + result = await htmlRunner(code, context, options) + } else { // FIXME: Clean up state management so that the `parseError` function is pure. // This is not a huge priority, but it would be good not to make use of // global state. - verboseErrors = hasVerboseErrors(code) - - // the sourceFilesRunner - return sourceFilesRunner(files, entrypointFilePath, context, options) + ;({ result, verboseErrors } = await sourceFilesRunner( + p => Promise.resolve(files[p]), + entrypointFilePath, + context, + { + ...options, + shouldAddFileName: options.shouldAddFileName ?? Object.keys(files).length > 1 + } + )) } - return runFilesInContextHelper(files, entrypointFilePath, context, options).then( - mapResult(context) - ) + return result } export function resume(result: Result): Finished | ResultError | Promise { @@ -320,23 +293,19 @@ export async function compileFiles( } } - const entrypointCode = files[entrypointFilePath] - if (entrypointCode === undefined) { - context.errors.push(new ModuleNotFoundError(entrypointFilePath)) - return undefined - } - - const preprocessedProgram = await preprocessFileImports( - files as SourceFiles, + const preprocessResult = await preprocessFileImports( + p => Promise.resolve(files[p]), entrypointFilePath, - context + context, + { shouldAddFileName: Object.keys(files).length > 1 } ) - if (!preprocessedProgram) { + + if (!preprocessResult.ok) { return undefined } try { - return compileToIns(preprocessedProgram, undefined, vmInternalFunctions) + return compileToIns(preprocessResult.program, undefined, vmInternalFunctions) } catch (error) { context.errors.push(error) return undefined diff --git a/src/infiniteLoops/instrument.ts b/src/infiniteLoops/instrument.ts index d5223c35b..acd6c7354 100644 --- a/src/infiniteLoops/instrument.ts +++ b/src/infiniteLoops/instrument.ts @@ -6,7 +6,6 @@ import type { Node } from '../types' import * as create from '../utils/ast/astCreator' import { recursive, simple, WalkerCallback } from '../utils/walkers' import { getIdsFromDeclaration } from '../utils/ast/helpers' -import assert from '../utils/assert' // transforms AST of program const globalIds = { @@ -587,10 +586,7 @@ function handleImports(programs: es.Program[]): string[] { program.body = [...importsToAdd, ...otherNodes] return importsToAdd.flatMap(decl => { const ids = getIdsFromDeclaration(decl) - return ids.map(id => { - assert(id !== null, 'Encountered a null identifier') - return id.name - }) + return ids.map(id => id.name) }) }) diff --git a/src/modules/moduleTypes.ts b/src/modules/moduleTypes.ts index 7b37fa2e1..0380c13d1 100644 --- a/src/modules/moduleTypes.ts +++ b/src/modules/moduleTypes.ts @@ -44,5 +44,5 @@ export type ImportOptions = { } & ImportAnalysisOptions & LinkerOptions -export type SourceFiles = Record +export type SourceFiles = Partial> export type FileGetter = (p: string) => Promise diff --git a/src/modules/preprocessor/__tests__/analyzer.ts b/src/modules/preprocessor/__tests__/analyzer.ts index d4778172a..40078d6cc 100644 --- a/src/modules/preprocessor/__tests__/analyzer.ts +++ b/src/modules/preprocessor/__tests__/analyzer.ts @@ -63,7 +63,7 @@ describe('Test throwing import validation errors', () => { ) // Return 'undefined' if there are errors while parsing. - if (context.errors.length !== 0 || !importGraphResult) { + if (context.errors.length !== 0 || !importGraphResult.ok) { throw context.errors[0] } diff --git a/src/modules/preprocessor/__tests__/linker.ts b/src/modules/preprocessor/__tests__/linker.ts index a73e17495..4de464c41 100644 --- a/src/modules/preprocessor/__tests__/linker.ts +++ b/src/modules/preprocessor/__tests__/linker.ts @@ -29,7 +29,7 @@ async function testCode(files: T, entrypointFilePath: key async function expectError(files: T, entrypointFilePath: keyof T) { const [context, result] = await testCode(files, entrypointFilePath) - expect(result).toBeUndefined() + expect(result.ok).toEqual(false) expect(context.errors.length).toBeGreaterThanOrEqual(1) return context.errors } @@ -130,8 +130,13 @@ test('Linker does tree-shaking', async () => { '/a.js' ) + // Wrap to appease typescript + function expectWrapper(cond: boolean): asserts cond { + expect(cond).toEqual(true) + } + expect(errors.length).toEqual(0) - expect(result).toBeDefined() + expectWrapper(result.ok) expect(resolver.default).not.toHaveBeenCalledWith('./b.js') - expect(Object.keys(result!.programs)).not.toContain('/b.js') + expect(Object.keys(result.programs)).not.toContain('/b.js') }) diff --git a/src/modules/preprocessor/__tests__/preprocessor.ts b/src/modules/preprocessor/__tests__/preprocessor.ts index fdef58028..5f0baa0af 100644 --- a/src/modules/preprocessor/__tests__/preprocessor.ts +++ b/src/modules/preprocessor/__tests__/preprocessor.ts @@ -1,9 +1,9 @@ import type { Program } from 'estree' import type { MockedFunction } from 'jest-mock' -import { parseError } from '../../..' +import { parseError, type IOptions } from '../../..' import { mockContext } from '../../../mocks/context' -import { Chapter } from '../../../types' +import { Chapter, type RecursivePartial } from '../../../types' import { memoizedGetModuleDocsAsync } from '../../loader/loaders' import preprocessFileImports from '..' import { sanitizeAST } from '../../../utils/ast/sanitizer' @@ -12,10 +12,13 @@ import { accessExportFunctionName, defaultExportLookupName } from '../../../stdlib/localImport.prelude' +import type { SourceFiles } from '../../moduleTypes' jest.mock('../../loader/loaders') describe('preprocessFileImports', () => { + const wrapFiles = (files: SourceFiles) => (p: string) => Promise.resolve(files[p]) + let actualContext = mockContext(Chapter.LIBRARY_PARSER) let expectedContext = mockContext(Chapter.LIBRARY_PARSER) @@ -24,6 +27,24 @@ describe('preprocessFileImports', () => { expectedContext = mockContext(Chapter.LIBRARY_PARSER) }) + async function expectSuccess( + files: SourceFiles, + entrypointFilePath: string, + options?: RecursivePartial + ) { + const preprocResult = await preprocessFileImports( + p => Promise.resolve(files[p]), + entrypointFilePath, + actualContext, + options + ) + if (!preprocResult.ok) { + throw actualContext.errors[0] + } + + return preprocResult.program + } + const assertASTsAreEquivalent = ( actualProgram: Program | undefined, expectedCode: string, @@ -45,8 +66,16 @@ describe('preprocessFileImports', () => { const files: Record = { '/a.js': '1 + 2;' } - const actualProgram = await preprocessFileImports(files, '/non-existent-file.js', actualContext) - expect(actualProgram).toBeUndefined() + const actualProgram = await preprocessFileImports( + wrapFiles(files), + '/non-existent-file.js', + actualContext + ) + expect(actualProgram).toMatchObject({ + ok: false, + verboseErrors: false + }) + expect(parseError(actualContext.errors)).toMatchInlineSnapshot( `"Module '/non-existent-file.js' not found."` ) @@ -56,8 +85,11 @@ describe('preprocessFileImports', () => { const files: Record = { '/a.js': `import { x } from './non-existent-file.js';` } - const actualProgram = await preprocessFileImports(files, '/a.js', actualContext) - expect(actualProgram).toBeUndefined() + const actualProgram = await preprocessFileImports(wrapFiles(files), '/a.js', actualContext) + expect(actualProgram).toMatchObject({ + ok: false, + verboseErrors: false + }) expect(parseError(actualContext.errors)).toMatchInlineSnapshot( `"Line 1: Module './non-existent-file.js' not found."` ) @@ -73,7 +105,7 @@ describe('preprocessFileImports', () => { ` } const expectedCode = files['/a.js'] - const actualProgram = await preprocessFileImports(files, '/a.js', actualContext) + const actualProgram = await expectSuccess(files, '/a.js') assertASTsAreEquivalent(actualProgram, expectedCode) }) @@ -102,7 +134,7 @@ describe('preprocessFileImports', () => { return x * x * x; } ` - const actualProgram = await preprocessFileImports(files, '/a.js', actualContext) + const actualProgram = await expectSuccess(files, '/a.js') assertASTsAreEquivalent(actualProgram, expectedCode) }) @@ -152,7 +184,7 @@ describe('preprocessFileImports', () => { const y = ${accessExportFunctionName}(___$not$$dash$$source$$dash$$module$$dot$$js___, "y"); const z = ${accessExportFunctionName}(___$not$$dash$$source$$dash$$module$$dot$$js___, "z"); ` - const actualProgram = await preprocessFileImports(files, '/a.js', actualContext, { + const actualProgram = await expectSuccess(files, '/a.js', { importOptions: { allowUndefinedImports: true }, @@ -221,7 +253,7 @@ describe('preprocessFileImports', () => { b; ` - const actualProgram = await preprocessFileImports(files, '/a.js', actualContext, { + const actualProgram = await expectSuccess(files, '/a.js', { importOptions: { allowUndefinedImports: true }, @@ -248,7 +280,7 @@ describe('preprocessFileImports', () => { export const c = 3; ` } - await preprocessFileImports(files, '/a.js', actualContext, { + await preprocessFileImports(wrapFiles(files), '/a.js', actualContext, { shouldAddFileName: true }) expect(parseError(actualContext.errors)).toMatchInlineSnapshot( @@ -274,7 +306,7 @@ describe('preprocessFileImports', () => { export const c = 3; ` } - await preprocessFileImports(files, '/a.js', actualContext) + await preprocessFileImports(wrapFiles(files), '/a.js', actualContext) expect(parseError(actualContext.errors, true)).toMatchInlineSnapshot(` "Circular import detected: '/a.js' -> '/b.js' -> '/c.js' -> '/a.js'. Break the circular import cycle by removing imports from any of the offending files. @@ -290,7 +322,7 @@ describe('preprocessFileImports', () => { export { x as y }; ` } - await preprocessFileImports(files, '/a.js', actualContext) + await preprocessFileImports(wrapFiles(files), '/a.js', actualContext) expect(parseError(actualContext.errors)).toMatchInlineSnapshot( `"Circular import detected: '/a.js' -> '/a.js'."` ) @@ -304,7 +336,7 @@ describe('preprocessFileImports', () => { export { x as y }; ` } - await preprocessFileImports(files, '/a.js', actualContext) + await preprocessFileImports(wrapFiles(files), '/a.js', actualContext) expect(parseError(actualContext.errors, true)).toMatchInlineSnapshot(` "Circular import detected: '/a.js' -> '/a.js'. Break the circular import cycle by removing imports from any of the offending files. @@ -378,7 +410,7 @@ describe('preprocessFileImports', () => { x + y; ` - const actualProgram = await preprocessFileImports(files, '/a.js', actualContext, { + const actualProgram = await expectSuccess(files, '/a.js', { importOptions: { allowUndefinedImports: true }, diff --git a/src/modules/preprocessor/__tests__/resolver.ts b/src/modules/preprocessor/__tests__/resolver.ts index 5c3d728ee..6050fcd4b 100644 --- a/src/modules/preprocessor/__tests__/resolver.ts +++ b/src/modules/preprocessor/__tests__/resolver.ts @@ -1,5 +1,5 @@ import { memoizedGetModuleManifestAsync } from '../../loader/loaders' -import resolveFile, { ImportResolutionOptions, defaultResolutionOptions } from '../resolver' +import resolveFile, { type ImportResolutionOptions, defaultResolutionOptions } from '../resolver' jest.mock('../../loader/loaders') @@ -20,7 +20,7 @@ test('If only local imports are used, the module manifest is not loaded', async expect(memoizedGetModuleManifestAsync).toHaveBeenCalledTimes(0) }) -test('Returns false and resolved path of source file when resolution fails', () => { +it('Returns false and resolved path of source file when resolution fails', () => { return expect( resolveModule('/', './a', () => false, { extensions: ['js'] @@ -28,7 +28,7 @@ test('Returns false and resolved path of source file when resolution fails', () ).resolves.toBeUndefined() }) -test('Will resolve extensions', () => { +it('Will resolve extensions', () => { const mockResolver = (p: string) => p === '/a.ts' return expect( @@ -42,7 +42,7 @@ test('Will resolve extensions', () => { }) }) -test('Will not resolve if the corresponding options are given as false', () => { +it('Will not resolve if the corresponding options are given as false', () => { const mockResolver = (p: string) => p === '/a.js' return expect( resolveModule('/', './a', mockResolver, { @@ -51,7 +51,7 @@ test('Will not resolve if the corresponding options are given as false', () => { ).resolves.toBeUndefined() }) -test('Checks the module manifest when importing source modules', async () => { +it('Checks the module manifest when importing source modules', async () => { const result = await resolveModule('/', 'one_module', () => false, { extensions: ['js'] }) @@ -60,7 +60,7 @@ test('Checks the module manifest when importing source modules', async () => { expect(result).toMatchObject({ type: 'source' }) }) -test('Returns false on failing to resolve a source module', async () => { +it('Returns false on failing to resolve a source module', async () => { const result = await resolveModule('/', 'unknown_module', () => true, { extensions: ['js'] }) @@ -68,3 +68,25 @@ test('Returns false on failing to resolve a source module', async () => { expect(memoizedGetModuleManifestAsync).toHaveBeenCalledTimes(1) expect(result).toBeUndefined() }) + +test('Resolving an absolute path from a local module', () => { + return expect( + resolveFile( + './dir0/dir1/a.js', + '/b.js', + p => + Promise.resolve( + { + '/b.js': 'contents' + }[p] + ), + { + extensions: null + } + ) + ).resolves.toMatchObject({ + type: 'local', + contents: 'contents', + absPath: '/b.js' + }) +}) diff --git a/src/modules/preprocessor/analyzer.ts b/src/modules/preprocessor/analyzer.ts index 563d0c93b..d066a50dc 100644 --- a/src/modules/preprocessor/analyzer.ts +++ b/src/modules/preprocessor/analyzer.ts @@ -82,8 +82,7 @@ export default function analyzeImportsAndExports( if (!options.allowUndefinedImports) { const ids = getIdsFromDeclaration(node.declaration) ids.forEach(id => { - assert(id !== null, 'Encountered a null identifier!') - return moduleDocs[sourceModule].add(id.name) + moduleDocs[sourceModule].add(id.name) }) } continue diff --git a/src/modules/preprocessor/index.ts b/src/modules/preprocessor/index.ts index 1b92bcf0f..76e3fb181 100644 --- a/src/modules/preprocessor/index.ts +++ b/src/modules/preprocessor/index.ts @@ -3,11 +3,23 @@ import type es from 'estree' import type { Context, IOptions } from '../..' import type { RecursivePartial } from '../../types' import loadSourceModules from '../loader' -import type { FileGetter, SourceFiles } from '../moduleTypes' +import type { FileGetter } from '../moduleTypes' import analyzeImportsAndExports from './analyzer' import parseProgramsAndConstructImportGraph from './linker' import defaultBundler, { type Bundler } from './bundler' +export type PreprocessResult = + | { + ok: true + program: es.Program + files: Record + verboseErrors: boolean + } + | { + ok: false + verboseErrors: boolean + } + /** * Preprocesses file imports and returns a transformed Abstract Syntax Tree (AST). * If an error is encountered at any point, returns `undefined` to signify that an @@ -26,26 +38,26 @@ import defaultBundler, { type Bundler } from './bundler' * @param context The information associated with the program evaluation. */ const preprocessFileImports = async ( - files: SourceFiles | FileGetter, + files: FileGetter, entrypointFilePath: string, context: Context, options: RecursivePartial = {}, bundler: Bundler = defaultBundler -): Promise => { +): Promise => { // Parse all files into ASTs and build the import graph. - const importGraphResult = await parseProgramsAndConstructImportGraph( - typeof files === 'function' ? files : p => Promise.resolve(files[p]), + const linkerResult = await parseProgramsAndConstructImportGraph( + files, entrypointFilePath, context, options?.importOptions, - options?.shouldAddFileName ?? (typeof files === 'function' || Object.keys(files).length > 1) + !!options?.shouldAddFileName ) // Return 'undefined' if there are errors while parsing. - if (!importGraphResult || context.errors.length !== 0) { - return undefined + if (!linkerResult.ok) { + return linkerResult } - const { programs, topoOrder, sourceModulesToImport } = importGraphResult + const { programs, topoOrder, sourceModulesToImport } = linkerResult try { await loadSourceModules(sourceModulesToImport, context, options.importOptions?.loadTabs ?? true) @@ -59,10 +71,19 @@ const preprocessFileImports = async ( ) } catch (error) { context.errors.push(error) - return undefined + return { + ok: false, + verboseErrors: linkerResult.verboseErrors + } } - return bundler(programs, entrypointFilePath, topoOrder, context) + const program = bundler(programs, entrypointFilePath, topoOrder, context) + return { + ok: true, + program, + files: linkerResult.files, + verboseErrors: linkerResult.verboseErrors + } } export default preprocessFileImports diff --git a/src/modules/preprocessor/linker.ts b/src/modules/preprocessor/linker.ts index 080e30992..4738a1fc0 100644 --- a/src/modules/preprocessor/linker.ts +++ b/src/modules/preprocessor/linker.ts @@ -7,6 +7,8 @@ import { CircularImportError, ModuleNotFoundError } from '../errors' import { getModuleDeclarationSource } from '../../utils/ast/helpers' import type { FileGetter } from '../moduleTypes' import { mapAndFilter } from '../../utils/misc' +import { parseAt } from '../../parser/utils' +import { isDirective } from '../../utils/ast/typeGuards' import { DirectedGraph } from './directedGraph' import resolveFile, { defaultResolutionOptions, type ImportResolutionOptions } from './resolver' @@ -19,11 +21,19 @@ type ModuleDeclarationWithSource = Exclude - sourceModulesToImport: Set - topoOrder: string[] -} +export type LinkerResult = + | { + ok: false + verboseErrors: boolean + } + | { + ok: true + programs: Record + files: Record + sourceModulesToImport: Set + topoOrder: string[] + verboseErrors: boolean + } export type LinkerOptions = { resolverOptions: ImportResolutionOptions @@ -46,9 +56,10 @@ export default async function parseProgramsAndConstructImportGraph( context: Context, options: RecursivePartial = defaultLinkerOptions, shouldAddFileName: boolean -): Promise { +): Promise { const importGraph = new DirectedGraph() const programs: Record = {} + const files: Record = {} const sourceModulesToImport = new Set() // Wrapper around resolve file to make calling it more convenient @@ -100,6 +111,7 @@ export default async function parseProgramsAndConstructImportGraph( } programs[fromModule] = program + files[fromModule] = fileText await Promise.all( mapAndFilter(program.body, node => { @@ -120,32 +132,74 @@ export default async function parseProgramsAndConstructImportGraph( ) } + let entrypointFileText: string | undefined = undefined + + function hasVerboseErrors() { + // Always try to infer if verbose errors should be enabled + let statement: es.Node | null = null + + if (!programs[entrypointFilePath]) { + if (entrypointFileText == undefined) { + // non-existent entrypoint + return false + } + // There are syntax errors in the entrypoint file + // we use parseAt to try parse the first line + statement = parseAt(entrypointFileText, 0) as es.Node | null + } else { + // Otherwise we can use the entrypoint program as it has been passed + const entrypointProgram = programs[entrypointFilePath] + + // Check if the program had any code at all + if (entrypointProgram.body.length === 0) return false + ;[statement] = entrypointProgram.body + } + + if (statement === null) return false + + // The two different parsers end up with two different ASTs + // These are the two cases where 'enable verbose' appears + // as a directive + if (isDirective(statement)) { + return statement.directive === 'enable verbose' + } + + return statement.type === 'Literal' && statement.value === 'enable verbose' + } + try { - const entrypointFileText = await fileGetter(entrypointFilePath) + entrypointFileText = await fileGetter(entrypointFilePath) // Not using boolean test here, empty strings are valid programs // but are falsy if (entrypointFileText === undefined) { throw new ModuleNotFoundError(entrypointFilePath) } + await parseAndEnumerateModuleDeclarations(entrypointFilePath, entrypointFileText) const topologicalOrderResult = importGraph.getTopologicalOrder() - if (!topologicalOrderResult.isValidTopologicalOrderFound) { - context.errors.push(new CircularImportError(topologicalOrderResult.firstCycleFound)) - return undefined + if (topologicalOrderResult.isValidTopologicalOrderFound) { + return { + ok: true, + topoOrder: topologicalOrderResult.topologicalOrder, + programs, + sourceModulesToImport, + files, + verboseErrors: hasVerboseErrors() + } } - return { - topoOrder: topologicalOrderResult.topologicalOrder, - programs, - sourceModulesToImport - } + context.errors.push(new CircularImportError(topologicalOrderResult.firstCycleFound)) } catch (error) { if (!(error instanceof LinkerError)) { // Any other error that occurs is just appended to the context // and we return undefined context.errors.push(error) } - return undefined + } + + return { + ok: false, + verboseErrors: hasVerboseErrors() } } diff --git a/src/modules/utils.ts b/src/modules/utils.ts index 080ca879c..ef658b32a 100644 --- a/src/modules/utils.ts +++ b/src/modules/utils.ts @@ -1,7 +1,7 @@ import * as _ from 'lodash' -import { RecursivePartial } from '../types' -import { ImportOptions } from './moduleTypes' +import type { RecursivePartial } from '../types' +import type { ImportOptions } from './moduleTypes' import { defaultAnalysisOptions } from './preprocessor/analyzer' const exportDefaultStr = 'export default' diff --git a/src/repl/__tests__/repl.ts b/src/repl/__tests__/repl.ts new file mode 100644 index 000000000..bf2b4b35e --- /dev/null +++ b/src/repl/__tests__/repl.ts @@ -0,0 +1,250 @@ +import type { SourceFiles } from '../../modules/moduleTypes' +import * as repl from 'repl' +import { Chapter } from '../../types' +import { asMockedFunc } from '../../utils/testing' +import { getReplCommand } from '../repl' +import { chapterParser } from '../utils' + +const readFileMocker = jest.fn() + +function mockReadFiles(files: SourceFiles) { + readFileMocker.mockImplementation((fileName: string) => { + if (fileName in files) return Promise.resolve(files[fileName]) + return Promise.reject({ code: 'ENOENT' }) + }) +} + +jest.mock('fs/promises', () => ({ + readFile: readFileMocker +})) + +jest.mock('path', () => { + const actualPath = jest.requireActual('path') + const newResolve = (...args: string[]) => actualPath.resolve('/', ...args) + return { + ...actualPath, + resolve: newResolve + } +}) + +jest.mock('../../modules/loader/loaders') + +jest.spyOn(console, 'log') +const mockedConsoleLog = asMockedFunc(console.log) + +jest.spyOn(repl, 'start') + +describe('Test chapter parser', () => + test.each([ + ['1', Chapter.SOURCE_1], + ['SOURCE_1', Chapter.SOURCE_1], + ['2', Chapter.SOURCE_2], + ['SOURCE_2', Chapter.SOURCE_2], + ['3', Chapter.SOURCE_3], + ['SOURCE_3', Chapter.SOURCE_3], + ['4', Chapter.SOURCE_4], + ['SOURCE_4', Chapter.SOURCE_4], + ['random string', undefined], + ['525600', undefined] + ])('%#', (value, expected) => { + if (!expected) { + expect(() => chapterParser(value)).toThrow() + return + } + + expect(chapterParser(value)).toEqual(expected) + })) + +describe('Test repl command', () => { + beforeEach(() => { + jest.clearAllMocks() + }) + + const runCommand = (...args: string[]) => { + const promise = getReplCommand().parseAsync(args, { from: 'user' }) + return expect(promise).resolves.not.toThrow() + } + + describe('Test running files', () => { + type TestCase = [desc: string, args: string[], files: SourceFiles, expected: string] + + const testCases: TestCase[] = [ + [ + 'Regular running', + ['d.js'], + { + '/a/a.js': ` + import { b } from './b.js'; + export function a() { + return b(); + } + `, + '/a/b.js': ` + import { c } from '../c/c.js'; + export function b() { + return c + " and b"; + } + `, + '/c/c.js': ` + export const c = "c"; + `, + '/d.js': ` + import { a } from './a/a.js'; + a(); + ` + }, + '"c and b"' + ], + [ + 'Unknown local import', + ['a.js'], + { + '/a.js': 'import { b } from "./b.js";' + }, + "Error: [/a.js] Line 1: Module './b.js' not found." + ], + [ + 'Unknown local import - verbose', + ['a.js', '--verbose'], + { + '/a.js': 'import { b } from "./b.js";' + }, + + "Error: [/a.js] Line 1, Column 0: Module './b.js' not found.\nYou should check your import declarations, and ensure that all are valid modules.\n" + ], + [ + 'Source imports are ok', + ['a.js'], + { + '/a.js': "import { foo } from 'one_module'; foo();" + }, + '"foo"' + ], + [ + 'Unknown Source imports are handled properly', + ['a.js'], + { + '/a.js': "import { foo } from 'unknown_module'; foo();" + }, + "Error: [/a.js] Line 1: Module 'unknown_module' not found." + ] + ] + test.each(testCases)('%s', async (_, args, files, expected) => { + mockReadFiles(files) + await runCommand(...args) + expect(mockedConsoleLog.mock.calls[0][0]).toEqual(expected) + }) + }) + + describe('Test running with REPL', () => { + function mockReplStart() { + type MockedReplReturn = (x: string) => Promise + + const mockedReplStart = asMockedFunc(repl.start) + return new Promise(resolve => { + mockedReplStart.mockImplementation((args: repl.ReplOptions) => { + const runCode = (code: string) => + new Promise(resolve => { + args.eval!.call({}, code, {} as any, '', (err: Error | null, result: any) => { + if (err) resolve(err) + resolve(result) + }) + }) + + resolve(async code => { + const output = await runCode(code) + return args.writer!.call({}, output) + }) + return {} as any + }) + }) + } + + const runRepl = async (args: string[], expected: [string, string][]) => { + const replPromise = mockReplStart() + await runCommand(...args) + const func = await replPromise + expect(repl.start).toHaveBeenCalledTimes(1) + + for (const [input, output] of expected) { + await expect(func(input)).resolves.toEqual(output) + } + } + + test('Running without file name', () => + runRepl( + [], + [ + ['const x = 1 + 1;', 'undefined'], + ['x;', '2'] + ] + )) + + test('REPL is able to recover from errors', () => + runRepl( + [], + [ + ['const x = 1 + 1;', 'undefined'], + ['var0;', 'Error: Line 1: Name var0 not declared.'], + ['x;', '2'], + ['var0;', 'Error: Line 1: Name var0 not declared.'], + ['const var0 = 0;', 'undefined'], + ['var0;', '0'] + ] + )) + + test('Running with a file name evaluates code and then enters the REPL', async () => { + mockReadFiles({ + '/a.js': ` + import { b } from './b.js'; + function a() { return "a"; } + const c = "c"; + `, + '/b.js': ` + export function b() { return "b"; } + ` + }) + + await runRepl( + ['a.js', '-r'], + [ + ['const x = 1 + 1;', 'undefined'], + ['x;', '2'], + ['const y = a();', 'undefined'], + ['y;', '"a"'], + ['b();', '"b"'], + ['c;', '"c"'], + ['const c = 0;', 'undefined'], + ['c;', '0'] + ] + ) + }) + + test('REPL handles Source import statements ok', () => + runRepl( + [], + [ + ['const foo = () => "bar";', 'undefined'], + ['foo();', '"bar"'], + ['import { foo } from "one_module";', 'undefined'], + ['foo();', '"foo"'] + ] + )) + + test('REPL handles local import statements ok', async () => { + mockReadFiles({ + '/a.js': ` + export function a() { return "a"; } + ` + }) + + await runRepl( + [], + [ + ['import { a } from "./a.js";', 'undefined'], + ['a();', '"a"'] + ] + ) + }) + }) +}) diff --git a/src/repl/index.ts b/src/repl/index.ts new file mode 100644 index 000000000..ae05cf82f --- /dev/null +++ b/src/repl/index.ts @@ -0,0 +1,11 @@ +import { Command } from '@commander-js/extra-typings' + +import { getReplCommand } from './repl' +import { nonDetCommand } from './repl-non-det' +import { transpilerCommand } from './transpiler' + +new Command() + .addCommand(transpilerCommand) + .addCommand(getReplCommand(), { isDefault: true }) + .addCommand(nonDetCommand) + .parseAsync() diff --git a/src/repl/repl-non-det.ts b/src/repl/repl-non-det.ts index 12153f5d3..0f5d09a73 100644 --- a/src/repl/repl-non-det.ts +++ b/src/repl/repl-non-det.ts @@ -1,9 +1,10 @@ -import * as fs from 'fs' +import type fslib from 'fs/promises' import * as repl from 'repl' // 'repl' here refers to the module named 'repl' in index.d.ts import { inspect } from 'util' +import { Command } from '@commander-js/extra-typings' +import { createContext, type IOptions, parseError, type Result, resume, runInContext } from '..' import { CUT, TRY_AGAIN } from '../constants' -import { createContext, IOptions, parseError, Result, resume, runInContext } from '../index' import Closure from '../interpreter/closure' import { Chapter, Context, SuspendedNonDet, Variant } from '../types' @@ -113,20 +114,16 @@ function _startRepl(chapter: Chapter = Chapter.SOURCE_1, useSubst: boolean, prel }) } -function main() { - const firstArg = process.argv[2] - if (process.argv.length === 3 && String(Number(firstArg)) !== firstArg.trim()) { - fs.readFile(firstArg, 'utf8', (err, data) => { - if (err) { - throw err - } - _startRepl(Chapter.SOURCE_3, false, data) - }) - } else { - const chapter = Chapter.SOURCE_3 - const useSubst = process.argv.length > 3 ? process.argv[3] === 'subst' : false - _startRepl(chapter, useSubst) - } -} +export const nonDetCommand = new Command('non-det') + .option('--useSubst') + .argument('') + .action(async (fileName, { useSubst }) => { + if (fileName !== undefined) { + const fs: typeof fslib = require('fs/promises') + const data = await fs.readFile(fileName, 'utf-8') -main() + _startRepl(Chapter.SOURCE_3, false, data) + } else { + _startRepl(Chapter.SOURCE_3, !!useSubst) + } + }) diff --git a/src/repl/repl.ts b/src/repl/repl.ts index 4452978c8..0e13c5cda 100644 --- a/src/repl/repl.ts +++ b/src/repl/repl.ts @@ -1,132 +1,97 @@ -#!/usr/bin/env node -import { start } from 'repl' // 'repl' here refers to the module named 'repl' in index.d.ts -import { inspect } from 'util' +import type fslib from 'fs/promises' +import { resolve } from 'path' +import { start } from 'repl' +import { Command } from '@commander-js/extra-typings' -import { pyLanguages, scmLanguages, sourceLanguages } from '../constants' -import { createContext, IOptions, parseError, runInContext } from '../index' -import Closure from '../interpreter/closure' -import { ExecutionMethod, Variant } from '../types' -import { Representation } from '../alt-langs/mapper' +import { createContext, type IOptions } from '..' +import { setModulesStaticURL } from '../modules/loader' +import { Chapter, type RecursivePartial, Variant } from '../types' +import { objectValues } from '../utils/misc' +import { runCodeInSource, sourceFilesRunner } from '../runner' +import type { FileGetter } from '../modules/moduleTypes' +import { + chapterParser, + getChapterOption, + getVariantOption, + handleResult, + validChapterVariant +} from './utils' -function startRepl( - chapter = 1, - executionMethod: ExecutionMethod = 'interpreter', - variant: Variant = Variant.DEFAULT, - useSubst: boolean = false, - useRepl: boolean, - prelude = '' -) { - // use defaults for everything - const context = createContext(chapter, variant, undefined, undefined) - const options: Partial = { - scheduler: 'preemptive', - executionMethod, - variant, - useSubst - } - runInContext(prelude, context, options).then(preludeResult => { - if (preludeResult.status === 'finished' || preludeResult.status === 'suspended-non-det') { - console.dir(preludeResult.value, { depth: null }) - if (!useRepl) { +export const getReplCommand = () => + new Command('run') + .addOption(getChapterOption(Chapter.SOURCE_4, chapterParser)) + .addOption(getVariantOption(Variant.DEFAULT, objectValues(Variant))) + .option('-v, --verbose', 'Enable verbose errors') + .option('--modulesBackend ') + .option('-r, --repl', 'Start a REPL after evaluating files') + .option('--optionsFile ', 'Specify a JSON file to read options from') + .argument('[filename]') + .action(async (filename, { modulesBackend, optionsFile, repl, verbose, ...lang }) => { + if (!validChapterVariant(lang)) { + console.log('Invalid language combination!') return } + + const fs: typeof fslib = require('fs/promises') + + const context = createContext(lang.chapter, lang.variant) + + if (modulesBackend !== undefined) { + setModulesStaticURL(modulesBackend) + } + + let options: RecursivePartial = {} + if (optionsFile !== undefined) { + const rawText = await fs.readFile(optionsFile, 'utf-8') + options = JSON.parse(rawText) + } + + const fileGetter: FileGetter = async p => { + try { + const text = await fs.readFile(p, 'utf-8') + return text + } catch (error) { + if (error.code === 'ENOENT') return undefined + throw error + } + } + + if (filename !== undefined) { + const entrypointFilePath = resolve(filename) + const { result, verboseErrors } = await sourceFilesRunner( + fileGetter, + entrypointFilePath, + context, + { + ...options, + shouldAddFileName: true + } + ) + + const toLog = handleResult(result, context, verbose ?? verboseErrors) + console.log(toLog) + + if (!repl) return + } + start( // the object being passed as argument fits the interface ReplOptions in the repl module. { eval: (cmd, unusedContext, unusedFilename, callback) => { - runInContext(cmd, context, options).then(obj => { - if (obj.status === 'finished' || obj.status === 'suspended-non-det') { - // if the result is a representation, display the representation. - // else, default to standard value representation. - callback(null, obj.representation !== undefined ? obj.representation : obj.value) - } else { - callback(new Error(parseError(context.errors)), undefined) - } - }) + context.errors = [] + runCodeInSource(cmd, context, options, '/default.js', fileGetter) + .then(obj => { + callback(null, obj) + }) + .catch(err => callback(err, undefined)) }, - // set depth to a large number so that `parse()` output will not be folded, - // setting to null also solves the problem, however a reference loop might crash - writer: output => { - return output instanceof Closure || - typeof output === 'function' || - output instanceof Representation - ? output.toString() - : inspect(output, { - depth: 1000, - colors: true - }) + writer: (output: Awaited> | Error) => { + if (output instanceof Error) { + return output.message + } + + return handleResult(output.result, context, verbose ?? output.verboseErrors) } } ) - } else { - console.error(parseError(context.errors)) - } - }) -} - -/** - * Returns true iff the given chapter and variant combination is supported. - */ -function validChapterVariant(chapter: any, variant: any) { - if (variant === 'interpreter') { - return true - } - // TODO explicit control should only be done with source chapter 4 - if (variant === 'explicit-control') { - return true - } - if (variant === 'substituter' && (chapter === 1 || chapter === 2)) { - return true - } - for (const lang of sourceLanguages) { - if (lang.chapter === chapter && lang.variant === variant) return true - } - for (const lang of scmLanguages) { - if (lang.chapter === chapter && lang.variant === variant) return true - } - for (const lang of pyLanguages) { - if (lang.chapter === chapter && lang.variant === variant) return true - } - - return false -} - -function main() { - const opt = require('node-getopt') - .create([ - ['c', 'chapter=CHAPTER', 'set the Source chapter number (i.e., 1-4)', '1'], - [ - 'v', - 'variant=VARIANT', - 'set the Source variant (i.e., default, interpreter, substituter, lazy, non-det, concurrent, wasm, gpu)', - 'default' - ], - ['h', 'help', 'display this help'], - ['e', 'eval', "don't show REPL, only display output of evaluation"] - ]) - .bindHelp() - .setHelp('Usage: js-slang [PROGRAM_STRING] [OPTION]\n\n[[OPTIONS]]') - .parseSystem() - - const variant = opt.options.variant - const chapter = parseInt(opt.options.chapter, 10) - const areValidChapterVariant: boolean = validChapterVariant(chapter, variant) - if (!areValidChapterVariant) { - throw new Error( - 'The chapter and variant combination provided is unsupported. Use the -h option to view valid chapters and variants.' - ) - } - - const executionMethod = - opt.options.variant === 'interpreter' || - opt.options.variant === 'non-det' || - opt.options.variant === 'explicit-control' - ? 'interpreter' - : 'native' - const useSubst = opt.options.variant === 'substituter' - const useRepl = !opt.options.e - const prelude = opt.argv[0] ?? '' - startRepl(chapter, executionMethod, variant, useSubst, useRepl, prelude) -} - -main() + }) diff --git a/src/repl/transpiler.ts b/src/repl/transpiler.ts index c812ea9e3..a2a8fa518 100644 --- a/src/repl/transpiler.ts +++ b/src/repl/transpiler.ts @@ -1,98 +1,91 @@ #!/usr/bin/env node +import type fslib from 'fs/promises' +import { resolve } from 'path' +import { Command } from '@commander-js/extra-typings' import { generate } from 'astring' -import { Program } from 'estree' -import { sourceLanguages } from '../constants' import { transpileToGPU } from '../gpu/gpu' import { createContext, parseError } from '../index' import { transpileToLazy } from '../lazy/lazy' -import { parse } from '../parser/parser' +import defaultBundler from '../modules/preprocessor/bundler' +import parseProgramsAndConstructImportGraph from '../modules/preprocessor/linker' import { transpile } from '../transpiler/transpiler' import { Chapter, Variant } from '../types' -import { validateAndAnnotate } from '../validator/validator' +import { + chapterParser, + getChapterOption, + getVariantOption, + validateChapterAndVariantCombo +} from './utils' -function transpileCode( - chapter: Chapter = Chapter.SOURCE_1, - variant: Variant = Variant.DEFAULT, - code = '', - pretranspile = false -) { - // use defaults for everything - const context = createContext(chapter, variant, undefined, undefined) - const program = parse(code, context) - if (program === null) { - throw Error(parseError(context.errors, true)) - } - validateAndAnnotate(program as Program, context) - switch (variant) { - case Variant.GPU: - transpileToGPU(program) - break - case Variant.LAZY: - transpileToLazy(program) - break - } - if (pretranspile) { - return generate(program) - } else { - const { transpiled } = transpile(program as Program, context) - return transpiled - } -} +export const transpilerCommand = new Command() + .addOption( + getVariantOption(Variant.DEFAULT, [Variant.DEFAULT, Variant.GPU, Variant.LAZY, Variant.NATIVE]) + ) + .addOption(getChapterOption(Chapter.SOURCE_4, chapterParser)) + .option( + '-p, --pretranspile', + "only pretranspile (e.g. GPU -> Source) and don't perform Source -> JS transpilation" + ) + .option('-o, --out ', 'Specify a file to write to') + .argument('') + .action(async (fileName, opts) => { + if (!validateChapterAndVariantCombo(opts)) { + console.log('Invalid language combination!') + return + } -/** - * Returns true iff the given chapter and variant combination is supported. - */ -function validChapterVariant(chapter: any, variant: any) { - for (const lang of sourceLanguages) { - if (lang.chapter === chapter && lang.variant === variant) return true - } + const fs: typeof fslib = require('fs/promises') + const context = createContext(opts.chapter, opts.variant) + const entrypointFilePath = resolve(fileName) - return false -} + const linkerResult = await parseProgramsAndConstructImportGraph( + async p => { + try { + const text = await fs.readFile(p, 'utf-8') + return text + } catch (error) { + if (error.code === 'ENOENT') return undefined + throw error + } + }, + entrypointFilePath, + context, + {}, + true + ) -function main() { - const opt = require('node-getopt') - .create([ - ['c', 'chapter=CHAPTER', 'set the Source chapter number (i.e., 1-4)', '1'], - [ - 'p', - 'pretranspile', - "only pretranspile (e.g. GPU -> Source) and don't perform Source -> JS transpilation" - ], - [ - 'v', - 'variant=VARIANT', - 'set the Source variant (i.e., default, interpreter, substituter, lazy, non-det, concurrent, wasm, gpu)', - 'default' - ], - ['h', 'help', 'display this help'] - ]) - .bindHelp() - .setHelp('Usage: js-slang-transpiler [OPTION]\n\n[[OPTIONS]]') - .parseSystem() + if (!linkerResult.ok) { + console.log(parseError(context.errors, linkerResult.verboseErrors)) + return + } - const pretranspile = opt.options.pretranspile - const variant = opt.options.variant - const chapter = parseInt(opt.options.chapter, 10) - const valid = validChapterVariant(chapter, variant) - if ( - !valid || - !(variant === Variant.DEFAULT || variant === Variant.LAZY || variant === Variant.GPU) - ) { - throw new Error( - 'The chapter and variant combination provided is unsupported. Use the -h option to view valid chapters and variants.' - ) - } + const { programs, topoOrder } = linkerResult + const bundledProgram = defaultBundler(programs, entrypointFilePath, topoOrder, context) - const chunks: Buffer[] = [] - process.stdin.on('data', chunk => { - chunks.push(chunk) - }) - process.stdin.on('end', () => { - const code = Buffer.concat(chunks).toString('utf-8') - process.stdout.write(transpileCode(chapter, variant, code, pretranspile).transpiled) - }) -} + switch (opts.variant) { + case Variant.GPU: + transpileToGPU(bundledProgram) + break + case Variant.LAZY: + transpileToLazy(bundledProgram) + break + } -main() + const transpiled = opts.pretranspile + ? generate(bundledProgram) + : transpile(bundledProgram, context).transpiled + + if (context.errors.length > 0) { + console.log(parseError(context.errors, linkerResult.verboseErrors)) + return + } + + if (opts.out) { + const resolvedOut = resolve(opts.out) + await fs.writeFile(resolvedOut, transpiled) + console.log(`Code written to ${resolvedOut}`) + } else { + console.log(transpiled) + } + }) diff --git a/src/repl/utils.ts b/src/repl/utils.ts new file mode 100644 index 000000000..cc7edec9d --- /dev/null +++ b/src/repl/utils.ts @@ -0,0 +1,80 @@ +import { Option } from '@commander-js/extra-typings' + +import { pyLanguages, scmLanguages, sourceLanguages } from '../constants' +import { Chapter, type Language, Variant, type Result } from '../types' +import { stringify } from '../utils/stringify' +import Closure from '../cse-machine/closure' +import { parseError, type Context } from '..' + +export function chapterParser(str: string): Chapter { + let foundChapter: string | undefined + + if (/^[0-9]$/.test(str)) { + // Chapter is fully numeric + const value = parseInt(str) + foundChapter = Object.keys(Chapter).find(chapterName => Chapter[chapterName] === value) + + if (foundChapter === undefined) { + throw new Error(`Invalid chapter value: ${str}`) + } + } else { + foundChapter = str + } + + if (foundChapter in Chapter) { + return Chapter[foundChapter] + } + throw new Error(`Invalid chapter value: ${str}`) +} + +export const getChapterOption = ( + defaultValue: T, + argParser: (value: string) => T +) => { + return new Option('--chapter ').default(defaultValue).argParser(argParser) +} + +export const getVariantOption = (defaultValue: T, choices: T[]) => { + return new Option('--variant ').default(defaultValue).choices(choices) +} + +export function validateChapterAndVariantCombo(language: Language) { + for (const { chapter, variant } of sourceLanguages) { + if (language.chapter === chapter && language.variant === variant) return true + } + return false +} + +/** + * Returns true iff the given chapter and variant combination is supported. + */ +export function validChapterVariant(language: Language) { + const { chapter, variant } = language + + for (const lang of sourceLanguages) { + if (lang.chapter === chapter && lang.variant === variant) return true + } + for (const lang of scmLanguages) { + if (lang.chapter === chapter && lang.variant === variant) return true + } + for (const lang of pyLanguages) { + if (lang.chapter === chapter && lang.variant === variant) return true + } + + return false +} + +export function handleResult(result: Result, context: Context, verboseErrors: boolean) { + if (result.status === 'finished' || result.status === 'suspended-non-det') { + if ( + result instanceof Closure || + typeof result === 'function' || + result.representation !== undefined + ) { + return result.toString() + } + return stringify(result.value) + } + + return `Error: ${parseError(context.errors, verboseErrors)}` +} diff --git a/src/runner/__tests__/files.ts b/src/runner/__tests__/files.ts index fc117bd08..a7ec831e6 100644 --- a/src/runner/__tests__/files.ts +++ b/src/runner/__tests__/files.ts @@ -9,23 +9,23 @@ describe('runFilesInContext', () => { context = mockContext(Chapter.SOURCE_4) }) - it('returns IllegalCharInFilePathError if any file path contains invalid characters', () => { + it('returns IllegalCharInFilePathError if any file path contains invalid characters', async () => { const files: Record = { '/a.js': '1 + 2;', '/+-.js': '"hello world";' } - runFilesInContext(files, '/a.js', context) + await runFilesInContext(files, '/a.js', context) expect(parseError(context.errors)).toMatchInlineSnapshot( `"File path '/+-.js' must only contain alphanumeric chars and/or '_', '/', '.', '-'."` ) }) - it('returns IllegalCharInFilePathError if any file path contains invalid characters - verbose', () => { + it('returns IllegalCharInFilePathError if any file path contains invalid characters - verbose', async () => { const files: Record = { '/a.js': '1 + 2;', '/+-.js': '"hello world";' } - runFilesInContext(files, '/a.js', context) + await runFilesInContext(files, '/a.js', context) expect(parseError(context.errors, true)).toMatchInlineSnapshot(` "File path '/+-.js' must only contain alphanumeric chars and/or '_', '/', '.', '-'. Rename the offending file path to only use valid chars. @@ -33,23 +33,23 @@ describe('runFilesInContext', () => { `) }) - it('returns ConsecutiveSlashesInFilePathError if any file path contains consecutive slash characters', () => { + it('returns ConsecutiveSlashesInFilePathError if any file path contains consecutive slash characters', async () => { const files: Record = { '/a.js': '1 + 2;', '/dir//dir2/b.js': '"hello world";' } - runFilesInContext(files, '/a.js', context) + await runFilesInContext(files, '/a.js', context) expect(parseError(context.errors)).toMatchInlineSnapshot( `"File path '/dir//dir2/b.js' cannot contain consecutive slashes '//'."` ) }) - it('returns ConsecutiveSlashesInFilePathError if any file path contains consecutive slash characters - verbose', () => { + it('returns ConsecutiveSlashesInFilePathError if any file path contains consecutive slash characters - verbose', async () => { const files: Record = { '/a.js': '1 + 2;', '/dir//dir2/b.js': '"hello world";' } - runFilesInContext(files, '/a.js', context) + await runFilesInContext(files, '/a.js', context) expect(parseError(context.errors, true)).toMatchInlineSnapshot(` "File path '/dir//dir2/b.js' cannot contain consecutive slashes '//'. Remove consecutive slashes from the offending file path. @@ -57,15 +57,15 @@ describe('runFilesInContext', () => { `) }) - it('returns ModuleNotFoundError if entrypoint file does not exist', () => { + it('returns ModuleNotFoundError if entrypoint file does not exist', async () => { const files: Record = {} - runFilesInContext(files, '/a.js', context) + await await runFilesInContext(files, '/a.js', context) expect(parseError(context.errors)).toMatchInlineSnapshot(`"Module '/a.js' not found."`) }) - it('returns ModuleNotFoundError if entrypoint file does not exist - verbose', () => { + it('returns ModuleNotFoundError if entrypoint file does not exist - verbose', async () => { const files: Record = {} - runFilesInContext(files, '/a.js', context) + await runFilesInContext(files, '/a.js', context) expect(parseError(context.errors, true)).toMatchInlineSnapshot(` "Module '/a.js' not found. You should check your import declarations, and ensure that all are valid modules. @@ -77,27 +77,27 @@ describe('runFilesInContext', () => { describe('compileFiles', () => { let context = mockContext(Chapter.SOURCE_4) - beforeEach(() => { + beforeEach(async () => { context = mockContext(Chapter.SOURCE_4) }) - it('returns IllegalCharInFilePathError if any file path contains invalid characters', () => { + it('returns IllegalCharInFilePathError if any file path contains invalid characters', async () => { const files: Record = { '/a.js': '1 + 2;', '/+-.js': '"hello world";' } - compileFiles(files, '/a.js', context) + await compileFiles(files, '/a.js', context) expect(parseError(context.errors)).toMatchInlineSnapshot( `"File path '/+-.js' must only contain alphanumeric chars and/or '_', '/', '.', '-'."` ) }) - it('returns IllegalCharInFilePathError if any file path contains invalid characters - verbose', () => { + it('returns IllegalCharInFilePathError if any file path contains invalid characters - verbose', async () => { const files: Record = { '/a.js': '1 + 2;', '/+-.js': '"hello world";' } - compileFiles(files, '/a.js', context) + await compileFiles(files, '/a.js', context) expect(parseError(context.errors, true)).toMatchInlineSnapshot(` "File path '/+-.js' must only contain alphanumeric chars and/or '_', '/', '.', '-'. Rename the offending file path to only use valid chars. @@ -105,23 +105,23 @@ describe('compileFiles', () => { `) }) - it('returns ConsecutiveSlashesInFilePathError if any file path contains consecutive slash characters', () => { + it('returns ConsecutiveSlashesInFilePathError if any file path contains consecutive slash characters', async () => { const files: Record = { '/a.js': '1 + 2;', '/dir//dir2/b.js': '"hello world";' } - compileFiles(files, '/a.js', context) + await compileFiles(files, '/a.js', context) expect(parseError(context.errors)).toMatchInlineSnapshot( `"File path '/dir//dir2/b.js' cannot contain consecutive slashes '//'."` ) }) - it('returns ConsecutiveSlashesInFilePathError if any file path contains consecutive slash characters - verbose', () => { + it('returns ConsecutiveSlashesInFilePathError if any file path contains consecutive slash characters - verbose', async () => { const files: Record = { '/a.js': '1 + 2;', '/dir//dir2/b.js': '"hello world";' } - compileFiles(files, '/a.js', context) + await compileFiles(files, '/a.js', context) expect(parseError(context.errors, true)).toMatchInlineSnapshot(` "File path '/dir//dir2/b.js' cannot contain consecutive slashes '//'. Remove consecutive slashes from the offending file path. @@ -129,15 +129,15 @@ describe('compileFiles', () => { `) }) - it('returns ModuleNotFoundError if entrypoint file does not exist', () => { + it('returns ModuleNotFoundError if entrypoint file does not exist', async () => { const files: Record = {} - compileFiles(files, '/a.js', context) + await compileFiles(files, '/a.js', context) expect(parseError(context.errors)).toMatchInlineSnapshot(`"Module '/a.js' not found."`) }) - it('returns ModuleNotFoundError if entrypoint file does not exist - verbose', () => { + it('returns ModuleNotFoundError if entrypoint file does not exist - verbose', async () => { const files: Record = {} - compileFiles(files, '/a.js', context) + await compileFiles(files, '/a.js', context) expect(parseError(context.errors, true)).toMatchInlineSnapshot(` "Module '/a.js' not found. You should check your import declarations, and ensure that all are valid modules. diff --git a/src/runner/__tests__/modules.ts b/src/runner/__tests__/modules.ts new file mode 100644 index 000000000..488cdab65 --- /dev/null +++ b/src/runner/__tests__/modules.ts @@ -0,0 +1,55 @@ +import { mockContext } from '../../mocks/context' +import { Chapter } from '../../types' +import { stripIndent } from '../../utils/formatters' +import { expectFinishedResult } from '../../utils/testing' +import { runCodeInSource } from '../sourceRunner' + +jest.mock('../../modules/loader/loaders') + +type DescribeCase = [string, Chapter[], string] +const describeCases: DescribeCase[] = [ + [ + 'javascript', + [ + Chapter.SOURCE_1, + Chapter.SOURCE_2, + Chapter.SOURCE_3, + Chapter.SOURCE_4, + Chapter.FULL_JS, + Chapter.FULL_TS, + Chapter.LIBRARY_PARSER + ], + 'import { foo } from "one_module"; foo();' + ], + [ + 'python', + [Chapter.PYTHON_1], + stripIndent` + from one_module import foo + foo() + ` + ], + [ + 'scheme', + [Chapter.SCHEME_1, Chapter.SCHEME_2, Chapter.SCHEME_3, Chapter.SCHEME_4, Chapter.FULL_SCHEME], + '(import "one_module" (foo)) (foo)' + ] +] + +describe.each(describeCases)( + 'Ensuring that %s chapters are able to load modules', + (_, chapters, code) => { + const chapterCases = chapters.map(chapterVal => { + const [chapterName] = Object.entries(Chapter).find(([, value]) => value === chapterVal)! + return [`Testing ${chapterName}`, chapterVal] as [string, Chapter] + }) + + test.each(chapterCases)('%s', async (_, chapter) => { + const context = mockContext(chapter) + const { result } = await runCodeInSource(code, context) + + expectFinishedResult(result) + expect(result.value).toEqual('foo') + }) + } +) diff --git a/src/runner/fullJSRunner.ts b/src/runner/fullJSRunner.ts index 81efc071d..ceedf567e 100644 --- a/src/runner/fullJSRunner.ts +++ b/src/runner/fullJSRunner.ts @@ -7,7 +7,6 @@ import type { Result } from '..' import { NATIVE_STORAGE_ID } from '../constants' import { RuntimeSourceError } from '../errors/runtimeSourceError' import type { ImportOptions } from '../modules/moduleTypes' -import hoistAndMergeImports from '../modules/preprocessor/transformers/hoistAndMergeImports' import { parse } from '../parser/parser' import { evallerReplacer, @@ -62,9 +61,6 @@ export async function fullJSRunner( ? [] : [...getBuiltins(context.nativeStorage), ...prelude] - // modules - hoistAndMergeImports(program) - // evaluate and create a separate block for preludes and builtins const preEvalProgram: es.Program = create.program([ ...preludeAndBuiltins, diff --git a/src/runner/sourceRunner.ts b/src/runner/sourceRunner.ts index 29ce2d6d4..4ac64147d 100644 --- a/src/runner/sourceRunner.ts +++ b/src/runner/sourceRunner.ts @@ -14,7 +14,6 @@ import { testForInfiniteLoop } from '../infiniteLoops/runtime' import { evaluateProgram as evaluate } from '../interpreter/interpreter' import { nonDetEvaluate } from '../interpreter/interpreter-non-det' import { transpileToLazy } from '../lazy/lazy' -import { ModuleNotFoundError } from '../modules/errors' import preprocessFileImports from '../modules/preprocessor' import { defaultAnalysisOptions } from '../modules/preprocessor/analyzer' import { defaultLinkerOptions } from '../modules/preprocessor/linker' @@ -24,25 +23,21 @@ import { callee, getEvaluationSteps, getRedex, - IStepperPropContents, + type IStepperPropContents, redexify } from '../stepper/stepper' import { sandboxedEval } from '../transpiler/evalContainer' import { transpile } from '../transpiler/transpiler' -import { Context, RecursivePartial, Scheduler, Variant } from '../types' +import { Chapter, type Context, type RecursivePartial, type Scheduler, Variant } from '../types' import { forceIt } from '../utils/operators' import { validateAndAnnotate } from '../validator/validator' import { compileForConcurrent } from '../vm/svml-compiler' import { runWithProgram } from '../vm/svml-machine' -import type { SourceFiles } from '../modules/moduleTypes' +import type { FileGetter } from '../modules/moduleTypes' +import { mapResult } from '../alt-langs/mapper' import { toSourceError } from './errors' import { fullJSRunner } from './fullJSRunner' -import { - determineExecutionMethod, - determineVariant, - hasVerboseErrors, - resolvedErrorPromise -} from './utils' +import { determineExecutionMethod, determineVariant, resolvedErrorPromise } from './utils' const DEFAULT_SOURCE_OPTIONS: Readonly = { scheduler: 'async', @@ -227,7 +222,7 @@ function runCSEMachine(program: es.Program, context: Context, options: IOptions) return CSEResultPromise(context, value) } -export async function sourceRunner( +async function sourceRunner( program: es.Program, context: Context, isVerboseErrorsEnabled: boolean, @@ -238,6 +233,14 @@ export async function sourceRunner( const theOptions = _.merge({ ...DEFAULT_SOURCE_OPTIONS }, options) context.variant = determineVariant(context, options) + if ( + context.chapter === Chapter.FULL_JS || + context.chapter === Chapter.FULL_TS || + context.chapter === Chapter.PYTHON_1 + ) { + return fullJSRunner(program, context, theOptions.importOptions) + } + validateAndAnnotate(program, context) if (context.errors.length > 0) { return resolvedErrorPromise @@ -288,19 +291,34 @@ export async function sourceRunner( return runInterpreter(program, context, theOptions) } +/** + * Returns both the Result of the evaluated program, as well as + * `verboseErrors`. + */ export async function sourceFilesRunner( - files: Partial>, + filesInput: FileGetter, entrypointFilePath: string, context: Context, options: RecursivePartial = {} -): Promise { - const entrypointCode = files[entrypointFilePath] - if (entrypointCode === undefined) { - context.errors.push(new ModuleNotFoundError(entrypointFilePath)) - return resolvedErrorPromise +): Promise<{ + result: Result + verboseErrors: boolean +}> { + const preprocessResult = await preprocessFileImports( + filesInput, + entrypointFilePath, + context, + options + ) + + if (!preprocessResult.ok) { + return { + result: { status: 'error' }, + verboseErrors: preprocessResult.verboseErrors + } } - const isVerboseErrorsEnabled = hasVerboseErrors(entrypointCode) + const { files, verboseErrors, program: preprocessedProgram } = preprocessResult context.variant = determineVariant(context, options) // FIXME: The type checker does not support the typing of multiple files, so @@ -308,7 +326,7 @@ export async function sourceFilesRunner( // involved in the program evaluation should be type-checked. Either way, // the type checker is currently not used at all so this is not very // urgent. - context.unTypecheckedCode.push(entrypointCode) + context.unTypecheckedCode.push(files[entrypointFilePath]) const currentCode = { files, @@ -317,16 +335,38 @@ export async function sourceFilesRunner( context.shouldIncreaseEvaluationTimeout = _.isEqual(previousCode, currentCode) previousCode = currentCode - const preprocessedProgram = await preprocessFileImports( - files as SourceFiles, - entrypointFilePath, + context.previousPrograms.unshift(preprocessedProgram) + + const result = await sourceRunner(preprocessedProgram, context, verboseErrors, options) + const resultMapper = mapResult(context) + + return { + result: resultMapper(result), + verboseErrors + } +} + +/** + * Useful for just running a single line of code with the given context + * However, if this single line of code is an import statement, + * then the FileGetter is necessary, otherwise all local imports will + * fail with ModuleNotFoundError + */ +export function runCodeInSource( + code: string, + context: Context, + options: RecursivePartial = {}, + defaultFilePath: string = '/default.js', + fileGetter?: FileGetter +) { + return sourceFilesRunner( + path => { + if (path === defaultFilePath) return Promise.resolve(code) + if (!fileGetter) return Promise.resolve(undefined) + return fileGetter(path) + }, + defaultFilePath, context, options ) - if (!preprocessedProgram) { - return resolvedErrorPromise - } - context.previousPrograms.unshift(preprocessedProgram) - - return sourceRunner(preprocessedProgram, context, isVerboseErrorsEnabled, options) } diff --git a/src/runner/utils.ts b/src/runner/utils.ts index cbff0e983..4b6cad602 100644 --- a/src/runner/utils.ts +++ b/src/runner/utils.ts @@ -1,10 +1,9 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ -import { DebuggerStatement, Literal, Program } from 'estree' +import type { DebuggerStatement, Program } from 'estree' -import { IOptions, Result } from '..' -import { parseAt } from '../parser/utils' +import type { IOptions, Result } from '..' import { areBreakpointsSet } from '../stdlib/inspector' -import { Context, RecursivePartial, Variant } from '../types' +import type { Context, RecursivePartial, Variant } from '../types' import { simple } from '../utils/walkers' // Context Utils @@ -83,14 +82,4 @@ export function determineExecutionMethod( // AST Utils -export function hasVerboseErrors(theCode: string): boolean { - const theProgramFirstExpression = parseAt(theCode, 0) - - if (theProgramFirstExpression && theProgramFirstExpression.type === 'Literal') { - return (theProgramFirstExpression as unknown as Literal).value === 'enable verbose' - } - - return false -} - export const resolvedErrorPromise = Promise.resolve({ status: 'error' } as Result) diff --git a/src/stepper/stepper.ts b/src/stepper/stepper.ts index f4ad26095..23be27b58 100644 --- a/src/stepper/stepper.ts +++ b/src/stepper/stepper.ts @@ -3303,7 +3303,7 @@ function evaluateImports(program: es.Program, context: Context) { try { const environment = currentEnvironment(context) - for (const [moduleName, nodes] of Object.entries(importNodeMap)) { + for (const [moduleName, nodes] of importNodeMap) { const functions = context.nativeStorage.loadedModules[moduleName] for (const node of nodes) { for (const spec of node.specifiers) { diff --git a/src/transpiler/transpiler.ts b/src/transpiler/transpiler.ts index a4cdebc51..9fc545d7b 100644 --- a/src/transpiler/transpiler.ts +++ b/src/transpiler/transpiler.ts @@ -1,12 +1,12 @@ /* eslint-disable @typescript-eslint/no-unused-vars */ import { generate } from 'astring' import type es from 'estree' -import { RawSourceMap, SourceMapGenerator } from 'source-map' +import { type RawSourceMap, SourceMapGenerator } from 'source-map' import { NATIVE_STORAGE_ID, UNKNOWN_LOCATION } from '../constants' import { Chapter, type Context, type NativeStorage, type Node, Variant } from '../types' import * as create from '../utils/ast/astCreator' -import { filterImportDeclarations } from '../utils/ast/helpers' +import { filterImportDeclarations, getImportedName } from '../utils/ast/helpers' import { getFunctionDeclarationNamesInProgram, getIdentifiersInNativeStorage, @@ -17,6 +17,7 @@ import { } from '../utils/uniqueIds' import { simple } from '../utils/walkers' import { checkForUndefinedVariables } from '../validator/validator' +import { isNamespaceSpecifier } from '../utils/ast/typeGuards' /** * This whole transpiler includes many many many many hacks to get stuff working. @@ -27,28 +28,18 @@ import { checkForUndefinedVariables } from '../validator/validator' export function transformImportDeclarations( program: es.Program, moduleExpr: es.Expression -): [es.VariableDeclaration[], es.Program['body']] { +): [es.VariableDeclaration[], Exclude[]] { const [importNodes, otherNodes] = filterImportDeclarations(program) - const declNodes = Object.entries(importNodes).flatMap(([moduleName, nodes]) => { + const declNodes = importNodes.flatMap((moduleName, nodes) => { const expr = create.memberExpression(moduleExpr, moduleName) return nodes.flatMap(({ specifiers }) => - specifiers.flatMap(spec => { - switch (spec.type) { - case 'ImportNamespaceSpecifier': - return create.constantDeclaration(spec.local.name, expr) - case 'ImportDefaultSpecifier': - return create.constantDeclaration( - spec.local.name, - create.memberExpression(expr, 'default') - ) - case 'ImportSpecifier': - return create.constantDeclaration( - spec.local.name, - create.memberExpression(expr, spec.imported.name) - ) - } - }) + specifiers.map(spec => + create.constantDeclaration( + spec.local.name, + isNamespaceSpecifier(spec) ? expr : create.memberExpression(expr, getImportedName(spec)) + ) + ) ) }) @@ -433,14 +424,15 @@ function transpileToSource( context: Context, skipUndefined: boolean ): TranspiledResult { + if (program.body.length === 0) { + return { transpiled: '' } + } + const usedIdentifiers = new Set([ ...getIdentifiersInProgram(program), ...getIdentifiersInNativeStorage(context.nativeStorage) ]) const globalIds = getNativeIds(program, usedIdentifiers) - if (program.body.length === 0) { - return { transpiled: '' } - } const functionsToStringMap = generateFunctionsToStringMap(program) @@ -460,17 +452,17 @@ function transpileToSource( program, create.memberExpression(globalIds.native, 'loadedModules') ) + program.body = (importNodes as es.Program['body']).concat(otherNodes) getGloballyDeclaredIdentifiers(program).forEach(id => context.nativeStorage.previousProgramsIdentifiers.add(id) ) - const statements = program.body as es.Statement[] const newStatements = [ ...getDeclarationsToAccessTranspilerInternals(globalIds), evallerReplacer(globalIds.native, usedIdentifiers), create.expressionStatement(create.identifier('undefined')), - ...statements + ...(program.body as es.Statement[]) ] program.body = @@ -499,7 +491,7 @@ function transpileToFullJS( const [importNodes, otherNodes] = transformImportDeclarations( program, - create.memberExpression(globalIds.native, 'loadedModules') + create.memberExpression(create.identifier(NATIVE_STORAGE_ID), 'loadedModules') ) program.body = (importNodes as es.Program['body']).concat(otherNodes) diff --git a/src/utils/ast/dummyAstCreator.ts b/src/utils/ast/dummyAstCreator.ts index 9f6c5cf2e..d6ec56e57 100644 --- a/src/utils/ast/dummyAstCreator.ts +++ b/src/utils/ast/dummyAstCreator.ts @@ -1,6 +1,6 @@ -import * as es from 'estree' +import type es from 'estree' -import { BlockExpression } from '../../types' +import type { BlockExpression } from '../../types' const DUMMY_STRING = '__DUMMY__' const DUMMY_UNARY_OPERATOR = '!' diff --git a/src/utils/ast/helpers.ts b/src/utils/ast/helpers.ts index 2c204128b..59b085bc0 100644 --- a/src/utils/ast/helpers.ts +++ b/src/utils/ast/helpers.ts @@ -2,8 +2,19 @@ import type es from 'estree' import assert from '../assert' import { simple } from '../walkers' +import { ArrayMap } from '../dict' import { isImportDeclaration, isVariableDeclaration } from './typeGuards' +export function getModuleDeclarationSource( + node: Exclude +): string { + assert( + typeof node.source?.value === 'string', + `Expected ${node.type} to have a source value of type string, got ${node.source?.value}` + ) + return node.source.value +} + /** * Filters out all import declarations from a program, and sorts them by * the module they import from @@ -11,28 +22,19 @@ import { isImportDeclaration, isVariableDeclaration } from './typeGuards' export function filterImportDeclarations({ body }: es.Program): [ - Record, + ArrayMap, Exclude[] ] { return body.reduce( ([importNodes, otherNodes], node) => { if (!isImportDeclaration(node)) return [importNodes, [...otherNodes, node]] - const moduleName = node.source.value - assert( - typeof moduleName === 'string', - `Expected import declaration to have source of type string, got ${moduleName}` - ) - - if (!(moduleName in importNodes)) { - importNodes[moduleName] = [] - } - - importNodes[moduleName].push(node) + const moduleName = getModuleDeclarationSource(node) + importNodes.add(moduleName, node) return [importNodes, otherNodes] }, - [{}, []] as [ - Record, + [new ArrayMap(), []] as [ + ArrayMap, Exclude[] ] ) @@ -50,10 +52,23 @@ export function extractIdsFromPattern(pattern: es.Pattern) { return identifiers } -export function getIdsFromDeclaration(decl: es.Declaration) { - return isVariableDeclaration(decl) +export function getIdsFromDeclaration( + decl: es.Declaration, + allowNull: true +): (es.Identifier | null)[] +export function getIdsFromDeclaration(decl: es.Declaration, allowNull?: false): es.Identifier[] +export function getIdsFromDeclaration(decl: es.Declaration, allowNull?: boolean) { + const rawIds = isVariableDeclaration(decl) ? decl.declarations.flatMap(({ id }) => extractIdsFromPattern(id)) : [decl.id] + + if (!allowNull) { + rawIds.forEach(each => { + assert(each !== null, 'Encountered a null identifier!') + }) + } + + return rawIds } export const getImportedName = ( @@ -70,13 +85,3 @@ export const getImportedName = ( return spec.local.name } } - -export function getModuleDeclarationSource( - node: Exclude -): string { - assert( - typeof node.source?.value === 'string', - `Expected ${node.type} to have a source value of type string, got ${node.source?.value}` - ) - return node.source.value -} diff --git a/src/utils/testing.ts b/src/utils/testing.ts index b30ab2fb2..a2f049632 100644 --- a/src/utils/testing.ts +++ b/src/utils/testing.ts @@ -9,7 +9,15 @@ import { mockContext } from '../mocks/context' import { ImportOptions } from '../modules/moduleTypes' import { parse } from '../parser/parser' import { transpile } from '../transpiler/transpiler' -import { Chapter, Context, CustomBuiltIns, SourceError, Value, Variant } from '../types' +import { + Chapter, + Context, + CustomBuiltIns, + SourceError, + Value, + Variant, + type Finished +} from '../types' import { stringify } from './stringify' export interface CodeSnippetTestCase { @@ -371,3 +379,7 @@ export async function expectNativeToTimeoutAndError(code: string, timeout: numbe export function asMockedFunc any>(func: T) { return func as MockedFunction } + +export function expectFinishedResult(result: Result): asserts result is Finished { + expect(result.status).toEqual('finished') +} diff --git a/yarn.lock b/yarn.lock index 96d4dea95..1c41c0f5a 100644 --- a/yarn.lock +++ b/yarn.lock @@ -1042,6 +1042,11 @@ resolved "https://registry.yarnpkg.com/@bcoe/v8-coverage/-/v8-coverage-0.2.3.tgz#75a2e8b51cb758a7553d6804a5932d7aace75c39" integrity sha512-0hYQ8SB4Db5zvZB4axdMHGwEaQjkZzFjQiN9LVYvIFB2nSUHW9tYpxWriPrWDASIxiaXax83REcLxuSdnGPZtw== +"@commander-js/extra-typings@^12.0.1": + version "12.0.1" + resolved "https://registry.yarnpkg.com/@commander-js/extra-typings/-/extra-typings-12.0.1.tgz#4a74a9ccf1d19ef24e71df59359c6d90a605a469" + integrity sha512-OvkMobb1eMqOCuJdbuSin/KJkkZr7n24/UNV+Lcz/0Dhepf3r2p9PaGwpRpAWej7A+gQnny4h8mGhpFl4giKkg== + "@eslint-community/eslint-utils@^4.2.0", "@eslint-community/eslint-utils@^4.4.0": version "4.4.0" resolved "https://registry.yarnpkg.com/@eslint-community/eslint-utils/-/eslint-utils-4.4.0.tgz#a23514e8fb9af1269d5f7788aa556798d61c6b59" @@ -2321,6 +2326,11 @@ combined-stream@^1.0.6, combined-stream@^1.0.8, combined-stream@~1.0.6: dependencies: delayed-stream "~1.0.0" +commander@^12.0.0: + version "12.0.0" + resolved "https://registry.yarnpkg.com/commander/-/commander-12.0.0.tgz#b929db6df8546080adfd004ab215ed48cf6f2592" + integrity sha512-MwVNWlYjDTtOjX5PiD7o5pK0UrFU/OYgcJfjjK4RaHZETNtjJqrZa9Y9ds88+A+f+d5lv+561eZ+yCKoS3gbAA== + concat-map@0.0.1: version "0.0.1" resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b" @@ -4668,11 +4678,6 @@ node-abi@^3.26.0, node-abi@^3.3.0: dependencies: semver "^7.3.5" -node-getopt@^0.3.2: - version "0.3.2" - resolved "https://registry.npmjs.org/node-getopt/-/node-getopt-0.3.2.tgz" - integrity sha512-yqkmYrMbK1wPrfz7mgeYvA4tBperLg9FQ4S3Sau3nSAkpOA0x0zC8nQ1siBwozy1f4SE8vq2n1WKv99r+PCa1Q== - node-gyp@^9.2.0: version "9.4.1" resolved "https://registry.yarnpkg.com/node-gyp/-/node-gyp-9.4.1.tgz#8a1023e0d6766ecb52764cc3a734b36ff275e185"